diff --git a/boot/initramfs/root/init b/boot/initramfs/root/init index 9556f126..86f05636 100755 --- a/boot/initramfs/root/init +++ b/boot/initramfs/root/init @@ -10,6 +10,7 @@ mount -t devtmpfs -o mode=0755,nosuid dev /dev exec /dev/console 2>&1 stop() { + echo "<3>orchard initrd: $1; rescue shell" > /dev/kmsg echo "orchard initrd: $1" echo 'Starting a shell on the console; the system did not start.' exec sh @@ -31,8 +32,11 @@ done [ ! -e /etc/orchard/usb-modules ] || for module in $(cat /etc/orchard/usb-modules); do modprobe "$module" || echo "orchard initrd: could not load $module; no USB" done +echo '<6>orchard initrd: storage load begin' > /dev/kmsg for module in $(cat /etc/orchard/modules); do + echo "<6>orchard initrd: modprobe $module begin" > /dev/kmsg modprobe "$module" || stop "could not load $module" + echo "<6>orchard initrd: modprobe $module complete" > /dev/kmsg done # An install boot: orchard's add-on, appended to this initrd by the CLI, writes the @@ -87,8 +91,35 @@ done mkdir -p /newroot mount ${fstype:+-t "$fstype"} -o "$mode${flags:+,$flags}" "$device" /newroot || stop "could not mount $device" +echo '<6>orchard initrd: root mounted' > /dev/kmsg [ -x "/newroot$init" ] || [ -L "/newroot$init" ] || stop "the root has no $init" +# Preserve the running boot's *pristine* tree before any package update can +# replace /boot or same-release module assets. The matching initrd embeds cold +# DTBs; warm boots name their retained stage instead. Never infer it by uname. +mkdir -p /run /newroot/run +mount -t tmpfs -o mode=0755,nosuid,nodev tmpfs /run +mkdir -p /run/orchard-kexec +warm_pristine= +for argument in $(cat /proc/cmdline); do + case "$argument" in orchard_warm.dtb=*) warm_pristine=${argument#orchard_warm.dtb=} ;; esac +done +if [ -n "$warm_pristine" ]; then + case "$warm_pristine" in + /var/tmp/orchard-warm.*/pristine.dtb) + case "$warm_pristine" in *..*) stop 'invalid warm DT provenance' ;; esac + cp "/newroot$warm_pristine" /run/orchard-kexec/pristine.dtb || stop 'missing warm pristine DT' ;; + *) stop 'invalid warm DT provenance' ;; + esac +elif [ -f /etc/orchard/pristine-map ]; then + board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + while read compatible file; do + [ "$compatible" != "$board" ] || cp "/etc/orchard/dtbs/$file" /run/orchard-kexec/pristine.dtb +done < /etc/orchard/pristine-map +fi +cat /proc/sys/kernel/random/boot_id > /run/orchard-kexec/boot-id +mount --move /run /newroot/run || stop 'cannot retain boot provenance' + # USB for the host: network (usb0) and a serial console (ttyGS0), named after # the system, so `orchard devices` sees it. What runs on them is the system's. usb_gadget() { @@ -97,13 +128,29 @@ usb_gadget() { [ ! -e "$g" ] || return 0 name=$(sed -n 's/^NAME=//p' /newroot/etc/os-release /newroot/usr/lib/os-release 2>/dev/null | head -1 | tr -d "\"'") board=$(tr '\000' '\n' < /proc/device-tree/compatible | head -1) + # Opt-in compatibility identity in macOS's 2400 mA capability table. + # This requests a host grant; it does not set a charger limit. Keep the + # generic composite personality (no iOS/usbmux or preferred-config API). + pid=0x4145 + device_class=0x00 device_subclass=0x00 device_protocol=0x00 + if [ "$board" = apple,j71t ] && + [ "$(cat /newroot/etc/orchard/mac-usb-current 2>/dev/null)" = 1 ]; then + pid=0x1014 + # Standard IAD device: macOS defers its CDC composite match here, + # allowing the generic composite driver to deliver the grant. + device_class=0xef device_subclass=0x02 device_protocol=0x01 + fi + serial=$(sh /etc/orchard/usb-identity) || return 1 mkdir "$g" && echo 0x05ac > "$g/idVendor" && - echo 0x4145 > "$g/idProduct" && + echo "$pid" > "$g/idProduct" && + echo "$device_class" > "$g/bDeviceClass" && + echo "$device_subclass" > "$g/bDeviceSubClass" && + echo "$device_protocol" > "$g/bDeviceProtocol" && mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" && echo 'Orchard Linux' > "$g/strings/0x409/manufacturer" && echo "${name:-Linux}" > "$g/strings/0x409/product" && - echo "BOOT:$(cat /proc/sys/kernel/random/boot_id) BOARD:${board#apple,}" > "$g/strings/0x409/serialnumber" && + echo "$serial" > "$g/strings/0x409/serialnumber" && echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" && echo 500 > "$g/configs/c.1/MaxPower" && mkdir "$g/functions/ncm.usb0" "$g/functions/acm.usb0" && @@ -112,6 +159,7 @@ usb_gadget() { basename "$(ls -d /sys/class/udc/* | head -1)" > "$g/UDC" } usb_gadget || echo 'orchard initrd: no USB gadget; continuing without USB' +echo '<6>orchard initrd: USB setup complete; switching root' > /dev/kmsg for name in dev proc sys; do mount --move "/$name" "/newroot/$name" diff --git a/boot/m1n1/README.md b/boot/m1n1/README.md index bd78f93b..0a7c6e1c 100644 --- a/boot/m1n1/README.md +++ b/boot/m1n1/README.md @@ -28,3 +28,16 @@ gives the same tree as Hoolock's tip before updating `sources.toml`. Scratch stays in ignored `boot/m1n1/.work//`. `--resume` reuses it only if the recorded patch set is unchanged. + +## Resident PSCI on A7-A9 + +Orchard patch 0015 adds the resident EL3 PSCI monitor (CPU_ON, CPU_OFF, +AFFINITY_INFO); 0016-0019 make it the default whenever m1n1 is entered at EL3 +and add SYSTEM_RESET (warm reboot back into m1n1, which boots the default BLS +entry with the last launch's DTB) and SYSTEM_OFF (halt). Any PSCI-aware +kernel reboots, powers off and kexecs through its ordinary paths; m1n1 takes +IPI-stopped CPUs back by trapping EL1 WFI/WFE to EL3. EL1/EL2 entries keep +the spin table. Payload variables (append to `m1n1.bin`, e.g. with +`psci-loader`): `psci=0` turns PSCI off, `psci_reset=0` keeps only the CPU +lifecycle (no WFI/WFE traps), `psci_reset=proxy` stops in the proxy after a +reset. 0020 marks retained hardware in the device tree after a re-entry. diff --git a/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch b/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch new file mode 100644 index 00000000..4eb87533 --- /dev/null +++ b/boot/m1n1/patches/orchard/0015-psci-Add-opt-in-resident-EL3-CPU-lifecycle-for-A7-A9.patch @@ -0,0 +1,880 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sat, 7 Nov 2026 15:00:00 +0100 +Subject: [PATCH] psci: Add opt-in resident EL3 CPU lifecycle for A7-A9 + +On A7-A9 m1n1 is entered at EL3 and stays resident: its image and the +per-CPU EL3 stacks are memreserved, and VBAR_EL3 points at its vectors +while Linux runs at EL1. Use that to give Linux PSCI 1.0 CPU_ON, CPU_OFF +and AFFINITY_INFO (plus VERSION, FEATURES and MIGRATE_INFO_TYPE). The +spin table has no cpu_die, so without this the kernel refuses kexec on +an SMP system, can't offline CPUs, and a crash kernel inherits CPUs +stuck in the old kernel. + +CPU_OFF does not power the core down. It cleans and invalidates its L1D, +stops its timers, and waits in WFE at EL3 (MMU off) on a one-line +mailbox. CPU_ON posts an entry point; the core leaves with a clean +I-cache and TLB, SCTLR_EL1 at its reset value, DAIF masked and SP_EL3 on +a static per-CPU stack. SYSTEM_OFF/SYSTEM_RESET, CPU_SUSPEND and +SMCCC_VERSION return NOT_SUPPORTED, so callers fall back to their own +methods and assume SMCCC 1.0. + +Opt-in with the payload variable psci=1, and only used when m1n1 itself +was entered at EL3. kboot then parks the secondaries in the pen (a core +that doesn't park is pruned), sets enable-method = "psci" on every CPU, +adds /psci (method = "smc") and records the re-entry address and the +original boot_args in /chosen. + +Re-entry: _reentry can be jumped to from a kexec'd trampoline (EL1, MMU +off, x0 = boot_args copy). It drops this image's lines from the caches, +flags the re-entry and restarts m1n1, which then skips the (stale) +payload area and runs the proxy. smp_start_secondaries() takes cores +back from the pen with CPU_ON, and the mailboxes and the psci=1 setting +live in .data so they survive the .bss clear. + +Rebase the resident boot shim onto Orchard's complete m1n1 series +without changing its runtime behavior. The payload variable is not a +kernel command-line option; a bare proxy loader also needs it appended. + +Assisted-by: LLM +--- + Makefile | 1 + + src/kboot.c | 56 +++++++++++++- + src/main.c | 8 +- + src/payload.c | 3 + + src/psci.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + src/psci.h | 78 ++++++++++++++++++++ + src/psci_asm.S | 170 ++++++++++++++++++++++++++++++++++++++++++ + src/smp.c | 73 ++++++++++++++++++ + src/smp.h | 1 + + src/start.S | 6 +- + src/startup.c | 6 +- + 11 files changed, 626 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index bb72266..4aefb37 100644 +--- a/Makefile ++++ b/Makefile +@@ -166,6 +166,7 @@ OBJECTS := \ + pcie.o \ + pmgr.o \ + proxy.o \ ++ psci.o psci_asm.o \ + ringbuffer.o \ + rtkit.o \ + sart.o \ +diff --git a/src/kboot.c b/src/kboot.c +index b75bb45..2316709 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -21,6 +21,7 @@ + #include "memory.h" + #include "pcie.h" + #include "pmgr.h" ++#include "psci.h" + #include "sep.h" + #include "sio.h" + #include "smc.h" +@@ -574,6 +575,53 @@ static int dt_set_smbios(void) + return 0; + } + ++/* ++ * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * (see psci.c) instead of the spin table, so the kernel can offline them ++ * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. ++ */ ++static int dt_set_psci(void) ++{ ++ static const char compat[] = "arm,psci-1.0\0arm,psci-0.2"; ++ int node; ++ ++ if (!psci_active()) { ++ printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ return 0; ++ } ++ ++ int cpus = fdt_path_offset(dt, "/cpus"); ++ if (cpus < 0) ++ bail("FDT: /cpus node not found in devtree\n"); ++ ++ fdt_for_each_subnode(node, dt, cpus) ++ { ++ if (strncmp(fdt_get_name(dt, node, NULL), "cpu@", 4)) ++ continue; ++ if (fdt_setprop_string(dt, node, "enable-method", "psci")) ++ bail("FDT: couldn't set enable-method\n"); ++ } ++ ++ node = fdt_path_offset(dt, "/psci"); ++ if (node < 0) ++ node = fdt_add_subnode(dt, 0, "psci"); ++ if (node < 0) ++ bail("FDT: couldn't add /psci\n"); ++ if (fdt_setprop(dt, node, "compatible", compat, sizeof(compat)) || ++ fdt_setprop_string(dt, node, "method", "smc")) ++ bail("FDT: couldn't fill /psci\n"); ++ ++ node = fdt_path_offset(dt, "/chosen"); ++ if (node < 0) ++ bail("FDT: /chosen node not found in devtree\n"); ++ if (fdt_setprop_u64(dt, node, "linux-a9,m1n1-reentry", (u64)_reentry) || ++ fdt_setprop(dt, node, "linux-a9,m1n1-boot-args", &cur_boot_args, sizeof(cur_boot_args))) ++ bail("FDT: couldn't set the m1n1 re-entry properties\n"); ++ ++ printf("PSCI: on, enable-method = \"psci\", m1n1 re-entry at %p\n", _reentry); ++ return 0; ++} ++ + static int dt_set_cpus(void) + { + int ret = 0; +@@ -587,6 +635,10 @@ static int dt_set_cpus(void) + if (!pruned_phandles) + bail("FDT: out of memory\n"); + ++ /* Secondaries wait in the PSCI pen; one that can't get there is pruned below */ ++ if (psci_active()) ++ printf("PSCI: parked %d secondary CPU(s)\n", smp_park_secondaries_psci()); ++ + /* Prune CPU nodes */ + int node, cpu = 0; + for (node = fdt_first_subnode(dt, cpus); node >= 0;) { +@@ -618,7 +670,7 @@ static int dt_set_cpus(void) + } else { + printf("FDT: Reserving stack for CPU %d 0x%lx\n", cpu, (uint64_t)secondary_stacks[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks[cpu], SECONDARY_STACK_SIZE); +- if (has_el3()) { ++ if (has_el3() && secondary_stacks_el3[cpu]) { + printf("FDT: Reserving EL3 stack for CPU %d 0x%lx\n", cpu, + (uint64_t)secondary_stacks_el3[cpu]); + fdt_add_mem_rsv(dt, (uint64_t)secondary_stacks_el3[cpu], SECONDARY_STACK_SIZE); +@@ -3206,6 +3258,8 @@ int kboot_prepare_dt(void *fdt) + return -1; + if (dt_set_cpus()) + return -1; ++ if (dt_set_psci()) ++ return -1; + if (dt_set_mac_addresses()) + return -1; + if (dt_set_wifi()) +diff --git a/src/main.c b/src/main.c +index d4e828c..3f32380 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -21,6 +21,7 @@ + #include "pcie.h" + #include "pmgr.h" + #include "sep.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "tps6598x.h" +@@ -117,9 +118,12 @@ void run_actions(void) + #endif + #endif + +- printf("Checking for payloads...\n"); ++ if (psci_reentered()) ++ printf("Re-entered from Linux: whatever follows m1n1 is stale, not loading payloads\n"); ++ else ++ printf("Checking for payloads...\n"); + +- if (payload_run() == 0) { ++ if (!psci_reentered() && payload_run() == 0) { + printf("Valid payload found\n"); + return; + } +diff --git a/src/payload.c b/src/payload.c +index 5b1547f..4ffc9e2 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -12,6 +12,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "utils.h" + +@@ -205,6 +206,8 @@ static bool check_var(u8 **p) + mitigations_configure(val); + } else if (IS_VAR("tso=")) { + enable_tso = val[0] == '1'; ++ } else if (IS_VAR("psci=")) { ++ psci_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +new file mode 100644 +index 0000000..2b8ca0d +--- /dev/null ++++ b/src/psci.c +@@ -0,0 +1,229 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++#include "smp.h" ++#include "types.h" ++#include "utils.h" ++ ++struct psci_cpu { ++ volatile u64 state; ++ volatile u64 entry; ++ volatile u64 context; ++ u64 pad[PSCI_CPU_SIZE / 8 - 3]; ++}; ++ ++_Static_assert(sizeof(struct psci_cpu) == PSCI_CPU_SIZE, "psci_cpu layout"); ++ ++/* ++ * Mailboxes, one cache line each, only ever accessed at EL3 with the MMU off. ++ * They live in .data rather than .bss so that a parked CPU's state survives ++ * m1n1 being re-entered from Linux (which clears .bss). ++ */ ++struct psci_cpu psci_cpus[PSCI_MAX_CPUS] ++ __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* Written with the MMU off only (_reentry sets reentry_flag, the first member) */ ++struct { ++ u64 reentry_flag; ++ u64 boot_el3; ++} psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++/* psci=1 from the payload, kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))); ++ ++/* EL3 stacks a CPU gets when it leaves the pen */ ++static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); ++ ++static bool psci_have_el3; ++static bool reentered; ++ ++/* EL3 side */ ++ ++static inline void psci_line_sync(volatile void *p) ++{ ++ __asm__ volatile("dc civac, %0\n\tdsb sy" ::"r"(p) : "memory"); ++} ++ ++static int psci_mpidr_index(u64 mpidr) ++{ ++ mpidr &= 0xff00ffffffUL; ++ ++ /* A7-A9: one cluster, Aff0 is the core */ ++ if (mpidr >= PSCI_MAX_CPUS) ++ return -1; ++ ++ return mpidr; ++} ++ ++static s64 psci_cpu_off(void) ++{ ++ int idx = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (idx < 0) ++ return PSCI_RET_DENIED; ++ ++ psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); ++} ++ ++static s64 psci_cpu_on(u64 mpidr, u64 entry, u64 context) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || !entry) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ struct psci_cpu *c = &psci_cpus[idx]; ++ ++ psci_line_sync(c); ++ switch (c->state) { ++ case PSCI_AFF_OFF: ++ break; ++ case PSCI_AFF_ON_PENDING: ++ return PSCI_RET_ON_PENDING; ++ default: ++ /* Running, or never seen by us (still in m1n1 or never started) */ ++ return PSCI_RET_ALREADY_ON; ++ } ++ ++ /* State before entry: the pen sets ON as soon as it sees the entry */ ++ c->context = context; ++ c->state = PSCI_AFF_ON_PENDING; ++ psci_line_sync(c); ++ c->entry = entry; ++ psci_line_sync(c); ++ sysop("sev"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++static s64 psci_affinity(u64 mpidr, u64 level) ++{ ++ int idx = psci_mpidr_index(mpidr); ++ ++ if (idx < 0 || level) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ psci_line_sync(&psci_cpus[idx]); ++ return psci_cpus[idx].state; ++} ++ ++static s64 psci_features(u32 fid) ++{ ++ switch (fid) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ case PSCI_0_2_FN_CPU_OFF: ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return PSCI_RET_SUCCESS; ++ default: ++ /* ++ * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make ++ * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) ++{ ++ u32 fn = fid; ++ ++ /* SMC32 calls carry 32-bit arguments */ ++ if (!(fn & 0x40000000)) { ++ a1 = (u32)a1; ++ a2 = (u32)a2; ++ a3 = (u32)a3; ++ } ++ ++ switch (fn) { ++ case PSCI_0_2_FN_PSCI_VERSION: ++ return 0x10000; /* 1.0 */ ++ case PSCI_0_2_FN_CPU_OFF: ++ return psci_cpu_off(); ++ case PSCI_0_2_FN_CPU_ON: ++ case PSCI_0_2_FN64_CPU_ON: ++ return psci_cpu_on(a1, a2, a3); ++ case PSCI_0_2_FN_AFFINITY_INFO: ++ case PSCI_0_2_FN64_AFFINITY_INFO: ++ return psci_affinity(a1, a2); ++ case PSCI_0_2_FN_MIGRATE_INFO_TYPE: ++ return 2; /* no trusted OS, nothing to migrate */ ++ case PSCI_1_0_FN_PSCI_FEATURES: ++ return psci_features(a1); ++ default: ++ /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ ++ return PSCI_RET_NOT_SUPPORTED; ++ } ++} ++ ++/* EL1 side */ ++ ++void psci_note_boot_el(u64 el) ++{ ++ /* MMU still off: these .data accesses are uncached, like the writers' */ ++ if (psci_persist.reentry_flag) { ++ reentered = true; ++ psci_persist.reentry_flag = 0; ++ } ++ if (el == 3) ++ psci_persist.boot_el3 = 1; ++ ++ psci_have_el3 = psci_persist.boot_el3 && has_el3(); ++} ++ ++void psci_configure(const char *val) ++{ ++ psci_mode = val[0] == '1'; ++} ++ ++bool psci_active(void) ++{ ++ return psci_mode && psci_have_el3; ++} ++ ++bool psci_reentered(void) ++{ ++ return reentered; ++} ++ ++int psci_affinity_info(u64 mpidr) ++{ ++ return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); ++} ++ ++static void psci_self_off(void) ++{ ++ /* Only returns on failure; the core then stays in m1n1's loop */ ++ psci_smc(PSCI_0_2_FN_CPU_OFF, 0, 0, 0); ++} ++ ++int psci_park_cpu(int cpu, u64 mpidr) ++{ ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ ++ smp_call0(cpu, psci_self_off); ++ ++ for (int i = 0; i < 100; i++) { ++ if (psci_affinity_info(mpidr) == PSCI_AFF_OFF) ++ return 0; ++ udelay(1000); ++ } ++ ++ return -1; ++} ++ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top) ++{ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return -1; ++ ++ s64 ret = psci_smc(PSCI_0_2_FN64_CPU_ON, mpidr, (u64)psci_secondary_entry, ++ (u64)stack_top | (u64)index); ++ ++ return ret == PSCI_RET_SUCCESS ? 0 : -1; ++} +diff --git a/src/psci.h b/src/psci.h +new file mode 100644 +index 0000000..6769ebe +--- /dev/null ++++ b/src/psci.h +@@ -0,0 +1,78 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#ifndef PSCI_H ++#define PSCI_H ++ ++#ifndef __ASSEMBLER__ ++#include "types.h" ++#endif ++ ++/* ++ * Minimal PSCI 1.0 for SoCs where m1n1 owns EL3 (A7-A9 entered at EL3). ++ * Only CPU_ON/CPU_OFF/AFFINITY_INFO and the discovery calls are implemented. ++ * A CPU that calls CPU_OFF is not powered down: it waits at EL3, MMU and ++ * caches off, until CPU_ON hands it a new EL1 entry point. That is what the ++ * kernel needs to offline CPUs for kexec and CPU hotplug. ++ */ ++ ++#define PSCI_0_2_FN_PSCI_VERSION 0x84000000 ++#define PSCI_0_2_FN_CPU_SUSPEND 0x84000001 ++#define PSCI_0_2_FN_CPU_OFF 0x84000002 ++#define PSCI_0_2_FN_CPU_ON 0x84000003 ++#define PSCI_0_2_FN_AFFINITY_INFO 0x84000004 ++#define PSCI_0_2_FN_MIGRATE_INFO_TYPE 0x84000006 ++#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 ++#define PSCI_0_2_FN_SYSTEM_RESET 0x84000009 ++#define PSCI_1_0_FN_PSCI_FEATURES 0x8400000a ++#define PSCI_0_2_FN64_CPU_ON 0xc4000003 ++#define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 ++ ++#define PSCI_RET_SUCCESS 0 ++#define PSCI_RET_NOT_SUPPORTED -1 ++#define PSCI_RET_INVALID_PARAMS -2 ++#define PSCI_RET_DENIED -3 ++#define PSCI_RET_ALREADY_ON -4 ++#define PSCI_RET_ON_PENDING -5 ++#define PSCI_RET_INTERNAL_FAILURE -6 ++ ++#define PSCI_AFF_ON 0 ++#define PSCI_AFF_OFF 1 ++#define PSCI_AFF_ON_PENDING 2 ++ ++#define PSCI_MAX_CPUS 4 ++#define PSCI_STACK_SIZE 0x4000 ++ ++/* Layout shared with psci_asm.S */ ++#define PSCI_CPU_STATE 0x00 ++#define PSCI_CPU_ENTRY 0x08 ++#define PSCI_CPU_CONTEXT 0x10 ++#define PSCI_CPU_SIZE 0x80 ++ ++#ifndef __ASSEMBLER__ ++ ++/* Payload variable psci=1 */ ++void psci_configure(const char *val); ++/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++bool psci_active(void); ++/* Called once from _start_c with the EL m1n1 was entered at */ ++void psci_note_boot_el(u64 el); ++/* Set when m1n1 was re-entered from Linux (see psci_asm.S _reentry) */ ++bool psci_reentered(void); ++ ++/* EL1 side: SMC into our own EL3 */ ++s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3); ++int psci_affinity_info(u64 mpidr); ++/* Park a secondary that is spinning in smp_secondary_entry() at EL3 */ ++int psci_park_cpu(int cpu, u64 mpidr); ++/* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ ++int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); ++ ++/* EL3 side */ ++u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); ++void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_secondary_entry(void); ++void _reentry(void); ++ ++#endif ++ ++#endif +diff --git a/src/psci_asm.S b/src/psci_asm.S +new file mode 100644 +index 0000000..14f842f +--- /dev/null ++++ b/src/psci_asm.S +@@ -0,0 +1,170 @@ ++/* SPDX-License-Identifier: MIT */ ++ ++#include "psci.h" ++ ++#define DCSW_OP_DCCISW 0x1 // memory.h ++ ++.extern _v_sp0_sync ++.extern _start ++.extern _cpu_reset_c ++.extern psci_handle ++.extern psci_persist ++ ++.text ++ ++/* ++ * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) ++ * goes to the PSCI dispatcher; everything else, and every exception taken ++ * at EL1/EL2, takes the usual path. ++ */ ++.globl _v_lower_sync ++.type _v_lower_sync, @function ++_v_lower_sync: ++ stp x16, x17, [sp, #-16]! ++ mrs x16, CurrentEL ++ cmp x16, #0xc ++ b.ne 1f ++ mrs x16, esr_el3 ++ lsr x17, x16, #26 ++ cmp x17, #0x17 // EC: SMC from AArch64 ++ b.ne 1f ++ tst x16, #0xffff // imm16 == 0: SMCCC ++ b.ne 1f ++ ldp x16, x17, [sp], #16 ++ b psci_smc_entry ++1: ++ ldp x16, x17, [sp], #16 ++ b _v_sp0_sync ++ ++/* ++ * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the ++ * caller. x0-x3 go straight to psci_handle(). ELR/SPSR are saved in case ++ * anything below takes an exception at EL3. ++ */ ++.type psci_smc_entry, @function ++psci_smc_entry: ++ stp x29, x30, [sp, #-16]! ++ mrs x29, elr_el3 ++ mrs x30, spsr_el3 ++ stp x29, x30, [sp, #-16]! ++ str x18, [sp, #-16]! ++ bl psci_handle ++ ldr x18, [sp], #16 ++ ldp x29, x30, [sp], #16 ++ msr elr_el3, x29 ++ msr spsr_el3, x30 ++ ldp x29, x30, [sp], #16 ++ eret ++ ++/* ++ * psci_pen(struct psci_cpu *me, u64 stack_top): CPU_OFF. Never returns. ++ * ++ * The core is not powered down. It writes its L1D back, marks itself OFF and ++ * waits in WFE at EL3 (MMU off, so every access here is non-cacheable) for ++ * CPU_ON to post an entry point. It then enters EL1 there with the MMU off, ++ * a clean I-cache and TLB, and x0 = context id, like a core coming out of ++ * reset. SP_EL3 is reset to this core's own static stack. ++ */ ++.globl psci_pen ++.type psci_pen, @function ++psci_pen: ++ mov x19, x0 ++ mov x20, x1 ++ ++ // A pending timer interrupt must not keep waking WFE. ++ msr cntp_ctl_el0, xzr ++ msr cntv_ctl_el0, xzr ++ isb ++ ++ // Clean and invalidate this core's L1D (L2 is shared and coherent). ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_level1 ++ dsb sy ++ ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_OFF ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ sev ++ ++1: ++ wfe ++ dc civac, x19 ++ dsb sy ++ ldr x2, [x19, #PSCI_CPU_ENTRY] ++ cbz x2, 1b ++ ++ ldr x0, [x19, #PSCI_CPU_CONTEXT] ++ str xzr, [x19, #PSCI_CPU_ENTRY] ++ mov x1, #PSCI_AFF_ON ++ str x1, [x19, #PSCI_CPU_STATE] ++ dc civac, x19 ++ dsb sy ++ ++ // Nothing of the previous kernel's EL1 context may survive. ++ ic iallu ++ tlbi alle1 ++ dsb sy ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ msr elr_el3, x2 ++ mov sp, x20 ++ isb ++ eret ++ ++/* EL1 side: s64 psci_smc(u64 fid, u64 a1, u64 a2, u64 a3) */ ++.globl psci_smc ++.type psci_smc, @function ++psci_smc: ++ smc #0 ++ ret ++ ++/* ++ * CPU_ON target when m1n1 takes a parked core back after a re-entry. ++ * x0 = EL1 stack top | m1n1 CPU index (stack tops are 16 KiB aligned). ++ */ ++.globl psci_secondary_entry ++.type psci_secondary_entry, @function ++psci_secondary_entry: ++ and x1, x0, #0xf ++ bic x0, x0, #0xf ++ msr tpidr_el1, x1 ++ mov sp, x0 ++ bl _cpu_reset_c ++ b . ++ ++/* ++ * Re-entry from Linux (a kexec'd trampoline, see docs/boot/warm-reboot.md): ++ * EL1, MMU and caches off, x0 = a copy of the original boot_args. ++ * Linux may hold clean lines of this image in its linear map, so drop them ++ * before m1n1 writes its .bss uncached and then reads it with caches on. ++ */ ++.globl _reentry ++.type _reentry, @function ++_reentry: ++ mov x19, x0 ++ adrp x1, _base ++ adrp x2, _end ++1: ++ dc civac, x1 ++ add x1, x1, #64 ++ cmp x1, x2 ++ b.lo 1b ++ dsb sy ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #1 ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ mov x0, x19 ++ b _start +diff --git a/src/smp.c b/src/smp.c +index 83eadff..7f5b336 100644 +--- a/src/smp.c ++++ b/src/smp.c +@@ -8,6 +8,7 @@ + #include "malloc.h" + #include "memory.h" + #include "pmgr.h" ++#include "psci.h" + #include "soc.h" + #include "string.h" + #include "types.h" +@@ -227,6 +228,74 @@ static void smp_stop_cpu(int index, int die, int cluster, int core, u64 impl, u6 + } + } + ++/* ++ * After a re-entry from Linux, a secondary that Linux offlined waits in the ++ * PSCI pen at EL3 rather than in smp_secondary_entry(). It can't be reset ++ * through the PMGR (it is running), so hand it an EL1 entry with CPU_ON. ++ */ ++static bool smp_reclaim_cpu(int index, u64 mpidr) ++{ ++ int i; ++ ++ if (index >= MAX_CPUS || spin_table[index].flag) ++ return false; ++ ++ if (psci_affinity_info(mpidr) != PSCI_AFF_OFF) ++ return false; ++ ++ printf("Taking CPU %d back from the PSCI pen... ", index); ++ ++ memset(&spin_table[index], 0, sizeof(struct spin_table)); ++ target_cpu = index; ++ secondary_stacks[index] = memalign(0x4000, SECONDARY_STACK_SIZE); ++ ++ /* The core starts with its MMU off */ ++ dc_civac_range(&spin_table[index], sizeof(struct spin_table)); ++ dc_civac_range(&target_cpu, sizeof(target_cpu)); ++ dc_civac_range(&boot_cpu_idx, sizeof(boot_cpu_idx)); ++ dc_civac_range(&boot_cpu_mpidr, sizeof(boot_cpu_mpidr)); ++ ++ if (psci_unpark_cpu(index, mpidr, secondary_stacks[index] + SECONDARY_STACK_SIZE) < 0) { ++ printf("Failed (CPU_ON)\n"); ++ return true; ++ } ++ ++ for (i = 0; i < 100; i++) { ++ sysop("dmb ld"); ++ if (spin_table[index].flag) ++ break; ++ udelay(1000); ++ } ++ ++ printf("%s\n", i >= 100 ? "Failed!" : " Back."); ++ return true; ++} ++ ++/* ++ * Put every live secondary into the PSCI pen, for a kernel that boots them ++ * with enable-method = "psci". A CPU that doesn't make it is dropped (and ++ * then pruned from the DT by the caller). ++ */ ++int smp_park_secondaries_psci(void) ++{ ++ int parked = 0; ++ ++ for (int i = 0; i < MAX_CPUS; i++) { ++ if (i == boot_cpu_idx || !smp_is_alive(i)) ++ continue; ++ ++ u64 mpidr = spin_table[i].mpidr; ++ if (psci_park_cpu(i, mpidr) < 0) { ++ printf("PSCI: CPU %d (MPIDR 0x%lx) did not park, dropping it\n", i, mpidr); ++ memset(&spin_table[i], 0, sizeof(struct spin_table)); ++ continue; ++ } ++ parked++; ++ } ++ ++ return parked; ++} ++ + static u64 smp_get_iorvbar(int node, int cpu) + { + u64 cpu_impl_reg[2]; +@@ -405,6 +474,10 @@ void smp_start_secondaries(void) + u8 cluster = FIELD_GET(CPU_REG_CLUSTER, reg); + u8 die = FIELD_GET(CPU_REG_DIE, reg); + ++ if (psci_active() && psci_reentered() && ++ smp_reclaim_cpu(i, core | (cluster << 8) | (die << 16))) ++ continue; ++ + smp_start_cpu(i, die, cluster, core, iorvbar, pmgr_reg + cpu_start_off); + } + } +diff --git a/src/smp.h b/src/smp.h +index 0296978..6c90557 100644 +--- a/src/smp.h ++++ b/src/smp.h +@@ -18,6 +18,7 @@ void smp_secondary_prep_el3(void); + + void smp_start_secondaries(void); + void smp_stop_secondaries(bool deep_sleep); ++int smp_park_secondaries_psci(void); + + #define smp_call0(i, f) smp_call4(i, f, 0, 0, 0, 0) + #define smp_call1(i, f, a) smp_call4(i, f, a, 0, 0, 0) +diff --git a/src/start.S b/src/start.S +index aa11e18..a3f35ae 100644 +--- a/src/start.S ++++ b/src/start.S +@@ -11,6 +11,7 @@ + .extern _v_sp0_irq + .extern _v_sp0_fiq + .extern _v_sp0_serr ++.extern _v_lower_sync + .extern _reset_stack + .extern _cpu_reset_c + .extern wdt_reboot +@@ -41,7 +42,7 @@ _vectors_start: + .align 7 + b _v_sp0_serr + .align 7 +- b _v_sp0_sync ++ b _v_lower_sync + .align 7 + b _v_sp0_irq + .align 7 +@@ -103,6 +104,8 @@ _start: + + mov x0, x19 + mov x1, x20 ++ mrs x2, CurrentEL ++ lsr x2, x2, #2 + bl _start_c + b . + +@@ -115,6 +118,7 @@ start_el3: + msr tpidr_el3, xzr + adrp x6, _stack_bot + adr x7, _start_c ++ mov x2, #3 // _start_c: entered at EL3 + b el3_eret_to_el1 + + .globl exc_unk +diff --git a/src/startup.c b/src/startup.c +index 61afe16..167fae8 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -5,6 +5,7 @@ + #include "dockchannel_uart.h" + #include "exception.h" + #include "firmware.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "types.h" +@@ -159,12 +160,15 @@ void dump_boot_args(struct boot_args *ba) + } + + extern void get_device_info(void); +-void _start_c(void *boot_args, void *base) ++void _start_c(void *boot_args, void *base, u64 boot_el) + { + UNUSED(base); + u32 cpu_id = 0; + + memset64(_bss_start, 0, _bss_end - _bss_start); ++ psci_note_boot_el(boot_el); ++ if (psci_reentered()) ++ boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); + diff --git a/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch b/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch new file mode 100644 index 00000000..edfc7592 --- /dev/null +++ b/boot/m1n1/patches/orchard/0016-psci-Turn-the-resident-EL3-PSCI-on-by-default.patch @@ -0,0 +1,84 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 15:56:40 +0100 +Subject: [PATCH] psci: Turn the resident EL3 PSCI on by default + +When m1n1 was entered at EL3 (A7-A9 booted by Pongo) it stays resident +and already implements CPU_ON/CPU_OFF/AFFINITY_INFO for the kernel, but +only when the payload asked for it with psci=1. Without it the kernel +gets the spin table and can neither offline CPUs nor kexec on SMP, so +every loader had to be built with the variable appended. + +Make it the default. psci=0 turns it off again (spin table, no /psci +node), psci=1 is still accepted. Nothing changes where m1n1 is entered +at EL1 or EL2: psci_active() still requires the EL3 entry. + +Assisted-by: LLM +--- + src/kboot.c | 4 ++-- + src/psci.c | 6 +++--- + src/psci.h | 4 ++-- + 3 files changed, 7 insertions(+), 7 deletions(-) + +diff --git a/src/kboot.c b/src/kboot.c +index 7123d60..a5f4238 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -576,7 +576,7 @@ static int dt_set_smbios(void) + } + + /* +- * With psci=1 and m1n1 owning EL3 (A7-A9), CPUs boot through m1n1's PSCI ++ * When m1n1 owns EL3 (A7-A9), CPUs boot through m1n1's PSCI + * (see psci.c) instead of the spin table, so the kernel can offline them + * for kexec and CPU hotplug. Also tells Linux how to re-enter m1n1. + */ +@@ -586,7 +586,7 @@ static int dt_set_psci(void) + int node; + + if (!psci_active()) { +- printf("PSCI: off (needs psci=1 and m1n1 entered at EL3)\n"); ++ printf("PSCI: off (m1n1 not entered at EL3, or psci=0)\n"); + return 0; + } + +diff --git a/src/psci.c b/src/psci.c +index 2b8ca0d..25e2a4f 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -28,8 +28,8 @@ struct { + u64 boot_el3; + } psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + +-/* psci=1 from the payload, kept across a re-entry */ +-static u32 psci_mode __attribute__((section(".data"))); ++/* On unless the payload says psci=0; kept across a re-entry */ ++static u32 psci_mode __attribute__((section(".data"))) = 1; + + /* EL3 stacks a CPU gets when it leaves the pen */ + static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); +@@ -177,7 +177,7 @@ void psci_note_boot_el(u64 el) + + void psci_configure(const char *val) + { +- psci_mode = val[0] == '1'; ++ psci_mode = val[0] != '0'; + } + + bool psci_active(void) +diff --git a/src/psci.h b/src/psci.h +index 6769ebe..9a213af 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -50,9 +50,9 @@ + + #ifndef __ASSEMBLER__ + +-/* Payload variable psci=1 */ ++/* Payload variable psci=0 turns PSCI off; anything else (psci=1) leaves it on */ + void psci_configure(const char *val); +-/* m1n1 entered at EL3 and psci=1 given: the kernel may use PSCI */ ++/* m1n1 entered at EL3 and not psci=0: the kernel uses PSCI */ + bool psci_active(void); + /* Called once from _start_c with the EL m1n1 was entered at */ + void psci_note_boot_el(u64 el); diff --git a/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch b/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch new file mode 100644 index 00000000..a1c26412 --- /dev/null +++ b/boot/m1n1/patches/orchard/0017-psci-Trap-EL1-WFI-WFE-to-EL3-for-a-firmware-CPU-rend.patch @@ -0,0 +1,286 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 15:58:57 +0100 +Subject: [PATCH] psci: Trap EL1 WFI/WFE to EL3 for a firmware CPU rendezvous + +An ordinary arm64 reboot or power-off does not offline the secondaries +with CPU_OFF: smp_send_stop() leaves them spinning in "wfe; wfi" at EL1 +with interrupts masked, still ON as far as the monitor knows. A9 has no +way to reset a core from another one (Apple and m1n1 power cores down +from the core itself), so SYSTEM_RESET and SYSTEM_OFF need those CPUs to +come to EL3 on their own. + +Give the monitor that hook: a private SiP call, made by m1n1 on the boot +CPU right before it starts a kernel, sets SCR_EL3.TWI and TWE there and +on every CPU that later leaves the pen, and records the AIC and watchdog +bases for later EL3 use. A trapped WFE returns at once (a spurious +wake-up is architectural, and the exception return may have cleared the +exclusive monitor whose event the caller waits for); a trapped WFI waits +at EL3 with IRQ/FIQ routed there only around the wfi, so an interrupt +meant for EL1 still wakes it. While a rendezvous is pending, a trapped +CPU instead parks itself in the pen exactly as CPU_OFF would, cleaning +its L1D on the way. + +The trap enable has no user yet. The CPUs that have been in the pen are +recorded, so a rendezvous knows which ones the kernel was given. + +Assisted-by: LLM +--- + src/psci.c | 56 ++++++++++++++++++++++++++++++++++++++ + src/psci.h | 20 ++++++++++++++ + src/psci_asm.S | 73 ++++++++++++++++++++++++++++++++++++++++++++++++-- + 3 files changed, 147 insertions(+), 2 deletions(-) + +diff --git a/src/psci.c b/src/psci.c +index 25e2a4f..af4f325 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -28,6 +28,24 @@ struct { + u64 boot_el3; + } psci_persist __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + ++/* ++ * Rendezvous state, one line, only accessed at EL3 with the MMU off. A CPU ++ * that wants all others in the pen (SYSTEM_RESET/SYSTEM_OFF) sets pending; ++ * every other CPU then parks itself at its next trapped WFI/WFE. ++ */ ++struct psci_rdv { ++ volatile u64 pending; ++ volatile u64 traps; /* SCR_EL3.TWI|TWE for CPUs leaving the pen */ ++ volatile u64 cpus; /* MPIDR indices that have been in the pen: handed to the kernel */ ++ u64 aic; ++ u64 wdt; ++ u64 pad[PSCI_CPU_SIZE / 8 - 5]; ++} psci_rdv __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ ++_Static_assert(sizeof(struct psci_rdv) == PSCI_CPU_SIZE, "psci_rdv layout"); ++_Static_assert(offsetof(struct psci_rdv, pending) == PSCI_RDV_PENDING, "psci_rdv layout"); ++_Static_assert(offsetof(struct psci_rdv, traps) == PSCI_RDV_TRAPS, "psci_rdv layout"); ++ + /* On unless the payload says psci=0; kept across a re-entry */ + static u32 psci_mode __attribute__((section(".data"))) = 1; + +@@ -62,6 +80,10 @@ static s64 psci_cpu_off(void) + if (idx < 0) + return PSCI_RET_DENIED; + ++ psci_line_sync(&psci_rdv); ++ psci_rdv.cpus |= BIT(idx); ++ psci_line_sync(&psci_rdv); ++ + psci_pen(&psci_cpus[idx], (u64)&psci_stacks[idx][PSCI_STACK_SIZE]); + } + +@@ -128,6 +150,32 @@ static s64 psci_features(u32 fid) + } + } + ++/* Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the pen */ ++static s64 psci_m1n1_handoff(u64 traps, u64 aic, u64 wdt) ++{ ++ u64 bits = traps ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ ++ psci_rdv.pending = 0; ++ psci_rdv.traps = bits; ++ psci_rdv.aic = aic; ++ psci_rdv.wdt = wdt; ++ psci_line_sync(&psci_rdv); ++ ++ msr(SCR_EL3, (mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)) | bits); ++ sysop("isb"); ++ ++ return PSCI_RET_SUCCESS; ++} ++ ++/* A trapped WFI/WFE saw the rendezvous flag: become CPU_OFF right here */ ++void psci_rendezvous_off(void) ++{ ++ psci_cpu_off(); ++ /* Not a CPU we know (can't happen after the pen saw it): stay out of the way */ ++ for (;;) ++ sysop("wfe"); ++} ++ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + { + u32 fn = fid; +@@ -154,6 +202,8 @@ u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + return 2; /* no trusted OS, nothing to migrate */ + case PSCI_1_0_FN_PSCI_FEATURES: + return psci_features(a1); ++ case PSCI_M1N1_SIP_HANDOFF: ++ return psci_m1n1_handoff(a1, a2, a3); + default: + /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ + return PSCI_RET_NOT_SUPPORTED; +@@ -190,6 +240,12 @@ bool psci_reentered(void) + return reentered; + } + ++void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base) ++{ ++ if (psci_active()) ++ psci_smc(PSCI_M1N1_SIP_HANDOFF, wfx_traps, aic_base, wdt_base); ++} ++ + int psci_affinity_info(u64 mpidr) + { + return psci_smc(PSCI_0_2_FN64_AFFINITY_INFO, mpidr, 0, 0); +diff --git a/src/psci.h b/src/psci.h +index 9a213af..1d3a88e 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -27,6 +27,13 @@ + #define PSCI_0_2_FN64_CPU_ON 0xc4000003 + #define PSCI_0_2_FN64_AFFINITY_INFO 0xc4000004 + ++/* ++ * m1n1-private SiP call, made by m1n1 itself on the boot CPU right before it ++ * starts a kernel: x1 = 1 to trap EL1 WFI/WFE to EL3 on this and every CPU ++ * that later leaves the pen, x2 = AIC base, x3 = primary watchdog base. ++ */ ++#define PSCI_M1N1_SIP_HANDOFF 0xc200ff00 ++ + #define PSCI_RET_SUCCESS 0 + #define PSCI_RET_NOT_SUPPORTED -1 + #define PSCI_RET_INVALID_PARAMS -2 +@@ -48,6 +55,15 @@ + #define PSCI_CPU_CONTEXT 0x10 + #define PSCI_CPU_SIZE 0x80 + ++/* struct psci_rdv, one line, EL3 only (MMU off) */ ++#define PSCI_RDV_PENDING 0x00 ++#define PSCI_RDV_TRAPS 0x08 ++ ++#define SCR_EL3_IRQ (1 << 1) ++#define SCR_EL3_FIQ (1 << 2) ++#define SCR_EL3_TWI (1 << 12) ++#define SCR_EL3_TWE (1 << 13) ++ + #ifndef __ASSEMBLER__ + + /* Payload variable psci=0 turns PSCI off; anything else (psci=1) leaves it on */ +@@ -67,9 +83,13 @@ int psci_park_cpu(int cpu, u64 mpidr); + /* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ + int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); + ++/* EL1, boot CPU, last thing before the kernel: see PSCI_M1N1_SIP_HANDOFF */ ++void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base); ++ + /* EL3 side */ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); + void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); ++void psci_rendezvous_off(void) __attribute__((noreturn)); + void psci_secondary_entry(void); + void _reentry(void); + +diff --git a/src/psci_asm.S b/src/psci_asm.S +index 14f842f..ff31f3a 100644 +--- a/src/psci_asm.S ++++ b/src/psci_asm.S +@@ -9,13 +9,16 @@ + .extern _cpu_reset_c + .extern psci_handle + .extern psci_persist ++.extern psci_rdv ++.extern psci_rendezvous_off + + .text + + /* + * Lower-EL synchronous exceptions. At EL3, an SMCCC call (smc #0 from EL1) +- * goes to the PSCI dispatcher; everything else, and every exception taken +- * at EL1/EL2, takes the usual path. ++ * goes to the PSCI dispatcher and a trapped WFI/WFE to psci_wfx_entry; ++ * everything else, and every exception taken at EL1/EL2, takes the usual ++ * path. + */ + .globl _v_lower_sync + .type _v_lower_sync, @function +@@ -26,6 +29,8 @@ _v_lower_sync: + b.ne 1f + mrs x16, esr_el3 + lsr x17, x16, #26 ++ cmp x17, #0x01 // EC: trapped WFI/WFE ++ b.eq 2f + cmp x17, #0x17 // EC: SMC from AArch64 + b.ne 1f + tst x16, #0xffff // imm16 == 0: SMCCC +@@ -35,6 +40,59 @@ _v_lower_sync: + 1: + ldp x16, x17, [sp], #16 + b _v_sp0_sync ++2: ++ ldp x16, x17, [sp], #16 ++ b psci_wfx_entry ++ ++/* ++ * WFI/WFE from EL1/EL0, trapped by SCR_EL3.TWI/TWE (see PSCI_M1N1_SIP_HANDOFF). ++ * This is how firmware gets hold of a CPU that the kernel stopped with an ++ * IPI and left spinning in "wfe; wfi" with interrupts masked (an ordinary ++ * reboot does that; only kexec and hotplug use CPU_OFF): while a ++ * SYSTEM_RESET/SYSTEM_OFF rendezvous is pending, the CPU parks itself as if ++ * it had called CPU_OFF. Otherwise: ++ * - WFE completes at once. A spurious WFE wake-up is architectural, and ++ * waiting here could miss the event the caller waits for (the exception ++ * return may clear its exclusive monitor). ++ * - WFI waits here, with IRQ/FIQ routed to EL3 only around the wfi so that ++ * an interrupt for EL1 wakes it (DAIF is masked at EL3, nothing is taken). ++ * Both then return past the instruction. Only x0-x3 are used. ++ */ ++.type psci_wfx_entry, @function ++psci_wfx_entry: ++ stp x0, x1, [sp, #-16]! ++ stp x2, x3, [sp, #-16]! ++ adrp x0, psci_rdv ++ add x0, x0, :lo12:psci_rdv ++ dc civac, x0 ++ dsb sy ++ ldr x1, [x0, #PSCI_RDV_PENDING] ++ cbnz x1, 3f ++ mrs x1, esr_el3 ++ tbnz x1, #0, 2f // ISS.TI bit 0 set: WFE (or WFET) ++ mrs x2, scr_el3 ++ orr x3, x2, #(SCR_EL3_IRQ | SCR_EL3_FIQ) ++ msr scr_el3, x3 ++ isb ++ dsb sy ++ wfi ++ msr scr_el3, x2 ++ isb ++ dc civac, x0 ++ dsb sy ++ ldr x1, [x0, #PSCI_RDV_PENDING] ++ cbnz x1, 3f ++2: ++ mrs x1, elr_el3 ++ add x1, x1, #4 ++ msr elr_el3, x1 ++ ldp x2, x3, [sp], #16 ++ ldp x0, x1, [sp], #16 ++ eret ++3: ++ add sp, sp, #32 ++ bl psci_rendezvous_off ++ b . + + /* + * SMCCC 1.0: x0-x17 may be clobbered, x18-x30 and SP_EL1 belong to the +@@ -106,6 +164,17 @@ psci_pen: + ic iallu + tlbi alle1 + dsb sy ++ ++ // WFI/WFE traps as the boot CPU set them for the kernel (psci_rdv.traps) ++ adrp x3, psci_rdv ++ add x3, x3, :lo12:psci_rdv ++ dc civac, x3 ++ dsb sy ++ ldr x4, [x3, #PSCI_RDV_TRAPS] ++ mrs x1, scr_el3 ++ bic x1, x1, #(SCR_EL3_TWI | SCR_EL3_TWE) ++ orr x1, x1, x4 ++ msr scr_el3, x1 + mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off + movk x1, #0x30d0, lsl #16 + msr sctlr_el1, x1 diff --git a/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch b/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch new file mode 100644 index 00000000..e5dd449e --- /dev/null +++ b/boot/m1n1/patches/orchard/0018-psci-Implement-SYSTEM_RESET-and-SYSTEM_OFF-with-a-CP.patch @@ -0,0 +1,497 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 16:01:53 +0100 +Subject: [PATCH] psci: Implement SYSTEM_RESET and SYSTEM_OFF with a CPU + rendezvous + +Give the kernel's ordinary reboot and power-off paths somewhere to go. +On SYSTEM_RESET or SYSTEM_OFF the calling CPU sets the rendezvous flag, +sends an event and an AIC IPI to every other CPU the kernel was given, +and waits up to a second for all of them to park in the pen through the +WFI/WFE traps. If one does not arrive, it clears the flag and returns +without having changed anything, so the kernel goes on to its next +restart or power-off method as it does today. + +Once they are parked it disarms the primary watchdog (the kernel, or +systemd's reboot watchdog, may have armed it), stops its timers and +drops the traps. SYSTEM_RESET then turns the EL1 MMU and caches off, +writes back every data cache level by set/way, invalidates the EL1 TLB +and I-cache and restarts m1n1 at EL1h with the original iBoot boot_args, +saved in .data on the first cold start before top_of_memory_alloc() +changes them. psci_reentered() and the new psci_reset_reentered() tell +the restarted m1n1 where it came from. SYSTEM_OFF parks the caller too: +there is no PMU driver here, so this halts rather than powers off. + +The payload variable psci_reset selects what SYSTEM_RESET leads to: boot +(the default, used by the next change), proxy, or 0 for NOT_SUPPORTED +with no WFI/WFE traps. It is reported by PSCI_FEATURES and armed by +kboot_boot() through the private handoff call, which also records the +boot CPU and needs an AIC v1 and a watchdog in the ADT. + +Assisted-by: LLM +--- + src/kboot.c | 1 + + src/payload.c | 2 + + src/psci.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++-- + src/psci.h | 25 +++++- + src/psci_asm.S | 44 +++++++++- + src/startup.c | 1 + + 6 files changed, 280 insertions(+), 13 deletions(-) + +diff --git a/src/kboot.c b/src/kboot.c +index a5f4238..299e388 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -3301,6 +3301,7 @@ int kboot_boot(void *kernel) + + printf("Setting SMP mode to WFE...\n"); + smp_set_wfe_mode(true); ++ psci_handoff(); + printf("Preparing to boot kernel at %p with fdt at %p\n", kernel, dt); + + next_stage.entry = kernel; +diff --git a/src/payload.c b/src/payload.c +index 4ffc9e2..0b7dc14 100644 +--- a/src/payload.c ++++ b/src/payload.c +@@ -208,6 +208,8 @@ static bool check_var(u8 **p) + enable_tso = val[0] == '1'; + } else if (IS_VAR("psci=")) { + psci_configure(val); ++ } else if (IS_VAR("psci_reset=")) { ++ psci_reset_configure(val); + } else { + printf("Unknown variable %s\n", *p); + } +diff --git a/src/psci.c b/src/psci.c +index af4f325..1821ebc 100644 +--- a/src/psci.c ++++ b/src/psci.c +@@ -1,9 +1,19 @@ + /* SPDX-License-Identifier: MIT */ + + #include "psci.h" ++#include "adt.h" ++#include "aic_regs.h" ++#include "memory.h" + #include "smp.h" ++#include "string.h" + #include "types.h" + #include "utils.h" ++#include "xnuboot.h" ++ ++#define WDT_CTL 0x1c ++ ++/* SYSTEM_RESET/SYSTEM_OFF wait this long for the other CPUs to park */ ++#define RENDEZVOUS_TIMEOUT_US 1000000 + + struct psci_cpu { + volatile u64 state; +@@ -39,7 +49,8 @@ struct psci_rdv { + volatile u64 cpus; /* MPIDR indices that have been in the pen: handed to the kernel */ + u64 aic; + u64 wdt; +- u64 pad[PSCI_CPU_SIZE / 8 - 5]; ++ u64 reset; /* enum psci_reset_mode the kernel was started with */ ++ u64 pad[PSCI_CPU_SIZE / 8 - 6]; + } psci_rdv __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); + + _Static_assert(sizeof(struct psci_rdv) == PSCI_CPU_SIZE, "psci_rdv layout"); +@@ -49,11 +60,18 @@ _Static_assert(offsetof(struct psci_rdv, traps) == PSCI_RDV_TRAPS, "psci_rdv lay + /* On unless the payload says psci=0; kept across a re-entry */ + static u32 psci_mode __attribute__((section(".data"))) = 1; + ++/* psci_reset=0|proxy|boot, kept across a re-entry */ ++static u32 psci_reset_mode __attribute__((section(".data"))) = PSCI_RESET_BOOT; ++ ++/* The iBoot boot_args m1n1 was first started with; SYSTEM_RESET starts m1n1 with them again */ ++static struct boot_args reset_boot_args __attribute__((section(".data"), aligned(PSCI_CPU_SIZE))); ++ + /* EL3 stacks a CPU gets when it leaves the pen */ + static u8 psci_stacks[PSCI_MAX_CPUS][PSCI_STACK_SIZE] __attribute__((aligned(16))); + + static bool psci_have_el3; + static bool reentered; ++static bool reset_reentered; + + /* EL3 side */ + +@@ -129,9 +147,113 @@ static s64 psci_affinity(u64 mpidr, u64 level) + return psci_cpus[idx].state; + } + ++/* ++ * Gets every other CPU the kernel was given into the pen: they park at their ++ * next trapped WFI/WFE (see psci_wfx_entry). A CPU waiting in WFI is woken ++ * with an IPI, one in WFE with an event. If one doesn't arrive in time, ++ * nothing has been changed that the kernel cares about. ++ */ ++static bool psci_rendezvous(int self) ++{ ++ psci_line_sync(&psci_rdv); ++ u64 others = psci_rdv.cpus & ~BIT(self); ++ ++ psci_rdv.pending = 1; ++ psci_line_sync(&psci_rdv); ++ sysop("sev"); ++ ++ for (int i = 0; i < PSCI_MAX_CPUS; i++) { ++ if (!(others & BIT(i))) ++ continue; ++ psci_line_sync(&psci_cpus[i]); ++ if (psci_cpus[i].state != PSCI_AFF_OFF) ++ write32(psci_rdv.aic + AIC_IPI_SEND, AIC_IPI_SEND_CPU(i)); ++ } ++ ++ u64 timeout = timeout_calculate(RENDEZVOUS_TIMEOUT_US); ++ do { ++ bool all = true; ++ ++ for (int i = 0; i < PSCI_MAX_CPUS; i++) { ++ if (!(others & BIT(i))) ++ continue; ++ psci_line_sync(&psci_cpus[i]); ++ if (psci_cpus[i].state != PSCI_AFF_OFF) ++ all = false; ++ } ++ if (all) { ++ psci_rdv.pending = 0; ++ psci_line_sync(&psci_rdv); ++ return true; ++ } ++ sysop("sev"); ++ udelay(100); ++ } while (!timeout_expired(timeout)); ++ ++ psci_rdv.pending = 0; ++ psci_line_sync(&psci_rdv); ++ return false; ++} ++ ++/* Everything that must happen before this CPU leaves the kernel for good */ ++static s64 psci_system_stop(void) ++{ ++ int self = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ psci_line_sync(&psci_rdv); ++ if (psci_rdv.reset == PSCI_RESET_OFF) ++ return PSCI_RET_NOT_SUPPORTED; ++ if (self < 0 || !psci_rendezvous(self)) ++ return PSCI_RET_INTERNAL_FAILURE; ++ ++ msr(CNTP_CTL_EL0, 0); ++ msr(CNTV_CTL_EL0, 0); ++ /* The kernel (or systemd's reboot watchdog) may have armed it */ ++ write32(psci_rdv.wdt + WDT_CTL, 0); ++ read32(psci_rdv.wdt + WDT_CTL); ++ ++ /* ++ * The WFx traps go away with the kernel; the next handoff sets them ++ * again, and the next kboot records which CPUs the next kernel gets. ++ */ ++ psci_rdv.traps = 0; ++ psci_rdv.cpus = 0; ++ psci_line_sync(&psci_rdv); ++ msr(SCR_EL3, mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)); ++ sysop("isb"); ++ return PSCI_RET_SUCCESS; ++} ++ ++/* Returns only if the other CPUs could not be stopped: the kernel tries its next method */ ++static s64 psci_system_reset(void) ++{ ++ s64 ret = psci_system_stop(); ++ ++ if (ret != PSCI_RET_SUCCESS) ++ return ret; ++ ++ psci_system_reset_entry(&reset_boot_args); ++} ++ ++/* No PMU driver here: this halts the SoC with every CPU parked, it does not power off */ ++static s64 psci_system_off(void) ++{ ++ s64 ret = psci_system_stop(); ++ ++ if (ret != PSCI_RET_SUCCESS) ++ return ret; ++ ++ for (;;) ++ sysop("wfi"); ++} ++ + static s64 psci_features(u32 fid) + { + switch (fid) { ++ case PSCI_0_2_FN_SYSTEM_OFF: ++ case PSCI_0_2_FN_SYSTEM_RESET: ++ psci_line_sync(&psci_rdv); ++ return psci_rdv.reset != PSCI_RESET_OFF ? PSCI_RET_SUCCESS : PSCI_RET_NOT_SUPPORTED; + case PSCI_0_2_FN_PSCI_VERSION: + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_CPU_ON: +@@ -144,21 +266,35 @@ static s64 psci_features(u32 fid) + default: + /* + * Includes SMCCC_VERSION (so callers assume SMCCC 1.0 and make +- * no other SMCs), CPU_SUSPEND, SYSTEM_OFF and SYSTEM_RESET. ++ * no other SMCs), CPU_SUSPEND and SYSTEM_RESET2. + */ + return PSCI_RET_NOT_SUPPORTED; + } + } + +-/* Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the pen */ +-static s64 psci_m1n1_handoff(u64 traps, u64 aic, u64 wdt) ++/* ++ * Arms (or disarms) the WFx traps on this CPU and on every CPU leaving the ++ * pen, and with them SYSTEM_RESET/SYSTEM_OFF (reset = enum psci_reset_mode). ++ */ ++static s64 psci_m1n1_handoff(u64 reset, u64 aic, u64 wdt) + { +- u64 bits = traps ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ if (reset > PSCI_RESET_PROXY || (reset && (!aic || !wdt))) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ u64 bits = reset ? (SCR_EL3_TWI | SCR_EL3_TWE) : 0; ++ int self = psci_mpidr_index(mrs(MPIDR_EL1)); ++ ++ if (self < 0) ++ return PSCI_RET_DENIED; + ++ psci_line_sync(&psci_rdv); ++ /* The secondaries were added when kboot parked them */ ++ psci_rdv.cpus |= BIT(self); + psci_rdv.pending = 0; + psci_rdv.traps = bits; + psci_rdv.aic = aic; + psci_rdv.wdt = wdt; ++ psci_rdv.reset = reset; + psci_line_sync(&psci_rdv); + + msr(SCR_EL3, (mrs(SCR_EL3) & ~(u64)(SCR_EL3_TWI | SCR_EL3_TWE)) | bits); +@@ -202,10 +338,13 @@ u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3) + return 2; /* no trusted OS, nothing to migrate */ + case PSCI_1_0_FN_PSCI_FEATURES: + return psci_features(a1); ++ case PSCI_0_2_FN_SYSTEM_RESET: ++ return psci_system_reset(); ++ case PSCI_0_2_FN_SYSTEM_OFF: ++ return psci_system_off(); + case PSCI_M1N1_SIP_HANDOFF: + return psci_m1n1_handoff(a1, a2, a3); + default: +- /* SYSTEM_RESET/SYSTEM_OFF too: the caller falls back to its own */ + return PSCI_RET_NOT_SUPPORTED; + } + } +@@ -216,6 +355,7 @@ void psci_note_boot_el(u64 el) + { + /* MMU still off: these .data accesses are uncached, like the writers' */ + if (psci_persist.reentry_flag) { ++ reset_reentered = psci_persist.reentry_flag == PSCI_REENTRY_RESET; + reentered = true; + psci_persist.reentry_flag = 0; + } +@@ -240,10 +380,72 @@ bool psci_reentered(void) + return reentered; + } + +-void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base) ++void psci_reset_configure(const char *val) ++{ ++ if (val[0] == '0') ++ psci_reset_mode = PSCI_RESET_OFF; ++ else if (!strncmp(val, "proxy", 5)) ++ psci_reset_mode = PSCI_RESET_PROXY; ++ else ++ psci_reset_mode = PSCI_RESET_BOOT; ++} ++ ++enum psci_reset_mode psci_reset_mode_get(void) + { +- if (psci_active()) +- psci_smc(PSCI_M1N1_SIP_HANDOFF, wfx_traps, aic_base, wdt_base); ++ return psci_reset_mode; ++} ++ ++bool psci_reset_reentered(void) ++{ ++ return reset_reentered; ++} ++ ++void psci_reset_save_boot_args(void) ++{ ++ if (reentered) ++ return; ++ memcpy(&reset_boot_args, &cur_boot_args, sizeof(reset_boot_args)); ++ dc_civac_range(&reset_boot_args, sizeof(reset_boot_args)); ++} ++ ++/* ++ * The CPU rendezvous needs the AIC v1 IPI (A7-A9) and the primary watchdog ++ * (wdt-version 1 there, nothing else to stop). Without them SYSTEM_RESET and ++ * SYSTEM_OFF stay NOT_SUPPORTED and the kernel uses its own methods. ++ */ ++void psci_handoff(void) ++{ ++ enum psci_reset_mode reset = psci_reset_mode; ++ u64 aic = 0, wdt = 0; ++ int path[8]; ++ ++ if (!psci_active()) ++ return; ++ ++ if (reset != PSCI_RESET_OFF) { ++ int node = adt_path_offset_trace(adt, "/arm-io/aic", path); ++ if (node < 0 || !adt_is_compatible(adt, node, "aic,1") || ++ adt_get_reg(adt, path, "reg", 0, &aic, NULL) < 0) { ++ printf("PSCI: SYSTEM_RESET/SYSTEM_OFF unavailable: no AIC v1\n"); ++ reset = PSCI_RESET_OFF; ++ } ++ node = adt_path_offset_trace(adt, "/arm-io/wdt", path); ++ if (node < 0 || adt_get_reg(adt, path, "reg", 0, &wdt, NULL) < 0) { ++ printf("PSCI: SYSTEM_RESET/SYSTEM_OFF unavailable: no watchdog\n"); ++ reset = PSCI_RESET_OFF; ++ } ++ } ++ ++ if (psci_smc(PSCI_M1N1_SIP_HANDOFF, reset, aic, wdt) != PSCI_RET_SUCCESS) { ++ printf("PSCI: handoff refused, SYSTEM_RESET/SYSTEM_OFF unavailable\n"); ++ psci_smc(PSCI_M1N1_SIP_HANDOFF, PSCI_RESET_OFF, 0, 0); ++ return; ++ } ++ ++ static const char *const names[] = {"off", "warm reboot into the default entry", ++ "warm reboot into the proxy"}; ++ printf("PSCI: SYSTEM_RESET: %s%s\n", names[reset], ++ reset ? "; EL1 WFI/WFE trap to EL3" : ""); + } + + int psci_affinity_info(u64 mpidr) +diff --git a/src/psci.h b/src/psci.h +index 1d3a88e..7c81af7 100644 +--- a/src/psci.h ++++ b/src/psci.h +@@ -29,8 +29,9 @@ + + /* + * m1n1-private SiP call, made by m1n1 itself on the boot CPU right before it +- * starts a kernel: x1 = 1 to trap EL1 WFI/WFE to EL3 on this and every CPU +- * that later leaves the pen, x2 = AIC base, x3 = primary watchdog base. ++ * starts a kernel: x1 = enum psci_reset_mode (nonzero also traps EL1 WFI/WFE ++ * to EL3 on this and every CPU that later leaves the pen), x2 = AIC base, ++ * x3 = primary watchdog base. + */ + #define PSCI_M1N1_SIP_HANDOFF 0xc200ff00 + +@@ -59,6 +60,10 @@ + #define PSCI_RDV_PENDING 0x00 + #define PSCI_RDV_TRAPS 0x08 + ++/* psci_persist.reentry_flag */ ++#define PSCI_REENTRY_KEXEC 1 ++#define PSCI_REENTRY_RESET 2 ++ + #define SCR_EL3_IRQ (1 << 1) + #define SCR_EL3_FIQ (1 << 2) + #define SCR_EL3_TWI (1 << 12) +@@ -83,13 +88,27 @@ int psci_park_cpu(int cpu, u64 mpidr); + /* Bring a parked CPU back into m1n1 (re-entry); returns 0 if it was parked */ + int psci_unpark_cpu(int index, u64 mpidr, void *stack_top); + ++/* What SYSTEM_RESET does: payload variable psci_reset=0|proxy|boot (default boot) */ ++enum psci_reset_mode { ++ PSCI_RESET_OFF = 0, /* NOT_SUPPORTED: the kernel resets the SoC its own way */ ++ PSCI_RESET_BOOT = 1, /* back into m1n1, which boots the default boot entry */ ++ PSCI_RESET_PROXY = 2, /* back into m1n1, which only runs the proxy */ ++}; ++void psci_reset_configure(const char *val); ++enum psci_reset_mode psci_reset_mode_get(void); ++/* Set when m1n1 was re-entered by SYSTEM_RESET (psci_reentered() is too) */ ++bool psci_reset_reentered(void); ++/* _start_c: keep the first boot_args, SYSTEM_RESET starts m1n1 with them */ ++void psci_reset_save_boot_args(void); ++ + /* EL1, boot CPU, last thing before the kernel: see PSCI_M1N1_SIP_HANDOFF */ +-void psci_handoff(bool wfx_traps, u64 aic_base, u64 wdt_base); ++void psci_handoff(void); + + /* EL3 side */ + u64 psci_handle(u64 fid, u64 a1, u64 a2, u64 a3); + void psci_pen(void *cpu, u64 stack_top) __attribute__((noreturn)); + void psci_rendezvous_off(void) __attribute__((noreturn)); ++void psci_system_reset_entry(void *boot_args) __attribute__((noreturn)); + void psci_secondary_entry(void); + void _reentry(void); + +diff --git a/src/psci_asm.S b/src/psci_asm.S +index ff31f3a..61a6a02 100644 +--- a/src/psci_asm.S ++++ b/src/psci_asm.S +@@ -230,10 +230,52 @@ _reentry: + + adrp x1, psci_persist // .reentry_flag + add x1, x1, :lo12:psci_persist +- mov x2, #1 ++ mov x2, #PSCI_REENTRY_KEXEC + str x2, [x1] + dc civac, x1 + dsb sy + + mov x0, x19 + b _start ++ ++/* ++ * psci_system_reset_entry(boot_args): the end of SYSTEM_RESET, at EL3 on the ++ * calling CPU once every other CPU is in the pen (L1D clean) and nothing ++ * else runs. The kernel's EL1 MMU and caches may still be on; its stack and ++ * memory are never used again. Turn EL1 translation and caching off, write ++ * all data caches back, flag the reset and restart m1n1 at EL1h with the ++ * original boot_args, like a cold boot. _start resets the stack and reapplies ++ * the relocations; SP_EL3 goes back to the boot stack. ++ */ ++.globl psci_system_reset_entry ++.type psci_system_reset_entry, @function ++psci_system_reset_entry: ++ msr daifset, #0xf ++ mov x19, x0 ++ mov x1, #0x0800 // SCTLR_EL1: RES1 bits only, MMU and caches off ++ movk x1, #0x30d0, lsl #16 ++ msr sctlr_el1, x1 ++ isb ++ mov x0, #DCSW_OP_DCCISW ++ bl dcsw_op_all ++ tlbi alle1 ++ ic iallu ++ dsb sy ++ isb ++ ++ adrp x1, psci_persist // .reentry_flag ++ add x1, x1, :lo12:psci_persist ++ mov x2, #PSCI_REENTRY_RESET ++ str x2, [x1] ++ dc civac, x1 ++ dsb sy ++ ++ adrp x1, _start ++ add x1, x1, :lo12:_start ++ msr elr_el3, x1 ++ mov x1, #0x3c5 // EL1h, DAIF masked ++ msr spsr_el3, x1 ++ adrp x1, _stack_bot_el3 ++ mov sp, x1 ++ mov x0, x19 ++ eret +diff --git a/src/startup.c b/src/startup.c +index 167fae8..4d9ac39 100644 +--- a/src/startup.c ++++ b/src/startup.c +@@ -171,6 +171,7 @@ void _start_c(void *boot_args, void *base, u64 boot_el) + boot_cpu_idx = -1; // .data keeps the last run's value; find it again + boot_args_addr = (u64)boot_args; + memcpy(&cur_boot_args, boot_args, sizeof(cur_boot_args)); ++ psci_reset_save_boot_args(); + + adt = + (void *)(((u64)cur_boot_args.devtree) - cur_boot_args.virt_base + cur_boot_args.phys_base); diff --git a/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch b/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch new file mode 100644 index 00000000..a8039549 --- /dev/null +++ b/boot/m1n1/patches/orchard/0019-bootmgr-Boot-the-default-entry-after-a-PSCI-SYSTEM_R.patch @@ -0,0 +1,298 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 9 Nov 2026 16:03:26 +0100 +Subject: [PATCH] bootmgr: Boot the default entry after a PSCI SYSTEM_RESET + +SYSTEM_RESET brings m1n1 back with every other CPU parked and the +original boot_args, so m1n1 can finish the reboot itself the way the +host's orchard launch starts a system: bring main storage up, take the +first bootable entry in BLS order (the same default the host suggests; +there is still no remembered default) and start it with bootmgr_start(). +The board DTB file name comes from the last launch, which m1n1 now keeps +in .data; if no entry was ever launched (a kernel from the payload, say) +m1n1 doesn't guess the board and stays in the proxy. + +A re-entered m1n1 also skips the stale payload area for the logo, keeps +the picture on screen instead of clearing it and drawing the logo (the +entry's splash replaces it), and leaves SEP alone, which was never +reset. A 96x96 square at the top left shows how far it got: cyan back in +m1n1, green loading the entry, magenta no previous launch, red storage +or entry failure. Any failure ends in the USB proxy, as a cold boot with +nothing to do does; m1n1 never turns it into a cold reset. + +With psci_reset=proxy it stops in the proxy right after the re-entry. + +Assisted-by: LLM +--- + rust/src/bls.rs | 48 +++++++++++++++++++++++++++++ + src/bootmgr.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ + src/bootmgr.h | 2 ++ + src/fb.c | 4 ++- + src/main.c | 16 +++++++--- + 5 files changed, 146 insertions(+), 5 deletions(-) + +diff --git a/rust/src/bls.rs b/rust/src/bls.rs +index 35ddc1b..4570d63 100644 +--- a/rust/src/bls.rs ++++ b/rust/src/bls.rs +@@ -629,6 +629,54 @@ pub unsafe extern "C" fn rust_boot_entries( + text.leak().as_mut_ptr() as *mut c_void + } + ++/// The entry a boot without a host starts: the first bootable one in BLS ++/// order, as the host's `orchard launch` suggests (there is no remembered ++/// default). ++pub fn default_entry(listing: &Listing) -> Option<&Entry> { ++ listing.entries.iter().find(|entry| entry.bootable()) ++} ++ ++/// Writes the id of main storage's default entry (see `default_entry`), ++/// NUL-terminated, to `id` (`size` bytes). Main storage must be up; ++/// `devicetree` may be NULL. Returns 0, or -1 with the reason on the console. ++#[no_mangle] ++pub unsafe extern "C" fn rust_boot_default_entry( ++ devicetree: *const c_char, ++ id: *mut c_char, ++ size: c_size_t, ++) -> c_int { ++ let devicetree = match devicetree.is_null() { ++ true => None, ++ false => unsafe { CStr::from_ptr(devicetree) }.to_str().ok(), ++ }; ++ let listing = read_entries(storage::MainStorage::new, devicetree); ++ log(&listing); ++ let entry = match &listing { ++ Err(error) => { ++ println!("bls: {}", error); ++ return -1; ++ } ++ Ok(listing) => match default_entry(listing) { ++ Some(entry) => entry, ++ None => { ++ println!("bls: no bootable entry"); ++ return -1; ++ } ++ }, ++ }; ++ let name = entry.id.as_bytes(); ++ if name.len() >= size || name.contains(&0) { ++ println!("bls: entry id too long"); ++ return -1; ++ } ++ unsafe { ++ core::ptr::copy_nonoverlapping(name.as_ptr(), id as *mut u8, name.len()); ++ *id.add(name.len()) = 0; ++ } ++ println!("bls: default entry {}", entry.id); ++ 0 ++} ++ + #[repr(C)] + pub struct BootFiles { + kernel: *mut c_void, +diff --git a/src/bootmgr.c b/src/bootmgr.c +index b91b6a6..a746673 100644 +--- a/src/bootmgr.c ++++ b/src/bootmgr.c +@@ -8,6 +8,7 @@ + #include "heapblock.h" + #include "kboot.h" + #include "mitigations.h" ++#include "psci.h" + #include "smp.h" + #include "string.h" + #include "utils.h" +@@ -32,6 +33,32 @@ struct boot_files { + + int rust_boot_entry_load(const char *id, const char *devicetree, bool splash, + struct boot_files *files); ++int rust_boot_default_entry(const char *devicetree, char *id, size_t size); ++ ++/* ++ * The board DTB file name of the last launch (the host's orchard launch ++ * passes it), kept across a SYSTEM_RESET so m1n1 can boot an entry without ++ * the host and without guessing the board. ++ */ ++static struct { ++ bool launched; ++ bool have_devicetree; ++ char devicetree[MAX_NAME]; ++} last_launch __attribute__((section(".data"))); ++ ++/* Warm reset progress, top left */ ++#define MARK_REENTERED 0x00b7ff ++#define MARK_LOADING 0x00c853 ++#define MARK_NO_LAUNCH 0xff00ff ++#define MARK_FAILED 0xff1744 ++ ++static void bootmgr_mark(u32 color) ++{ ++ if (!fb.ptr) ++ return; ++ fb_fill(0, 0, min(fb.width, 96U), min(fb.height, 96U), int2rgb(color)); ++ fb_update(); ++} + + /* Called from Rust: the entry's splash, pixels as b, g, r, 0, centred on black. */ + void bootmgr_paint_splash(const void *pixels, u32 width, u32 height) +@@ -104,6 +131,11 @@ int bootmgr_start(const char *id_arg, const char *devicetree_arg, u64 flags) + return -1; + } + ++ last_launch.launched = true; ++ last_launch.have_devicetree = devicetree_arg; ++ if (devicetree_arg) ++ memcpy(last_launch.devicetree, devicetree, sizeof(devicetree)); ++ + /* The kernel, initrd and DT are m1n1's now, past any limit the host set. */ + heapblock_set_limit(NULL); + +@@ -136,8 +168,57 @@ int bootmgr_start(const char *id_arg, const char *devicetree_arg, u64 flags) + return kboot_boot(kernel); + } + ++/* ++ * m1n1 was restarted by a PSCI SYSTEM_RESET from the kernel: every other CPU ++ * is parked, the watchdog is off and memory is as after a cold start. Do ++ * what the host's orchard launch would: bring up main storage and start the ++ * default entry with the last launch's board DTB. Returns 0 when the kernel ++ * is set up to run, -1 (proxy) otherwise. ++ */ ++int bootmgr_warm_reset(void) ++{ ++ char id[MAX_NAME]; ++ const char *devicetree = last_launch.have_devicetree ? last_launch.devicetree : NULL; ++ ++ bootmgr_mark(MARK_REENTERED); ++ if (psci_reset_mode_get() == PSCI_RESET_PROXY) { ++ printf("bootmgr: warm reset: psci_reset=proxy, not booting\n"); ++ return -1; ++ } ++ if (!last_launch.launched) { ++ printf("bootmgr: warm reset: no boot entry was launched before, not guessing the board\n"); ++ bootmgr_mark(MARK_NO_LAUNCH); ++ return -1; ++ } ++ ++ printf("bootmgr: warm reset: starting the default entry (devicetree %s)\n", ++ devicetree ? devicetree : "from the entry"); ++ if (!main_storage_init()) { ++ printf("bootmgr: warm reset: main storage did not come up\n"); ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ if (rust_boot_default_entry(devicetree, id, sizeof(id)) < 0) { ++ main_storage_shutdown(); ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ bootmgr_mark(MARK_LOADING); ++ if (bootmgr_start(id, devicetree, 0) < 0) { ++ bootmgr_mark(MARK_FAILED); ++ return -1; ++ } ++ return 0; ++} ++ + #else + ++int bootmgr_warm_reset(void) ++{ ++ printf("bootmgr: not supported in this build\n"); ++ return -1; ++} ++ + int bootmgr_start(const char *id, const char *devicetree, u64 flags) + { + UNUSED(id); +diff --git a/src/bootmgr.h b/src/bootmgr.h +index 8a03baa..f3b86ee 100644 +--- a/src/bootmgr.h ++++ b/src/bootmgr.h +@@ -8,5 +8,7 @@ + #define BOOTMGR_NO_SPLASH BIT(0) + + int bootmgr_start(const char *id, const char *devicetree, u64 flags); ++/* After a PSCI SYSTEM_RESET: boot the default entry the way the last launch did */ ++int bootmgr_warm_reset(void); + + #endif +diff --git a/src/fb.c b/src/fb.c +index ba606b2..0aa88b8 100644 +--- a/src/fb.c ++++ b/src/fb.c +@@ -1,6 +1,7 @@ + /* SPDX-License-Identifier: MIT */ + + #include "fb.h" ++#include "psci.h" + #include "assert.h" + #include "iodev.h" + #include "malloc.h" +@@ -464,14 +465,16 @@ void fb_init(bool clear) + &orig_logo); + } + ++ // After a re-entry from Linux the payload area behind m1n1 is stale ++ bool payload_ok = !psci_reentered(); + u32 *splash, splash_w, splash_h; +- if (payload_splash(&splash, &splash_w, &splash_h) && splash_w <= fb.width && ++ if (payload_ok && payload_splash(&splash, &splash_w, &splash_h) && splash_w <= fb.width && + splash_h <= fb.height) { + custom_logo.ptr = splash; + custom_logo.width = splash_w; + custom_logo.height = splash_h; + logo = &custom_logo; +- } else if (payload_logo(&custom_128, &custom_256)) { ++ } else if (payload_ok && payload_logo(&custom_128, &custom_256)) { + custom_logo = *logo; + if (custom_logo.width == 256) { + custom_logo.ptr = custom_256; +diff --git a/src/main.c b/src/main.c +index 3f32380..8966e41 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -7,6 +7,7 @@ + + #include "adt.h" + #include "aic.h" ++#include "bootmgr.h" + #include "cpufreq.h" + #include "display.h" + #include "exception.h" +@@ -127,6 +128,9 @@ void run_actions(void) + printf("Valid payload found\n"); + return; + } ++ ++ if (psci_reset_reentered() && bootmgr_warm_reset() == 0) ++ return; + fb_set_active(true); + + printf("No valid payload found\n"); +@@ -173,9 +177,11 @@ void m1n1_main(void) + // Kick DCP to sleep, so dodgy monitors which cause reconnect cycles don't cause us to lose the + // framebuffer. + display_shutdown(DCP_SLEEP_IF_EXTERNAL); +- // On idevice we need to always clear, because otherwise it looks scuffed on white devices +- fb_init(!is_mac); +- fb_display_logo(); ++ // On idevice we need to always clear, because otherwise it looks scuffed on white devices. ++ // After a warm reset, keep the picture until the next system's splash. ++ fb_init(!is_mac && !psci_reset_reentered()); ++ if (!psci_reset_reentered()) ++ fb_display_logo(); + #ifdef FB_SILENT_MODE + fb_set_active(!cur_boot_args.video.display); + #else +@@ -184,7 +190,9 @@ void m1n1_main(void) + #endif + + cpufreq_fixup(); +- sep_init(); ++ // SEP was never reset; it is still running what it ran at the first boot ++ if (!psci_reset_reentered()) ++ sep_init(); + #endif + + printf("Initialization complete.\n"); diff --git a/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch b/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch new file mode 100644 index 00000000..08935588 --- /dev/null +++ b/boot/m1n1/patches/orchard/0020-kboot-Mark-retained-hardware-after-PSCI-re-entry.patch @@ -0,0 +1,38 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 12:00:00 +0100 +Subject: [PATCH] kboot: Mark retained hardware after PSCI re-entry + +SYSTEM_RESET re-enters m1n1 without resetting all peripherals. Mark the +next kernel's /chosen with the existing linux,booted-from-kexec boolean +so retained hardware is not mistaken for the cold iBoot baseline. + +Use psci_reentered(), which covers both SYSTEM_RESET and the kexec +trampoline. This also preserves the contract when kboot loads a fresh +BLS devicetree instead of inheriting the outgoing kernel's tree. + +The Orchard kernel's G5P, PMGR, PCIe/ANS and AOP consumers all use this +property to select retained-state handling, with their existing hardware +and certification guards. The only other reader is PowerPC-only. No +kernel consumer needs a separate PSCI-specific property. + +Assisted-by: LLM +--- +diff --git a/src/kboot.c b/src/kboot.c +index 1d4b26f..7dc9874 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -278,6 +278,13 @@ static int dt_set_chosen(void) + printf("FDT: %s = '%s'\n", name, value); + } + ++ /* Re-entry retains peripherals, just like a kernel-to-kernel handoff. */ ++ if (psci_reentered()) { ++ if (fdt_setprop(dt, node, "linux,booted-from-kexec", NULL, 0)) ++ bail("FDT: couldn't mark retained hardware after m1n1 re-entry\n"); ++ printf("FDT: linux,booted-from-kexec (m1n1 re-entry)\n"); ++ } ++ + if (initrd_start && initrd_size) { + if (fdt_setprop_u64(dt, node, "linux,initrd-start", (u64)initrd_start)) + bail("FDT: couldn't set chosen.linux,initrd-start property\n"); diff --git a/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch new file mode 100644 index 00000000..6a30fc37 --- /dev/null +++ b/boot/m1n1/patches/orchard/0021-kboot-Pass-the-A9-AOP-s-boot-arguments-and-sensor-ca.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Mon, 2 Nov 2026 15:07:08 +0100 +Subject: [PATCH] kboot: Pass the A9 AOP's boot arguments and sensor + calibration + +The A9's AOP firmware takes more than the ECID from its loader: the AOP +target number (iop-aop-nub aop-target), the chip's ID (/chosen chip-id) +and revision (/arm-io chip-revision). Its accelerometer, gyroscope and +pressure sensor also send no reports until the driver has given them the +unit's factory calibration, which iBoot resolves from syscfg into the +sensors' nodes below iop-aop-nub. + +Copy the three numbers into the AOP's node (alias aop) as +apple,aop-target, apple,chip-id and apple,chip-revision, and each +sensor's calibration properties into the AOP's child node of the same +name, prefixed with "apple,". Unprovisioned "syscfg/" references and +nodes the device tree doesn't have are skipped. The orientation +properties are not copied: the A9 firmware rejects them and the board's +mount-matrix describes the mounting. + +The values are the unit's own and never leave the device: the kernel +sends them to the AOP and nothing else. + +Assisted-by: LLM +--- + src/kboot.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 68 insertions(+), 1 deletion(-) + +diff --git a/src/kboot.c b/src/kboot.c +index ebd6282..7ce9d72 100644 +--- a/src/kboot.c ++++ b/src/kboot.c +@@ -1005,6 +1005,73 @@ static int dt_set_als(void) + return 0; + } + ++/* ++ * The A9's AOP firmware also takes the AOP target number and the chip's ID and ++ * revision in its boot arguments, and each motion sensor's factory ++ * calibration, which iBoot resolves from syscfg into the sensor's node below ++ * iop-aop-nub. Copy them into the AOP's node and its sensor child nodes of the ++ * same name; the calibration keeps its name with an "apple," prefix. ++ */ ++static const char *const aop_sensor_calibrations[] = { ++ "low-temp-accel-offset", ++ "accel-sensitivity-calibration", ++ "accel-range-sensitivity-cal", ++ "accel-interrupt-calibration", ++ "gyro-sensitivity-calibration", ++ "gyro-range-sensitivity-cal", ++ "gyro-interrupt-calibration", ++ "gyro-trim-calibration", ++ "pressure-offset-calibration", ++ "pressure-accel-sensitivity", ++ "pressure-heater-resistance", ++}; ++ ++static int dt_set_aop_sensors(int node) ++{ ++ int nub = adt_path_offset(adt, "/arm-io/aop/iop-aop-nub"); ++ if (nub < 0) ++ return 0; ++ ++ u32 val; ++ if (ADT_GETPROP(adt, nub, "aop-target", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,aop-target", val)) ++ bail("FDT: failed to set apple,aop-target\n"); ++ ++ int chosen = adt_path_offset(adt, "/chosen"); ++ if (chosen >= 0 && ADT_GETPROP(adt, chosen, "chip-id", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-id", val)) ++ bail("FDT: failed to set apple,chip-id\n"); ++ ++ int arm_io = adt_path_offset(adt, "/arm-io"); ++ if (arm_io >= 0 && ADT_GETPROP(adt, arm_io, "chip-revision", &val) >= 0 && ++ fdt_setprop_u32(dt, node, "apple,chip-revision", val)) ++ bail("FDT: failed to set apple,chip-revision\n"); ++ ++ int sensor = nub; ++ ADT_FOREACH_CHILD(adt, sensor) ++ { ++ const char *name = adt_get_name(adt, sensor); ++ ++ for (size_t i = 0; i < ARRAY_SIZE(aop_sensor_calibrations); i++) { ++ u32 len; ++ const u8 *blob = adt_getprop(adt, sensor, aop_sensor_calibrations[i], &len); ++ if (!blob || !len || (len >= 7 && !memcmp(blob, "syscfg/", 7))) ++ continue; ++ ++ /* Earlier properties may have moved the sensor's node. */ ++ int child = fdt_subnode_offset(dt, node, name); ++ if (child < 0) ++ break; ++ ++ char prop[64]; ++ snprintf(prop, sizeof(prop), "apple,%s", aop_sensor_calibrations[i]); ++ if (fdt_setprop(dt, child, prop, blob, len)) ++ bail("FDT: failed to set %s/%s\n", name, prop); ++ } ++ } ++ return 0; ++} ++ + /* + * The AOP's firmware carries the chip's ECID in its boot arguments. Pass it + * only to the AOP's node; the driver removes it from the tree once read. +@@ -1028,7 +1095,7 @@ static int dt_set_aop(void) + + if (fdt_setprop_u64(dt, node, "apple,ecid", ecid)) + bail("FDT: failed to set apple,ecid\n"); +- return 0; ++ return dt_set_aop_sensors(node); + } + + static int dt_set_multitouch(void) diff --git a/boot/m1n1/psci-loader b/boot/m1n1/psci-loader new file mode 100755 index 00000000..c21f1801 --- /dev/null +++ b/boot/m1n1/psci-loader @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Append m1n1 PSCI payload variables to a bare m1n1.bin. + +m1n1 entered at EL3 (A7-A9 from Pongo) provides PSCI by default, including +SYSTEM_RESET as a warm reboot into the default boot entry; a bare m1n1.bin +needs no framing for that. Use this to get the other settings, before +assembling any kernel/DTB/logo payload. The output works with +orchard run/build --loader or launch's loaders/m1n1.bin. m1n1 entered at EL1 +always uses the spin table. +""" +import argparse +from pathlib import Path +import shutil + +RESET = {'boot': b'', 'proxy': b'psci_reset=proxy\n', 'off': b'psci_reset=0\n'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--device', required=True, choices=('j71t',)) + parser.add_argument('--loader', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + group = parser.add_mutually_exclusive_group() + group.add_argument('--system-reset', choices=tuple(RESET), default='boot', + help='boot: warm reboot into the default entry (default); proxy: stop in ' + 'the m1n1 proxy; off: NOT_SUPPORTED, kernel resets the SoC itself') + group.add_argument('--no-psci', action='store_true', help='psci=0: spin table, no PSCI') + args = parser.parse_args() + if not args.loader.is_file() or not args.loader.stat().st_size: + parser.error('input must be a nonempty bare m1n1.bin') + # Exclusive creation: never replace the original or another boot artifact. + with args.loader.open('rb') as source, args.output.open('xb') as output: + shutil.copyfileobj(source, output) + output.write(b'psci=0\n' if args.no_psci else b'psci=1\n' + RESET[args.system_reset]) + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/chips/s8000.toml b/chips/s8000.toml index 292e4f80..a2756a52 100644 --- a/chips/s8000.toml +++ b/chips/s8000.toml @@ -15,3 +15,4 @@ stages = ["iBSS", "iBEC"] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" platform = "xnu; same core as S8003, which ran on J71t" stages = "remote_boot two-stage CPID list" +usb_host = "apple,s8000-usb-role-switch: AUSB PHY shared by DWC2 and EHCI/OHCI" diff --git a/chips/s8003.toml b/chips/s8003.toml index e152fbed..40c86c20 100644 --- a/chips/s8003.toml +++ b/chips/s8003.toml @@ -13,5 +13,6 @@ stages = ["iBSS", "iBEC"] [evidence] page_size = "xnu-6153.141.1 osfmk/arm/proc_reg.h" -platform = "J71t on hardware" +platform = "J71t on hardware; USB serial and networking verified" stages = "remote_boot two-stage CPID list" +usb_host = "apple,s8000-usb-role-switch: AUSB PHY shared by DWC2 and EHCI/OHCI" diff --git a/devices/d10/device.toml b/devices/d10/device.toml index 37c7ff01..8be02219 100644 --- a/devices/d10/device.toml +++ b/devices/d10/device.toml @@ -33,6 +33,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -58,3 +60,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d101/device.toml b/devices/d101/device.toml index 46b30321..e119da44 100644 --- a/devices/d101/device.toml +++ b/devices/d101/device.toml @@ -33,6 +33,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -58,3 +60,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d11/device.toml b/devices/d11/device.toml index 83e1a85d..ec6f822f 100644 --- a/devices/d11/device.toml +++ b/devices/d11/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d111/device.toml b/devices/d111/device.toml index 99c20fbc..862c69e3 100644 --- a/devices/d111/device.toml +++ b/devices/d111/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d20/device.toml b/devices/d20/device.toml index fd995d68..8af49d42 100644 --- a/devices/d20/device.toml +++ b/devices/d20/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d201/device.toml b/devices/d201/device.toml index a973d29c..4856be86 100644 --- a/devices/d201/device.toml +++ b/devices/d201/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d21/device.toml b/devices/d21/device.toml index 9ecaee92..a615494f 100644 --- a/devices/d21/device.toml +++ b/devices/d21/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d211/device.toml b/devices/d211/device.toml index af13003d..60fe3550 100644 --- a/devices/d211/device.toml +++ b/devices/d211/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d22/device.toml b/devices/d22/device.toml index 938de5b5..ba8eb8f1 100644 --- a/devices/d22/device.toml +++ b/devices/d22/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "missing" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/d221/device.toml b/devices/d221/device.toml index fe36b4fa..cdddf3f2 100644 --- a/devices/d221/device.toml +++ b/devices/d221/device.toml @@ -31,6 +31,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "working" screen_off = "working" # apple_dsi_bl: bl_power off sends DBV 0, the panel goes dark; on restores the level. backlight = "working" # apple_dsi_bl: DCS 0x51 to the OLED panel (docs/bring-up/d221-backlight.md). @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j1/device.toml b/devices/j1/device.toml index 79d533fc..c48854b6 100644 --- a/devices/j1/device.toml +++ b/devices/j1/device.toml @@ -33,6 +33,8 @@ status = "experimental" # with GCC 11.4 and the boot kit's config (README.md), but orchard's own clang # build has not booted on a J1 yet. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" # simplefb on iBoot's scanout; a userspace dashboard drew to fb0 on the LCD. screen_off = "missing" backlight = "missing" @@ -59,3 +61,4 @@ usb_network = "untested" # CDC Ethernet, pinged both ways. audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j120/device.toml b/devices/j120/device.toml index 54bdeeb0..70b49024 100644 --- a/devices/j120/device.toml +++ b/devices/j120/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j121/device.toml b/devices/j121/device.toml index e952846b..abad08d2 100644 --- a/devices/j121/device.toml +++ b/devices/j121/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j127/device.toml b/devices/j127/device.toml index edfcceea..4bb732e5 100644 --- a/devices/j127/device.toml +++ b/devices/j127/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j128/device.toml b/devices/j128/device.toml index eccb2be0..49726f10 100644 --- a/devices/j128/device.toml +++ b/devices/j128/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j171/device.toml b/devices/j171/device.toml index 7e949d12..d26ad791 100644 --- a/devices/j171/device.toml +++ b/devices/j171/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "working" screen_off = "untested" backlight = "working" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "working" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j172/device.toml b/devices/j172/device.toml index b51d8510..c0305c07 100644 --- a/devices/j172/device.toml +++ b/devices/j172/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "untested" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j207/device.toml b/devices/j207/device.toml index 9f1b8d82..4da0903e 100644 --- a/devices/j207/device.toml +++ b/devices/j207/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j208/device.toml b/devices/j208/device.toml index a05ce52d..1204f2d2 100644 --- a/devices/j208/device.toml +++ b/devices/j208/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j71/device.toml b/devices/j71/device.toml index 4103d1d5..d66ff579 100644 --- a/devices/j71/device.toml +++ b/devices/j71/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j71b/device.toml b/devices/j71b/device.toml index c2aa8231..edcebc82 100644 --- a/devices/j71b/device.toml +++ b/devices/j71b/device.toml @@ -27,6 +27,8 @@ status = "supported" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "working" screen_off = "working" backlight = "working" @@ -52,3 +54,4 @@ usb_network = "working" audio_output = "working" audio_input = "missing" camera = "missing" +video_decode_hevc = "untested" diff --git a/devices/j71s/device.toml b/devices/j71s/device.toml index 7db1ed12..05b476f8 100644 --- a/devices/j71s/device.toml +++ b/devices/j71s/device.toml @@ -32,6 +32,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "untested" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -57,3 +59,4 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" +video_decode_hevc = "untested" diff --git a/devices/j71t/device.toml b/devices/j71t/device.toml index 1a78c1d1..099c12f7 100644 --- a/devices/j71t/device.toml +++ b/devices/j71t/device.toml @@ -29,14 +29,15 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +warm_restart = "working" display = "untested" screen_off = "untested" backlight = "untested" touch = "untested" buttons = "untested" -accelerometer = "missing" -gyroscope = "missing" -light_sensor = "untested" +accelerometer = "working" +gyroscope = "working" +light_sensor = "working" touch_id = "missing" storage_read = "untested" storage_write = "untested" @@ -49,8 +50,10 @@ charging = "untested" charging_control = "untested" suspend = "untested" resume = "untested" -usb_serial = "untested" -usb_network = "untested" +usb_serial = "working" +usb_network = "working" +usb_host = "working" audio_output = "untested" audio_input = "untested" camera = "missing" +video_decode_hevc = "working" diff --git a/devices/j71t/firmware.json b/devices/j71t/firmware.json index 3321e246..39e7e406 100644 --- a/devices/j71t/firmware.json +++ b/devices/j71t/firmware.json @@ -113,6 +113,12 @@ }, "device": "/usr/share/firmware/multitouch/J71t.mtprops", "sha256": "69f27fe207cc5810a748dac3dfe640b1c5a755709f49513b4b5125660784f0fc" + }, + "aop.im4p": { + "ipsw": { + "zip": "Firmware/AOP/aopfw-ipad6faop.im4p" + }, + "sha256": "5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa" } }, "outputs": [ @@ -279,6 +285,44 @@ "calibration_size": 1024 } ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f.bin", + "sha256": "af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP" + } + ] + }, + { + "destination": "lib/firmware/apple/aop-ipad6f-ext.bin", + "sha256": "2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22", + "steps": [ + { + "step": "im4p-payload@1", + "input": "aop.im4p" + }, + { + "step": "aop-image@1", + "coprocessor": "AOP_ext" + } + ] + }, + { + "destination": "lib/firmware/apple/agx-g5p-c0.ldr", + "sha256": "33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca", + "steps": [ + { + "step": "agx-g5p-ldr-from-kernelcache@1", + "input": "kernelcache" + } + ] } ] } diff --git a/devices/j72b/device.toml b/devices/j72b/device.toml index 00996e6c..f0c10a71 100644 --- a/devices/j72b/device.toml +++ b/devices/j72b/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "untested" audio_input = "missing" camera = "missing" +video_decode_hevc = "untested" diff --git a/devices/j72s/device.toml b/devices/j72s/device.toml index f64c9e5f..570eae3c 100644 --- a/devices/j72s/device.toml +++ b/devices/j72s/device.toml @@ -32,6 +32,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "untested" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -57,3 +59,4 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" +video_decode_hevc = "untested" diff --git a/devices/j72t/device.toml b/devices/j72t/device.toml index 38ba1313..6d602124 100644 --- a/devices/j72t/device.toml +++ b/devices/j72t/device.toml @@ -29,6 +29,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "untested" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -54,3 +56,4 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" +video_decode_hevc = "untested" diff --git a/devices/j73/device.toml b/devices/j73/device.toml index 76c7f04a..8534e048 100644 --- a/devices/j73/device.toml +++ b/devices/j73/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j81/device.toml b/devices/j81/device.toml index 5c750799..1e516520 100644 --- a/devices/j81/device.toml +++ b/devices/j81/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "working" screen_off = "untested" backlight = "working" @@ -55,3 +57,4 @@ usb_network = "working" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j82/device.toml b/devices/j82/device.toml index 0071e63e..e402f380 100644 --- a/devices/j82/device.toml +++ b/devices/j82/device.toml @@ -30,6 +30,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j85/device.toml b/devices/j85/device.toml index 7b2efdb4..619c335d 100644 --- a/devices/j85/device.toml +++ b/devices/j85/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j85m/device.toml b/devices/j85m/device.toml index 512eb26c..e9c8a9b2 100644 --- a/devices/j85m/device.toml +++ b/devices/j85m/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j86/device.toml b/devices/j86/device.toml index 8ceb823b..a6ad481b 100644 --- a/devices/j86/device.toml +++ b/devices/j86/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j86m/device.toml b/devices/j86m/device.toml index b0cecdc6..88272227 100644 --- a/devices/j86m/device.toml +++ b/devices/j86m/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j87/device.toml b/devices/j87/device.toml index efab182e..aa4596c2 100644 --- a/devices/j87/device.toml +++ b/devices/j87/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j87m/device.toml b/devices/j87m/device.toml index f253466c..6119cd26 100644 --- a/devices/j87m/device.toml +++ b/devices/j87m/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j96/device.toml b/devices/j96/device.toml index f73e8a35..a1369817 100644 --- a/devices/j96/device.toml +++ b/devices/j96/device.toml @@ -33,6 +33,8 @@ status = "experimental" # battery gauge were seen working on a local kernel tree with the same device tree # (touch on a different driver there), not yet on Orchard's build. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -58,3 +60,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j97/device.toml b/devices/j97/device.toml index f7bddef4..e235c526 100644 --- a/devices/j97/device.toml +++ b/devices/j97/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j98a/device.toml b/devices/j98a/device.toml index 1e4de72c..29ab7113 100644 --- a/devices/j98a/device.toml +++ b/devices/j98a/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/j99a/device.toml b/devices/j99a/device.toml index ce2762b4..29d6869e 100644 --- a/devices/j99a/device.toml +++ b/devices/j99a/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n51/device.toml b/devices/n51/device.toml index bd811030..8ece85df 100644 --- a/devices/n51/device.toml +++ b/devices/n51/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n53/device.toml b/devices/n53/device.toml index 835f3963..595da115 100644 --- a/devices/n53/device.toml +++ b/devices/n53/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n56/device.toml b/devices/n56/device.toml index fbe1eb31..4cd2bf3c 100644 --- a/devices/n56/device.toml +++ b/devices/n56/device.toml @@ -31,6 +31,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -56,3 +58,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n61/device.toml b/devices/n61/device.toml index 462fd978..4a485b59 100644 --- a/devices/n61/device.toml +++ b/devices/n61/device.toml @@ -29,6 +29,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "working" # The minimal-boot framebuffer console was visible on the LCD. screen_off = "untested" backlight = "untested" @@ -54,3 +56,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n66/device.toml b/devices/n66/device.toml index 445604e7..17127aa9 100644 --- a/devices/n66/device.toml +++ b/devices/n66/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n66m/device.toml b/devices/n66m/device.toml index 271a9576..2d9b673e 100644 --- a/devices/n66m/device.toml +++ b/devices/n66m/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n69/device.toml b/devices/n69/device.toml index 1274d806..ee15bca4 100644 --- a/devices/n69/device.toml +++ b/devices/n69/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n69u/device.toml b/devices/n69u/device.toml index 1b8ad683..92138bf6 100644 --- a/devices/n69u/device.toml +++ b/devices/n69u/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n71/device.toml b/devices/n71/device.toml index 5e65cd00..789eafa4 100644 --- a/devices/n71/device.toml +++ b/devices/n71/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/n71m/device.toml b/devices/n71m/device.toml index ef312d54..ad239ad5 100644 --- a/devices/n71m/device.toml +++ b/devices/n71m/device.toml @@ -30,6 +30,8 @@ status = "experimental" # this board with Orchard: "untested" means a driver binds in its device tree # (tools/catalog/drivers.json), plus the display's boot console. [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "missing" backlight = "untested" @@ -55,3 +57,4 @@ usb_network = "untested" audio_output = "missing" audio_input = "missing" camera = "missing" +video_decode_hevc = "missing" diff --git a/devices/p105/device.toml b/devices/p105/device.toml index 5cdcb0c0..7da83f4a 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -37,6 +37,8 @@ status = "experimental" # States: working, untested, missing, none (features.toml). [features] +usb_host = "missing" +warm_restart = "untested" display = "untested" screen_off = "untested" backlight = "untested" @@ -66,3 +68,4 @@ usb_network = "untested" audio_output = "untested" audio_input = "untested" camera = "missing" +video_decode_hevc = "missing" diff --git a/docs/bring-up/charging/j71t-mac-current.md b/docs/bring-up/charging/j71t-mac-current.md new file mode 100644 index 00000000..af31232a --- /dev/null +++ b/docs/bring-up/charging/j71t-mac-current.md @@ -0,0 +1,119 @@ +# J71t Mac extra-current candidate + +Host-tested on 2026-11-08; not yet qualified on an iPad. The matching root +initrd reads `/etc/orchard/mac-usb-current` from the mounted system. Exactly +`1` on `apple,j71t` selects the compatibility identity `05ac:1014`, explicit +IAD device class ef/02/01, and the existing ACM+NCM configuration at +500 mA MaxPower. Omarchy A carries that opt-in as a separate system change. +Removing the file selects the normal `05ac:4145` identity on the next boot. +Recovery, installer and RAM/NFS gadgets retain their existing identities. + +This Mac's AppleUSBHostMergeProperties personality lists `1014` in its +2400 mA UsbPowerSinkCapability table; `4145` is absent. The generic +AppleUSBHostCompositeDevice power-negotiation path can therefore allocate +extra power and send vendor device request `40/40`, zero length, signed +suspend mA in wValue and signed extra active mA in wIndex. No usbmux interface +or iOS preferred-configuration protocol is needed for this PID. Its IAD +class ef/02/01 matches Apple's generic composite personality. AppleUSBCDC's +matching IAD personality has IOMatchDefer=True; its non-deferred vendor +personality instead requires 0/0/0. Both ACM and NCM already emit IADs. The strings +continue to identify Orchard and Omarchy. This is a compatibility identity, +not an assigned Orchard PID or a claim to implement an Apple product. + +A new identity requires macOS accessory authorization, possibly on every boot +because the existing BOOT serial changes. Keep the Mac unlocked and reconnect +the cable if the device remains unregistered/unmatched. Do not silently change +the Mac's security settings. Orchard discovery recognizes this PID as a +system only when the existing serial contains a boot UUID and BOARD:j71t. + +The identity is never current evidence on the tablet. Only the host's real +request reaches composite_vbus_draw -> DWC2 -> lightning. The charger clamps +the allowance at its board maximum and retains pack guards, stepped ramp, +readback verification and source-epoch withdrawal. No request means the +existing 500 mA fallback. BC1.2 primary/secondary detection already exists; +SDP on this Mac must not be relabeled CDP just to obtain 1.5 A. + +A grant arriving while suspended cannot select its active allowance. The +existing policy discards the preceding active grant on suspend; resuming +needs a fresh grant. Reset and disconnect also clear it. The supplied suspend +field is logged but does not authorize an elevated suspend limit. This work +does not certify the pre-existing charger's USB suspend/preconfiguration +compliance: its unknown-source hardware fallback remains 500 mA. + +`current_max=0` on a configured link is a separate possible blocker. A report +is accepted only while present, with data enabled, in the current PMU source +epoch. Logs now show reported/accepted current, present/data and report epoch; +dynamic debug on the charger shows stale report/source epochs. Do not replay +an old allowance after a PMU event to make this number nonzero. A packet trace +and these logs must establish whether the live zero was a missing report, a +gated report, or epoch invalidation. That observation is still required. + +## Host verification + +Run `ORCHARD_KERNEL_SOURCE=/path/to/prepared/kernel PYTHONPATH=src python3 +-m unittest discover -s tests -p test_j71t_mac_current.py -v`. The C test +extracts the actual production functions and executes them with ASan/UBSan: +all request-type/request/short-length/vendor combinations, all signed extra +values, suspend, board caps, ramp/revoke, attachment and stale epochs. +The other tests run the initrd gadget function in a mock filesystem and test +both host discovery implementations. They do not contact USB hardware. + +## Device acceptance check + +Boot the new Image, DTB, matching initrd and modules together, with the +Omarchy A opt-in present. Do not rebind a live gadget to edit its identity. + +On the unlocked Mac, approve the new accessory and capture `ioreg -p IOUSB +-l -w0` plus `ioreg -l -w0 -r -c IOUSBHostDevice`. The device should be +registered/matched/configured, have PID 0x1014, MaxPower 500, +UsbPowerSinkCapability 2400 and UsbPowerSinkAllocation above 500 (2400 if +available). Check AppleUSBHostCompositeDevice binding and +kUSBConfigurationCurrentOverride. The host's available 3000 mA alone is not +a grant. Preserve ACM serial and NCM network connectivity. + +On the tablet, enable dynamic debug for composite.c, dwc2/gadget.c and +apple_tristar_charger.c. Capture dmesg and lightning's online, usb_type, +current_max and input_current_limit alongside battery's status/current_now, +capacity, temperature, voltage and the UDC state. A usual full grant is +extra=1900, giving accepted/current_max=2400000, followed by a guarded ramp +to input_current_limit=2400000. Never exceed min(500+signed extra, 2400). +A smaller grant must produce a correspondingly smaller limit, rounded down. +SDP may remain correct; it does not imply that a software grant failed. + +At a moderate state of charge and normal pack temperature, hold brightness +and desktop workload fixed for at least 60 seconds: battery status must be +Charging and signed current_now positive. Full/thermal-limited packs are not +a valid charging-current acceptance test. If current_max stays zero, preserve +the report/epoch logs and EP0 trace; do not force a PMIC limit. + +Also test: a plain 500 mA host (no extra request), reset, unplug/replug to a +different source, a smaller/revoked grant, and Mac sleep/wake. Any loss of a +valid grant must stop elevated draw and return to the fallback. Suspend may +lose extra current until macOS issues a new grant; record that limitation. +Mac allocation without an actual request and accepted allowance is a failed +test, even if battery current happens to be positive with the panel off. + +## Sustained NCM transfers + +The runner reported two carrier-up stalls with ACM still working, recovered +by cycling only usb0. USB patch 0012 fixes a reproducible NCM timer bug: +NETDEV_TX_BUSY during a timer flush used to abandon the partial NTB. Retry +at the usual 300 us interval, advancing the timer expiry before restart. +This is a supported cause of silent NCM stalls; the runner's exact endpoint +state was not captured, so it is not proof that both incidents had this cause. + +A 16 KiB NTB does not need to fit in a dedicated FIFO at once: DWC2 splits +transfers and streams USB packets through the FIFO. The 2056-entry total is +in 32-bit words. No FIFO, DMA, ZLP or negotiated NTB-size change is justified +by those totals alone. ACM surviving also differs from the known shared-OUT +FIFO starvation fixed by USB patch 0008. Do not change those controls without +capturing endpoint state and request completions during a stall. + +Run test_ncm_flush.py with ORCHARD_KERNEL_SOURCE to execute the production +callback under repeated request exhaustion and subsequent recovery. On the +next device run count every `usb0-stall-bounce` marker, including the runner's +transient oi-usb0-watchdog. Use one watchdog at a time so bounces are attributable. +The Omarchy service's new carrier-up fallback cycles only usb0 after 15 seconds +without replies from the previously reachable 172.16.42.2 peer. It leaves +UDC, ACM and host charging-grant ownership intact. A clean sustained-transfer +acceptance run requires zero watchdog bounces, not merely successful retries. diff --git a/docs/bring-up/j71t-04-evidence.md b/docs/bring-up/j71t-04-evidence.md new file mode 100644 index 00000000..43c4a2e4 --- /dev/null +++ b/docs/bring-up/j71t-04-evidence.md @@ -0,0 +1,16 @@ +# J71t 0.4 qualification + +The Nov 8 integration observations predate the rebased 0.4 topic branches. +They do not qualify these revisions. No feature is promoted to working here. + +Merge device/chip changes with their implementing topics. Keep suspend/resume +untested until real wake, peripheral resume, long sleep and stress tests pass. +Camera and Touch ID remain missing and outside 0.4. + +Vocabulary follow-ups: GPU/GL, warm reboot, RTC alarm, cover wake, pressure, +and hardware video decode need distinct definitions. RTC presence is not alarm +qualification; cover polling is not wake; pairing is not Bluetooth input/audio. + +Required receipts: exact Orchard/kernel/Mesa/loader revisions, artifact hashes, +boot invocation, boot ID, logs, observed successes and failures, and the other +board maintainer's J71b regression result for shared changes. diff --git a/docs/compatibility.md b/docs/compatibility.md index 25ce8915..4d4eba38 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -136,118 +136,119 @@ Not generated while in review: `j72` (PR #59). Boards Orchard builds for. States: working, untested, missing, none (no such hardware). -| Board | Display | Screen on/off | Backlight | Touchscreen | Buttons | Accelerometer | Gyroscope | Light sensor | Touch ID | Internal storage read | Internal storage write | Wi-Fi | Bluetooth input devices | Bluetooth audio | Cellular | Battery status | Charging | Charging control | Suspend | Resume | USB serial | USB network | Speakers | Microphone | Cameras | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `p105` | untested | untested | untested | untested | untested | missing | missing | missing | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | missing | missing | untested | untested | untested | untested | missing | -| `j1` | untested | missing | missing | untested | missing | missing | missing | missing | none | missing | missing | untested | missing | missing | none | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | -| `j71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j73` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j85` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j85m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j86` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j86m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j87` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j87m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n51` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n53` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j96` | untested | missing | untested | untested | untested | missing | missing | missing | missing | missing | missing | untested | missing | missing | none | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j97` | untested | missing | untested | untested | untested | missing | missing | missing | missing | missing | missing | untested | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n56` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n61` | working | untested | untested | working | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | missing | missing | untested | untested | working | untested | missing | missing | missing | -| `j81` | working | untested | working | working | working | missing | missing | missing | missing | missing | missing | working | untested | untested | none | working | missing | missing | untested | untested | working | working | missing | missing | missing | -| `j82` | untested | untested | untested | untested | untested | missing | missing | missing | missing | missing | missing | untested | untested | untested | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j71s` | untested | untested | untested | untested | untested | missing | missing | untested | missing | missing | missing | missing | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | -| `j72s` | untested | untested | untested | untested | untested | missing | missing | untested | missing | missing | missing | missing | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | -| `n66` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n69u` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j71t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | -| `j72t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | -| `n66m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n69` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `n71m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j127` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j128` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j98a` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j99a` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d10` | untested | untested | untested | missing | untested | untested | missing | missing | missing | untested | untested | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d101` | untested | untested | untested | missing | untested | untested | missing | missing | missing | untested | untested | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d11` | untested | untested | untested | untested | untested | untested | missing | untested | missing | untested | untested | untested | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d111` | untested | untested | untested | untested | untested | untested | missing | untested | missing | untested | untested | untested | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j171` | working | untested | working | working | working | untested | missing | untested | missing | working | working | working | missing | missing | none | working | untested | missing | untested | untested | untested | untested | working | missing | missing | -| `j172` | untested | untested | untested | untested | untested | untested | missing | untested | missing | untested | untested | untested | missing | missing | missing | untested | untested | missing | untested | untested | untested | untested | untested | missing | missing | -| `j71b` | working | working | working | working | working | working | missing | missing | missing | working | working | working | working | working | none | working | working | working | working | working | working | working | working | missing | missing | -| `j72b` | untested | untested | untested | untested | untested | untested | missing | missing | missing | untested | untested | untested | missing | missing | missing | untested | untested | untested | untested | untested | untested | untested | untested | missing | missing | -| `j120` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j121` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j207` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `j208` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d20` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d201` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d21` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d211` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d22` | untested | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | -| `d221` | working | working | working | working | working | missing | missing | missing | none | working | working | untested | missing | missing | missing | working | missing | missing | untested | untested | working | untested | missing | missing | missing | +| Board | Display | Screen on/off | Backlight | Touchscreen | Buttons | Accelerometer | Gyroscope | Light sensor | Touch ID | Internal storage read | Internal storage write | Wi-Fi | Bluetooth input devices | Bluetooth audio | Cellular | Battery status | Charging | Charging control | Suspend | Resume | USB serial | USB network | USB host | Speakers | Microphone | Cameras | Hardware H.265 video decoding | Warm restart | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `p105` | untested | untested | untested | untested | untested | missing | missing | missing | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | missing | missing | untested | untested | missing | untested | untested | missing | missing | untested | +| `j1` | untested | missing | missing | untested | missing | missing | missing | missing | none | missing | missing | untested | missing | missing | none | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | missing | untested | +| `j71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j73` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j85` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j85m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j86` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j86m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j87` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j87m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n51` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n53` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j96` | untested | missing | untested | untested | untested | missing | missing | missing | missing | missing | missing | untested | missing | missing | none | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j97` | untested | missing | untested | untested | untested | missing | missing | missing | missing | missing | missing | untested | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n56` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n61` | working | untested | untested | working | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | missing | missing | untested | untested | working | untested | missing | missing | missing | missing | missing | untested | +| `j81` | working | untested | working | working | working | missing | missing | missing | missing | missing | missing | working | untested | untested | none | working | missing | missing | untested | untested | working | working | missing | missing | missing | missing | missing | untested | +| `j82` | untested | untested | untested | untested | untested | missing | missing | missing | missing | missing | missing | untested | untested | untested | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j71s` | untested | untested | untested | untested | untested | missing | missing | untested | missing | missing | missing | missing | untested | untested | none | untested | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | untested | untested | +| `j72s` | untested | untested | untested | untested | untested | missing | missing | untested | missing | missing | missing | missing | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | untested | untested | +| `n66` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n69u` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n71` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j71t` | untested | untested | untested | untested | untested | working | working | working | missing | untested | untested | untested | untested | untested | none | untested | untested | untested | untested | untested | working | working | working | untested | untested | missing | working | working | +| `j72t` | untested | untested | untested | untested | untested | missing | missing | untested | missing | untested | untested | untested | untested | untested | missing | untested | untested | untested | untested | untested | untested | untested | untested | untested | untested | missing | untested | untested | +| `n66m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n69` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `n71m` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j127` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j128` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j98a` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j99a` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d10` | untested | untested | untested | missing | untested | untested | missing | missing | missing | untested | untested | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d101` | untested | untested | untested | missing | untested | untested | missing | missing | missing | untested | untested | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d11` | untested | untested | untested | untested | untested | untested | missing | untested | missing | untested | untested | untested | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d111` | untested | untested | untested | untested | untested | untested | missing | untested | missing | untested | untested | untested | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j171` | working | untested | working | working | working | untested | missing | untested | missing | working | working | working | missing | missing | none | working | untested | missing | untested | untested | untested | untested | missing | working | missing | missing | missing | untested | +| `j172` | untested | untested | untested | untested | untested | untested | missing | untested | missing | untested | untested | untested | missing | missing | missing | untested | untested | missing | untested | untested | untested | untested | missing | untested | missing | missing | missing | untested | +| `j71b` | working | working | working | working | working | working | missing | missing | missing | working | working | working | working | working | none | working | working | working | working | working | working | working | missing | working | missing | missing | untested | untested | +| `j72b` | untested | untested | untested | untested | untested | untested | missing | missing | missing | untested | untested | untested | missing | missing | missing | untested | untested | untested | untested | untested | untested | untested | missing | untested | missing | missing | untested | untested | +| `j120` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j121` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j207` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `j208` | untested | missing | untested | missing | untested | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d20` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d201` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d21` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d211` | untested | missing | untested | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d22` | untested | missing | missing | missing | missing | missing | missing | missing | missing | untested | untested | missing | missing | missing | missing | untested | missing | missing | untested | untested | untested | untested | missing | missing | missing | missing | missing | untested | +| `d221` | working | working | working | working | working | missing | missing | missing | none | working | working | untested | missing | missing | missing | working | missing | missing | untested | untested | working | untested | missing | missing | missing | missing | missing | untested | ## Drivers A yes means an enabled device tree node binds a driver in `7.3.0-rc1-orchard-apple-16k`, `7.3.0-rc1-orchard-apple-4k`. It does not mean the feature works. Read by `tools/catalog/generate kernel`. -| Board | Status | Backlight | Touchscreen | Buttons | Accelerometer | Light sensor | Internal storage read | Internal storage write | Wi-Fi | Bluetooth input devices | Bluetooth audio | Battery status | Charging | USB serial | USB network | Speakers | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `j71` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j73` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j85` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j85m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j86` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j86m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j87` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j87m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `n51` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `n53` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j96` | experimental | yes | yes | yes | – | – | – | – | yes | – | – | yes | – | yes | yes | – | -| `j97` | experimental | yes | yes | yes | – | – | – | – | yes | – | – | yes | – | yes | yes | – | -| `n56` | experimental | yes | – | yes | – | – | – | – | – | – | – | yes | – | yes | yes | – | -| `n61` | experimental | yes | yes | yes | – | – | – | – | – | – | – | yes | – | yes | yes | – | -| `j81` | experimental | yes | yes | yes | – | – | – | – | yes | yes | yes | yes | – | yes | yes | – | -| `j82` | experimental | yes | yes | yes | – | – | – | – | yes | yes | yes | yes | – | yes | yes | – | -| `j71s` | experimental | yes | yes | yes | – | yes | – | – | – | yes | yes | yes | yes | yes | yes | yes | -| `j72s` | experimental | yes | yes | yes | – | yes | – | – | – | yes | yes | yes | yes | yes | yes | yes | -| `n66` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `n69u` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `n71` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j71t` | experimental | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| `j72t` | experimental | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| `n66m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `n69` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `n71m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j127` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j128` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j98a` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j99a` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `d10` | experimental | yes | – | yes | yes | – | yes | yes | – | yes | yes | – | – | yes | yes | – | -| `d101` | experimental | yes | – | yes | yes | – | yes | yes | – | yes | yes | – | – | yes | yes | – | -| `d11` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | yes | – | -| `d111` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | yes | – | -| `j171` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| `j172` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| `j71b` | supported | yes | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| `j72b` | experimental | yes | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| `j120` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j121` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j207` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `j208` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | -| `d20` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | -| `d201` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | -| `d21` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | -| `d211` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | -| `d22` | experimental | yes | yes | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | -| `d221` | experimental | yes | yes | – | – | – | yes | yes | yes | – | – | yes | – | yes | yes | – | +| Board | Status | Backlight | Touchscreen | Buttons | Accelerometer | Light sensor | Internal storage read | Internal storage write | Wi-Fi | Bluetooth input devices | Bluetooth audio | Battery status | Charging | USB serial | USB network | USB host | Speakers | Hardware H.265 video decoding | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `j71` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j73` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j85` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j85m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j86` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j86m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j87` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j87m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `n51` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `n53` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j96` | experimental | yes | yes | yes | – | – | – | – | yes | – | – | yes | – | yes | yes | – | – | – | +| `j97` | experimental | yes | yes | yes | – | – | – | – | yes | – | – | yes | – | yes | yes | – | – | – | +| `n56` | experimental | yes | – | yes | – | – | – | – | – | – | – | yes | – | yes | yes | – | – | – | +| `n61` | experimental | yes | yes | yes | – | – | – | – | – | – | – | yes | – | yes | yes | – | – | – | +| `j81` | experimental | yes | yes | yes | – | – | – | – | yes | yes | yes | yes | – | yes | yes | – | – | – | +| `j82` | experimental | yes | yes | yes | – | – | – | – | yes | yes | yes | yes | – | yes | yes | – | – | – | +| `j71s` | experimental | yes | yes | yes | – | yes | – | – | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| `j72s` | experimental | yes | yes | yes | – | yes | – | – | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| `n66` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `n69u` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `n71` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j71t` | experimental | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| `j72t` | experimental | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| `n66m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `n69` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `n71m` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j127` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j128` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j98a` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j99a` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `d10` | experimental | yes | – | yes | yes | – | yes | yes | – | yes | yes | – | – | yes | yes | – | – | – | +| `d101` | experimental | yes | – | yes | yes | – | yes | yes | – | yes | yes | – | – | yes | yes | – | – | – | +| `d11` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | yes | – | – | – | +| `d111` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | yes | – | – | – | +| `j171` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | – | +| `j172` | experimental | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | – | +| `j71b` | supported | yes | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | yes | +| `j72b` | experimental | yes | yes | yes | yes | – | yes | yes | yes | yes | yes | yes | yes | yes | yes | – | yes | yes | +| `j120` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j121` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j207` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `j208` | experimental | yes | – | yes | – | – | – | – | – | – | – | – | – | yes | yes | – | – | – | +| `d20` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | – | – | +| `d201` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | – | – | +| `d21` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | – | – | +| `d211` | experimental | yes | – | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | – | – | +| `d22` | experimental | yes | yes | – | – | – | yes | yes | – | – | – | yes | – | yes | yes | – | – | – | +| `d221` | experimental | yes | yes | – | – | – | yes | yes | yes | – | – | yes | – | yes | yes | – | – | – | ### Where feature states and drivers disagree | Board | Feature | | | --- | --- | --- | +| `j71t` | Accelerometer | working, but no driver binds | | `d10` | Bluetooth input devices | missing, but a driver binds | | `d10` | Bluetooth audio | missing, but a driver binds | | `d101` | Bluetooth input devices | missing, but a driver binds | diff --git a/docs/device-protocol.md b/docs/device-protocol.md index f15bfac0..518087d8 100644 --- a/docs/device-protocol.md +++ b/docs/device-protocol.md @@ -15,6 +15,24 @@ the host and device side of an install always come from the same CLI version. - Private bundle: exactly the device firmware data plus `mixin.json`; the install script writes it to the installed root's `/usr/lib/firmware`. +## USB identity + +Installed-system and standard RAM gadgets use `BOARD: ID:` as +their USB serial number. The ID is the first 12 hex characters of SHA-256 over +`orchard-` followed by the device-tree `serial-number` without NUL +padding. It identifies the unit across boots without publishing the hardware +serial. No descriptor changes per boot, so macOS can retain accessory approval. +Trees without a serial number warn and retain the legacy `BOOT:` identity to +keep rescue USB available; those still need approval each boot. + +For RAM sessions, ACM returns `/proc/sys/kernel/random/boot_id` and the SSH +host public key together on every connection. The host validates the boot ID +and pins that boot's key. Installed systems use their persistent SSH keys and +do not allocate per-boot host state. Recovery and install keep their existing +`ECID:`/`BOOT:` serials and bootstrap/status checks. The install-to-system +handoff also retains `BOOT:`: the CLI's install add-on can run on older initrds +without the stable-identity helper. Normal installed boots use the new format. + ## Operations `orchard-device OPERATION [BASE64_JSON]`; requests may include `protocol: 1`. diff --git a/docs/firmware-recipes.md b/docs/firmware-recipes.md index 6eb10f97..2d0ca8db 100644 --- a/docs/firmware-recipes.md +++ b/docs/firmware-recipes.md @@ -238,3 +238,16 @@ The decompressed N61 kernel's SHA-256 is These fields are explicit recipe parameters, including zero-valued overrides. Physical qualification of firmware loading and input events remains separate from converting this shared firmware file. + + +## S8003 G5P META firmware + +`agx-g5p-ldr-from-kernelcache@1` reads the unique `__TEXT,__const` section +of `com.apple.AGXFirmwareKextG5P` in the user's own legacy prelinked +kernelcache. It rejects any size other than 119112 bytes or any digest other +than `33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca`. +The existing J71t kernelcache input names the active device kernelcache and +the pinned restore-image fallback. The normal recipe installer writes the +verified output to `/lib/firmware/apple/agx-g5p-c0.ldr`. No payload patch, +calibration, firmware bytes or captured job is stored in the source tree. +Other C0 revisions and other boards require separate digest qualification. diff --git a/features.toml b/features.toml index adcf0586..307c6cce 100644 --- a/features.toml +++ b/features.toml @@ -28,6 +28,9 @@ suspend = "Suspend" resume = "Resume" usb_serial = "USB serial" usb_network = "USB network" +usb_host = "USB host" audio_output = "Speakers" audio_input = "Microphone" camera = "Cameras" +video_decode_hevc = "Hardware H.265 video decoding" +warm_restart = "Warm restart" diff --git a/kernel/configs/apple-16k/README.md b/kernel/configs/apple-16k/README.md index 8cdcc06d..5411a9d1 100644 --- a/kernel/configs/apple-16k/README.md +++ b/kernel/configs/apple-16k/README.md @@ -72,3 +72,25 @@ To change configuration, use Kconfig tooling such as `menuconfig` or `scripts/config`, resolve it with `olddefconfig`, and review the resulting diff before updating the committed file. Preserve the pinned source/toolchain when checking reproducibility. + +## J71t system sleep preparation (0.4) + +Resolve with Orchard Clang 19. Enable KEYBOARD_GPIO_POLLED=y so the +existing polled cover switches can probe; keep GPIO_APPLE_D2365=y and +SUSPEND=y for PMU event wake and s2idle. Those latter options were already +enabled. Disable DWARF debug information (DEBUG_INFO_NONE=y) to keep the +local build footprint bounded. Kconfig also refreshes current derived options. +These config choices do not qualify cover wake or suspend on hardware. + +## USB host device classes (0.4) + +With USB host through the Lightning camera adapter, common USB devices need +their class drivers. Enable as modules: USB Ethernet (USB_NET_DRIVERS=y, +USB_USBNET, USB_RTL8152, USB_NET_AX88179_178A, USB_NET_CDCETHER, +USB_NET_CDC_NCM, USB_NET_CDC_EEM, USB_NET_RNDIS_HOST, USB_NET_AX8817X for +ASIX AX88772 adapters) and USB audio (SND_USB_AUDIO). Kconfig selects MII, +PHYLIB, PHYLINK, AX88796B_PHY (the AX88772's internal PHY), the MDIO helpers, +NET_SELFTESTS, SND_HWDEP and SND_RAWMIDI, and enables USB_RTL8153_ECM. The +default-on minidrivers nobody asked for (USB_NET_NET1080, +USB_NET_CDC_SUBSET, USB_NET_ZAURUS) stay off. RTL8152 needs no firmware; RTL8153/8156 load optional `rtl_nic` +firmware and work without it. Debug information stays off. diff --git a/kernel/configs/apple-16k/config b/kernel/configs/apple-16k/config index d3236c57..ffaad64c 100644 --- a/kernel/configs/apple-16k/config +++ b/kernel/configs/apple-16k/config @@ -837,6 +837,7 @@ CONFIG_MODPROBE_PATH="/sbin/modprobe" CONFIG_MODULES_TREE_LOOKUP=y CONFIG_BLOCK=y CONFIG_BLOCK_LEGACY_AUTOLOAD=y +CONFIG_BLK_DEV_BSG_COMMON=m # CONFIG_BLK_DEV_BSGLIB is not set # CONFIG_BLK_DEV_INTEGRITY is not set CONFIG_BLK_DEV_WRITE_MOUNTED=y @@ -1446,6 +1447,7 @@ CONFIG_RFKILL=m # CONFIG_LWTUNNEL is not set CONFIG_DST_CACHE=y CONFIG_GRO_CELLS=y +CONFIG_NET_SELFTESTS=m CONFIG_NET_SOCK_MSG=y CONFIG_PAGE_POOL=y # CONFIG_PAGE_POOL_STATS is not set @@ -1886,9 +1888,87 @@ CONFIG_NVME_APPLE=m # # SCSI device support # -CONFIG_SCSI_MOD=y +CONFIG_SCSI_MOD=m # CONFIG_RAID_ATTRS is not set -# CONFIG_SCSI is not set +CONFIG_SCSI_COMMON=m +CONFIG_SCSI=m +CONFIG_SCSI_DMA=y +CONFIG_SCSI_PROC_FS=y + +# +# SCSI support type (disk, tape, CD-ROM) +# +CONFIG_BLK_DEV_SD=m +# CONFIG_CHR_DEV_ST is not set +# CONFIG_BLK_DEV_SR is not set +# CONFIG_CHR_DEV_SG is not set +CONFIG_BLK_DEV_BSG=y +# CONFIG_CHR_DEV_SCH is not set +# CONFIG_SCSI_CONSTANTS is not set +# CONFIG_SCSI_LOGGING is not set +# CONFIG_SCSI_SCAN_ASYNC is not set + +# +# SCSI Transports +# +# CONFIG_SCSI_SPI_ATTRS is not set +# CONFIG_SCSI_FC_ATTRS is not set +# CONFIG_SCSI_ISCSI_ATTRS is not set +# CONFIG_SCSI_SAS_ATTRS is not set +# CONFIG_SCSI_SAS_LIBSAS is not set +# CONFIG_SCSI_SRP_ATTRS is not set +# end of SCSI Transports + +CONFIG_SCSI_LOWLEVEL=y +# CONFIG_ISCSI_TCP is not set +# CONFIG_ISCSI_BOOT_SYSFS is not set +# CONFIG_SCSI_CXGB3_ISCSI is not set +# CONFIG_SCSI_BNX2_ISCSI is not set +# CONFIG_BE2ISCSI is not set +# CONFIG_BLK_DEV_3W_XXXX_RAID is not set +# CONFIG_SCSI_HPSA is not set +# CONFIG_SCSI_3W_9XXX is not set +# CONFIG_SCSI_3W_SAS is not set +# CONFIG_SCSI_ACARD is not set +# CONFIG_SCSI_AACRAID is not set +# CONFIG_SCSI_AIC7XXX is not set +# CONFIG_SCSI_AIC79XX is not set +# CONFIG_SCSI_AIC94XX is not set +# CONFIG_SCSI_MVSAS is not set +# CONFIG_SCSI_MVUMI is not set +# CONFIG_SCSI_ADVANSYS is not set +# CONFIG_SCSI_ARCMSR is not set +# CONFIG_SCSI_ESAS2R is not set +# CONFIG_MEGARAID_NEWGEN is not set +# CONFIG_MEGARAID_LEGACY is not set +# CONFIG_MEGARAID_SAS is not set +# CONFIG_SCSI_MPT3SAS is not set +# CONFIG_SCSI_MPT2SAS is not set +# CONFIG_SCSI_MPI3MR is not set +# CONFIG_SCSI_LEAPRAID is not set +# CONFIG_SCSI_SMARTPQI is not set +# CONFIG_SCSI_HPTIOP is not set +# CONFIG_SCSI_BUSLOGIC is not set +# CONFIG_SCSI_MYRB is not set +# CONFIG_SCSI_MYRS is not set +# CONFIG_SCSI_SNIC is not set +# CONFIG_SCSI_DMX3191D is not set +# CONFIG_SCSI_FDOMAIN_PCI is not set +# CONFIG_SCSI_IPS is not set +# CONFIG_SCSI_INITIO is not set +# CONFIG_SCSI_INIA100 is not set +# CONFIG_SCSI_STEX is not set +# CONFIG_SCSI_SYM53C8XX_2 is not set +# CONFIG_SCSI_IPR is not set +# CONFIG_SCSI_QLOGIC_1280 is not set +# CONFIG_SCSI_QLA_ISCSI is not set +# CONFIG_SCSI_DC395x is not set +# CONFIG_SCSI_AM53C974 is not set +# CONFIG_SCSI_WD719X is not set +# CONFIG_SCSI_DEBUG is not set +# CONFIG_SCSI_PMCRAID is not set +# CONFIG_SCSI_PM8001 is not set +# CONFIG_SCSI_DH is not set # end of SCSI device support # CONFIG_ATA is not set @@ -1934,12 +2014,14 @@ CONFIG_DM_CRYPT=y # end of IEEE 1394 (FireWire) support CONFIG_NETDEVICES=y +CONFIG_MII=m CONFIG_NET_CORE=y # CONFIG_BONDING is not set # CONFIG_DUMMY is not set # CONFIG_WIREGUARD is not set # CONFIG_OVPN is not set # CONFIG_EQUALIZER is not set +# CONFIG_NET_FC is not set # CONFIG_IFB is not set # CONFIG_NET_TEAM is not set # CONFIG_MACVLAN is not set @@ -1964,9 +2046,101 @@ CONFIG_TUN=y # CONFIG_ARCNET is not set # CONFIG_ETHERNET is not set # CONFIG_FDDI is not set -# CONFIG_PHYLIB is not set +CONFIG_PHYLINK=m +CONFIG_PHYLIB=m +CONFIG_SWPHY=y +CONFIG_FIXED_PHY=m +# CONFIG_SFP is not set + +# +# MII PHY device drivers +# +# CONFIG_AS21XXX_PHY is not set +# CONFIG_AIR_AN8801_PHY is not set +# CONFIG_AIR_EN8811H_PHY is not set +# CONFIG_AMD_PHY is not set +# CONFIG_ADIN_PHY is not set +# CONFIG_ADIN1100_PHY is not set +# CONFIG_ADIN1140_PHY is not set +# CONFIG_AQUANTIA_PHY is not set +CONFIG_AX88796B_PHY=m +# CONFIG_BROADCOM_PHY is not set +# CONFIG_BCM54140_PHY is not set +# CONFIG_BCM7XXX_PHY is not set +# CONFIG_BCM84881_PHY is not set +# CONFIG_BCM87XX_PHY is not set +# CONFIG_CICADA_PHY is not set +# CONFIG_CORTINA_PHY is not set +# CONFIG_DAP8211R_PHY is not set +# CONFIG_DAVICOM_PHY is not set +# CONFIG_ICPLUS_PHY is not set +# CONFIG_LXT_PHY is not set +# CONFIG_INTEL_XWAY_PHY is not set +# CONFIG_LSI_ET1011C_PHY is not set +# CONFIG_MARVELL_PHY is not set +# CONFIG_MARVELL_10G_PHY is not set +# CONFIG_MARVELL_88Q2XXX_PHY is not set +# CONFIG_MARVELL_88X2222_PHY is not set +# CONFIG_MAXLINEAR_GPHY is not set +# CONFIG_MAXLINEAR_86110_PHY is not set +# CONFIG_MEDIATEK_GE_PHY is not set +# CONFIG_MICREL_PHY is not set +# CONFIG_MICROCHIP_T1S_PHY is not set +# CONFIG_MICROCHIP_PHY is not set +# CONFIG_MICROCHIP_T1_PHY is not set +# CONFIG_MICROSEMI_PHY is not set +# CONFIG_MOTORCOMM_PHY is not set +# CONFIG_NATIONAL_PHY is not set +# CONFIG_NXP_CBTX_PHY is not set +# CONFIG_NXP_C45_TJA11XX_PHY is not set +# CONFIG_NXP_TJA11XX_PHY is not set +# CONFIG_NCN26000_PHY is not set +# CONFIG_AT803X_PHY is not set +# CONFIG_QCA83XX_PHY is not set +# CONFIG_QCA808X_PHY is not set +# CONFIG_QCA807X_PHY is not set +# CONFIG_QSEMI_PHY is not set +# CONFIG_REALTEK_PHY is not set +# CONFIG_RENESAS_PHY is not set +# CONFIG_ROCKCHIP_PHY is not set +# CONFIG_SMSC_PHY is not set +# CONFIG_STE10XP is not set +# CONFIG_TERANETICS_PHY is not set +# CONFIG_DP83822_PHY is not set +# CONFIG_DP83TC811_PHY is not set +# CONFIG_DP83848_PHY is not set +# CONFIG_DP83867_PHY is not set +# CONFIG_DP83869_PHY is not set +# CONFIG_DP83TD510_PHY is not set +# CONFIG_DP83TG720_PHY is not set +# CONFIG_VITESSE_PHY is not set +# CONFIG_XILINX_GMII2RGMII is not set # CONFIG_PSE_CONTROLLER is not set +# +# MDIO controller drivers +# +CONFIG_FWNODE_MDIO=m +CONFIG_OF_MDIO=m +# CONFIG_MDIO_BITBANG is not set +# CONFIG_MDIO_BCM_UNIMAC is not set +# CONFIG_MDIO_HISI_FEMAC is not set +# CONFIG_MDIO_MVUSB is not set +# CONFIG_MDIO_MSCC_MIIM is not set +# CONFIG_MDIO_OCTEON is not set +# CONFIG_MDIO_IPQ4019 is not set +# CONFIG_MDIO_IPQ8064 is not set +# CONFIG_MDIO_THUNDER is not set + +# +# MDIO Multiplexers +# +# CONFIG_MDIO_BUS_MUX_GPIO is not set +# CONFIG_MDIO_BUS_MUX_MULTIPLEXER is not set +# CONFIG_MDIO_BUS_MUX_MMIOREG is not set +# end of MDIO Multiplexers +# end of MDIO controller drivers + # # PCS device drivers # @@ -1975,7 +2149,44 @@ CONFIG_TUN=y # CONFIG_PPP is not set # CONFIG_SLIP is not set -# CONFIG_USB_NET_DRIVERS is not set +CONFIG_USB_NET_DRIVERS=y +# CONFIG_USB_CATC is not set +# CONFIG_USB_KAWETH is not set +# CONFIG_USB_PEGASUS is not set +# CONFIG_USB_RTL8150 is not set +CONFIG_USB_RTL8152=m +# CONFIG_USB_LAN78XX is not set +CONFIG_USB_USBNET=m +CONFIG_USB_NET_AX8817X=m +CONFIG_USB_NET_AX88179_178A=m +CONFIG_USB_NET_CDCETHER=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_NCM=m +# CONFIG_USB_NET_HUAWEI_CDC_NCM is not set +# CONFIG_USB_NET_CDC_MBIM is not set +# CONFIG_USB_NET_DM9601 is not set +# CONFIG_USB_NET_SR9700 is not set +# CONFIG_USB_NET_SR9800 is not set +# CONFIG_USB_NET_SMSC75XX is not set +# CONFIG_USB_NET_SMSC95XX is not set +# CONFIG_USB_NET_GL620A is not set +# CONFIG_USB_NET_NET1080 is not set +# CONFIG_USB_NET_PLUSB is not set +# CONFIG_USB_NET_MCS7830 is not set +CONFIG_USB_NET_RNDIS_HOST=m +# CONFIG_USB_NET_CDC_SUBSET is not set +# CONFIG_USB_NET_ZAURUS is not set +# CONFIG_USB_NET_CX82310_ETH is not set +# CONFIG_USB_NET_KALMIA is not set +# CONFIG_USB_NET_QMI_WWAN is not set +# CONFIG_USB_HSO is not set +# CONFIG_USB_NET_INT51X1 is not set +# CONFIG_USB_IPHETH is not set +# CONFIG_USB_SIERRA_NET is not set +# CONFIG_USB_VL600 is not set +# CONFIG_USB_NET_CH9200 is not set +# CONFIG_USB_NET_AQC111 is not set +CONFIG_USB_RTL8153_ECM=m CONFIG_WLAN=y CONFIG_WLAN_VENDOR_ADMTEK=y # CONFIG_ADM8211 is not set @@ -2118,7 +2329,7 @@ CONFIG_INPUT_KEYBOARD=y # CONFIG_KEYBOARD_DLINK_DIR685 is not set # CONFIG_KEYBOARD_LKKBD is not set CONFIG_KEYBOARD_GPIO=y -# CONFIG_KEYBOARD_GPIO_POLLED is not set +CONFIG_KEYBOARD_GPIO_POLLED=y # CONFIG_KEYBOARD_TCA8418 is not set # CONFIG_KEYBOARD_MATRIX is not set # CONFIG_KEYBOARD_CHARLIEPLEX is not set @@ -2696,6 +2907,7 @@ CONFIG_BATTERY_BQ27XXX_HDQ=y CONFIG_APPLE_BELLATRIX_POWER=y CONFIG_APPLE_D2365_CHARGER=y CONFIG_APPLE_TRISTAR_CHARGER=y +CONFIG_APPLE_TRISTAR_ACCESSORY=y # CONFIG_BATTERY_MAX17040 is not set # CONFIG_BATTERY_MAX17042 is not set # CONFIG_BATTERY_MAX1720X is not set @@ -3940,6 +4152,7 @@ CONFIG_DRM_SIMPLEDRM=y # CONFIG_DRM_VKMS is not set # CONFIG_DRM_VMWGFX is not set CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DRM_APPLE_G5P=y # # Frame buffer Devices @@ -4051,6 +4264,8 @@ CONFIG_SND=m CONFIG_SND_TIMER=m CONFIG_SND_PCM=m CONFIG_SND_DMAENGINE_PCM=m +CONFIG_SND_HWDEP=m +CONFIG_SND_RAWMIDI=m CONFIG_SND_JACK=y CONFIG_SND_JACK_INPUT_DEV=y # CONFIG_SND_OSSEMUL is not set @@ -4146,7 +4361,9 @@ CONFIG_SND_PCI=y CONFIG_SND_SPI=y CONFIG_SND_USB=y -# CONFIG_SND_USB_AUDIO is not set +CONFIG_SND_USB_AUDIO=m +# CONFIG_SND_USB_AUDIO_MIDI_V2 is not set +CONFIG_SND_USB_AUDIO_USE_MEDIA_CONTROLLER=y # CONFIG_SND_USB_UA101 is not set # CONFIG_SND_USB_CAIAQ is not set # CONFIG_SND_USB_6FIRE is not set @@ -4158,6 +4375,7 @@ CONFIG_SND_USB=y # CONFIG_SND_USB_VARIAX is not set CONFIG_SND_SOC=m CONFIG_SND_SOC_GENERIC_DMAENGINE_PCM=y +# CONFIG_SND_SOC_USB is not set # # Analog Devices @@ -4729,11 +4947,28 @@ CONFIG_USB_OHCI_HCD_PLATFORM=y # # NOTE: USB_STORAGE depends on SCSI but BLK_DEV_SD may also be needed; see USB_STORAGE Help for more info # +CONFIG_USB_STORAGE=m +# CONFIG_USB_STORAGE_DEBUG is not set +# CONFIG_USB_STORAGE_REALTEK is not set +# CONFIG_USB_STORAGE_DATAFAB is not set +# CONFIG_USB_STORAGE_FREECOM is not set +# CONFIG_USB_STORAGE_ISD200 is not set +# CONFIG_USB_STORAGE_USBAT is not set +# CONFIG_USB_STORAGE_SDDR09 is not set +# CONFIG_USB_STORAGE_SDDR55 is not set +# CONFIG_USB_STORAGE_JUMPSHOT is not set +# CONFIG_USB_STORAGE_ALAUDA is not set +# CONFIG_USB_STORAGE_ONETOUCH is not set +# CONFIG_USB_STORAGE_KARMA is not set +# CONFIG_USB_STORAGE_CYPRESS_ATACB is not set +# CONFIG_USB_STORAGE_ENE_UB6250 is not set +CONFIG_USB_UAS=m # # USB Imaging devices # # CONFIG_USB_MDC800 is not set +# CONFIG_USB_MICROTEK is not set # CONFIG_USBIP_CORE is not set # @@ -4888,7 +5123,9 @@ CONFIG_USB_G_MULTI_CDC=y # CONFIG_TYPEC is not set CONFIG_USB_ROLE_SWITCH=y +CONFIG_USB_ROLES_APPLE_S8000=y # CONFIG_MMC is not set +# CONFIG_SCSI_UFSHCD is not set # CONFIG_MEMSTICK is not set # CONFIG_NEW_LEDS is not set # CONFIG_ACCESSIBILITY is not set @@ -5624,6 +5861,7 @@ CONFIG_APPLE_AOP_ACCEL=m # CONFIG_ADIS16260 is not set # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set +CONFIG_APPLE_AOP_GYRO=m # CONFIG_BMG160 is not set # CONFIG_FXAS21002C is not set # CONFIG_MPU3050_I2C is not set @@ -5859,6 +6097,7 @@ CONFIG_TSL2583=m # CONFIG_HP206C is not set # CONFIG_ZPA2326 is not set # CONFIG_ADP810 is not set +CONFIG_APPLE_AOP_PRESSURE=m # end of Pressure sensors # @@ -5997,6 +6236,7 @@ CONFIG_APPLE_M1_CPU_PMU=y # CONFIG_HNS3_PMU is not set # CONFIG_DWC_PCIE_PMU is not set # CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU is not set +CONFIG_APPLE_AMC_PMU=y # end of Performance monitor support # CONFIG_RAS is not set @@ -6155,7 +6395,8 @@ CONFIG_VFAT_FS=y CONFIG_FAT_DEFAULT_CODEPAGE=437 CONFIG_FAT_DEFAULT_IOCHARSET="iso8859-1" # CONFIG_FAT_DEFAULT_UTF8 is not set -# CONFIG_EXFAT_FS is not set +CONFIG_EXFAT_FS=m +CONFIG_EXFAT_DEFAULT_IOCHARSET="utf8" # CONFIG_NTFS_FS is not set # CONFIG_NTFS3_FS is not set # end of DOS/FAT/EXFAT/NT Filesystems @@ -6308,7 +6549,7 @@ CONFIG_NLS_ISO8859_1=y # CONFIG_NLS_MAC_INUIT is not set # CONFIG_NLS_MAC_ROMANIAN is not set # CONFIG_NLS_MAC_TURKISH is not set -# CONFIG_NLS_UTF8 is not set +CONFIG_NLS_UTF8=m # CONFIG_DLM is not set # CONFIG_UNICODE is not set CONFIG_IO_WQ=y @@ -6730,6 +6971,7 @@ CONFIG_FONT_SUPPORT=y # CONFIG_FONTS is not set CONFIG_FONT_8x8=y CONFIG_FONT_8x16=y +CONFIG_SG_POOL=y CONFIG_ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION=y CONFIG_GENERIC_CPU_CACHE_MAINTENANCE=y CONFIG_ARCH_STACKWALK=y diff --git a/kernel/configs/apple-4k/README.md b/kernel/configs/apple-4k/README.md index 38208c81..ad8662b0 100644 --- a/kernel/configs/apple-4k/README.md +++ b/kernel/configs/apple-4k/README.md @@ -18,3 +18,25 @@ From `../apple-16k/config`, with `CONFIG_ARM64_4K_PAGES=y` and `CONFIG_ARM64_16K_PAGES` unset, resolved with `olddefconfig` on the pinned source as described in `../apple-16k/README.md`. Keep the two files in step: a change to `apple-16k` is made here too, then resolved again. + +## J71t system sleep preparation (0.4) + +Resolve with Orchard Clang 19. Enable KEYBOARD_GPIO_POLLED=y so the +existing polled cover switches can probe; keep GPIO_APPLE_D2365=y and +SUSPEND=y for PMU event wake and s2idle. Those latter options were already +enabled. Disable DWARF debug information (DEBUG_INFO_NONE=y) to keep the +local build footprint bounded. Kconfig also refreshes current derived options. +These config choices do not qualify cover wake or suspend on hardware. + +## USB host device classes (0.4) + +With USB host through the Lightning camera adapter, common USB devices need +their class drivers. Enable as modules: USB Ethernet (USB_NET_DRIVERS=y, +USB_USBNET, USB_RTL8152, USB_NET_AX88179_178A, USB_NET_CDCETHER, +USB_NET_CDC_NCM, USB_NET_CDC_EEM, USB_NET_RNDIS_HOST, USB_NET_AX8817X for +ASIX AX88772 adapters) and USB audio (SND_USB_AUDIO). Kconfig selects MII, +PHYLIB, PHYLINK, AX88796B_PHY (the AX88772's internal PHY), the MDIO helpers, +NET_SELFTESTS, SND_HWDEP and SND_RAWMIDI, and enables USB_RTL8153_ECM. The +default-on minidrivers nobody asked for (USB_NET_NET1080, +USB_NET_CDC_SUBSET, USB_NET_ZAURUS) stay off. RTL8152 needs no firmware; RTL8153/8156 load optional `rtl_nic` +firmware and work without it. Debug information stays off. diff --git a/kernel/configs/apple-4k/config b/kernel/configs/apple-4k/config index 106c382a..44e95d99 100644 --- a/kernel/configs/apple-4k/config +++ b/kernel/configs/apple-4k/config @@ -1447,6 +1447,7 @@ CONFIG_RFKILL=m # CONFIG_LWTUNNEL is not set CONFIG_DST_CACHE=y CONFIG_GRO_CELLS=y +CONFIG_NET_SELFTESTS=m CONFIG_NET_SOCK_MSG=y CONFIG_PAGE_POOL=y # CONFIG_PAGE_POOL_STATS is not set @@ -1935,6 +1936,7 @@ CONFIG_DM_CRYPT=y # end of IEEE 1394 (FireWire) support CONFIG_NETDEVICES=y +CONFIG_MII=m CONFIG_NET_CORE=y # CONFIG_BONDING is not set # CONFIG_DUMMY is not set @@ -1965,9 +1967,101 @@ CONFIG_TUN=y # CONFIG_ARCNET is not set # CONFIG_ETHERNET is not set # CONFIG_FDDI is not set -# CONFIG_PHYLIB is not set +CONFIG_PHYLINK=m +CONFIG_PHYLIB=m +CONFIG_SWPHY=y +CONFIG_FIXED_PHY=m +# CONFIG_SFP is not set + +# +# MII PHY device drivers +# +# CONFIG_AS21XXX_PHY is not set +# CONFIG_AIR_AN8801_PHY is not set +# CONFIG_AIR_EN8811H_PHY is not set +# CONFIG_AMD_PHY is not set +# CONFIG_ADIN_PHY is not set +# CONFIG_ADIN1100_PHY is not set +# CONFIG_ADIN1140_PHY is not set +# CONFIG_AQUANTIA_PHY is not set +CONFIG_AX88796B_PHY=m +# CONFIG_BROADCOM_PHY is not set +# CONFIG_BCM54140_PHY is not set +# CONFIG_BCM7XXX_PHY is not set +# CONFIG_BCM84881_PHY is not set +# CONFIG_BCM87XX_PHY is not set +# CONFIG_CICADA_PHY is not set +# CONFIG_CORTINA_PHY is not set +# CONFIG_DAP8211R_PHY is not set +# CONFIG_DAVICOM_PHY is not set +# CONFIG_ICPLUS_PHY is not set +# CONFIG_LXT_PHY is not set +# CONFIG_INTEL_XWAY_PHY is not set +# CONFIG_LSI_ET1011C_PHY is not set +# CONFIG_MARVELL_PHY is not set +# CONFIG_MARVELL_10G_PHY is not set +# CONFIG_MARVELL_88Q2XXX_PHY is not set +# CONFIG_MARVELL_88X2222_PHY is not set +# CONFIG_MAXLINEAR_GPHY is not set +# CONFIG_MAXLINEAR_86110_PHY is not set +# CONFIG_MEDIATEK_GE_PHY is not set +# CONFIG_MICREL_PHY is not set +# CONFIG_MICROCHIP_T1S_PHY is not set +# CONFIG_MICROCHIP_PHY is not set +# CONFIG_MICROCHIP_T1_PHY is not set +# CONFIG_MICROSEMI_PHY is not set +# CONFIG_MOTORCOMM_PHY is not set +# CONFIG_NATIONAL_PHY is not set +# CONFIG_NXP_CBTX_PHY is not set +# CONFIG_NXP_C45_TJA11XX_PHY is not set +# CONFIG_NXP_TJA11XX_PHY is not set +# CONFIG_NCN26000_PHY is not set +# CONFIG_AT803X_PHY is not set +# CONFIG_QCA83XX_PHY is not set +# CONFIG_QCA808X_PHY is not set +# CONFIG_QCA807X_PHY is not set +# CONFIG_QSEMI_PHY is not set +# CONFIG_REALTEK_PHY is not set +# CONFIG_RENESAS_PHY is not set +# CONFIG_ROCKCHIP_PHY is not set +# CONFIG_SMSC_PHY is not set +# CONFIG_STE10XP is not set +# CONFIG_TERANETICS_PHY is not set +# CONFIG_DP83822_PHY is not set +# CONFIG_DP83TC811_PHY is not set +# CONFIG_DP83848_PHY is not set +# CONFIG_DP83867_PHY is not set +# CONFIG_DP83869_PHY is not set +# CONFIG_DP83TD510_PHY is not set +# CONFIG_DP83TG720_PHY is not set +# CONFIG_VITESSE_PHY is not set +# CONFIG_XILINX_GMII2RGMII is not set # CONFIG_PSE_CONTROLLER is not set +# +# MDIO controller drivers +# +CONFIG_FWNODE_MDIO=m +CONFIG_OF_MDIO=m +# CONFIG_MDIO_BITBANG is not set +# CONFIG_MDIO_BCM_UNIMAC is not set +# CONFIG_MDIO_HISI_FEMAC is not set +# CONFIG_MDIO_MVUSB is not set +# CONFIG_MDIO_MSCC_MIIM is not set +# CONFIG_MDIO_OCTEON is not set +# CONFIG_MDIO_IPQ4019 is not set +# CONFIG_MDIO_IPQ8064 is not set +# CONFIG_MDIO_THUNDER is not set + +# +# MDIO Multiplexers +# +# CONFIG_MDIO_BUS_MUX_GPIO is not set +# CONFIG_MDIO_BUS_MUX_MULTIPLEXER is not set +# CONFIG_MDIO_BUS_MUX_MMIOREG is not set +# end of MDIO Multiplexers +# end of MDIO controller drivers + # # PCS device drivers # @@ -1976,7 +2070,44 @@ CONFIG_TUN=y # CONFIG_PPP is not set # CONFIG_SLIP is not set -# CONFIG_USB_NET_DRIVERS is not set +CONFIG_USB_NET_DRIVERS=y +# CONFIG_USB_CATC is not set +# CONFIG_USB_KAWETH is not set +# CONFIG_USB_PEGASUS is not set +# CONFIG_USB_RTL8150 is not set +CONFIG_USB_RTL8152=m +# CONFIG_USB_LAN78XX is not set +CONFIG_USB_USBNET=m +CONFIG_USB_NET_AX8817X=m +CONFIG_USB_NET_AX88179_178A=m +CONFIG_USB_NET_CDCETHER=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_NCM=m +# CONFIG_USB_NET_HUAWEI_CDC_NCM is not set +# CONFIG_USB_NET_CDC_MBIM is not set +# CONFIG_USB_NET_DM9601 is not set +# CONFIG_USB_NET_SR9700 is not set +# CONFIG_USB_NET_SR9800 is not set +# CONFIG_USB_NET_SMSC75XX is not set +# CONFIG_USB_NET_SMSC95XX is not set +# CONFIG_USB_NET_GL620A is not set +# CONFIG_USB_NET_NET1080 is not set +# CONFIG_USB_NET_PLUSB is not set +# CONFIG_USB_NET_MCS7830 is not set +CONFIG_USB_NET_RNDIS_HOST=m +# CONFIG_USB_NET_CDC_SUBSET is not set +# CONFIG_USB_NET_ZAURUS is not set +# CONFIG_USB_NET_CX82310_ETH is not set +# CONFIG_USB_NET_KALMIA is not set +# CONFIG_USB_NET_QMI_WWAN is not set +# CONFIG_USB_HSO is not set +# CONFIG_USB_NET_INT51X1 is not set +# CONFIG_USB_IPHETH is not set +# CONFIG_USB_SIERRA_NET is not set +# CONFIG_USB_VL600 is not set +# CONFIG_USB_NET_CH9200 is not set +# CONFIG_USB_NET_AQC111 is not set +CONFIG_USB_RTL8153_ECM=m CONFIG_WLAN=y CONFIG_WLAN_VENDOR_ADMTEK=y # CONFIG_ADM8211 is not set @@ -2119,7 +2250,7 @@ CONFIG_INPUT_KEYBOARD=y # CONFIG_KEYBOARD_DLINK_DIR685 is not set # CONFIG_KEYBOARD_LKKBD is not set CONFIG_KEYBOARD_GPIO=y -# CONFIG_KEYBOARD_GPIO_POLLED is not set +CONFIG_KEYBOARD_GPIO_POLLED=y # CONFIG_KEYBOARD_TCA8418 is not set # CONFIG_KEYBOARD_MATRIX is not set # CONFIG_KEYBOARD_CHARLIEPLEX is not set @@ -4053,6 +4184,8 @@ CONFIG_SND=m CONFIG_SND_TIMER=m CONFIG_SND_PCM=m CONFIG_SND_DMAENGINE_PCM=m +CONFIG_SND_HWDEP=m +CONFIG_SND_RAWMIDI=m CONFIG_SND_JACK=y CONFIG_SND_JACK_INPUT_DEV=y # CONFIG_SND_OSSEMUL is not set @@ -4148,7 +4281,9 @@ CONFIG_SND_PCI=y CONFIG_SND_SPI=y CONFIG_SND_USB=y -# CONFIG_SND_USB_AUDIO is not set +CONFIG_SND_USB_AUDIO=m +# CONFIG_SND_USB_AUDIO_MIDI_V2 is not set +CONFIG_SND_USB_AUDIO_USE_MEDIA_CONTROLLER=y # CONFIG_SND_USB_UA101 is not set # CONFIG_SND_USB_CAIAQ is not set # CONFIG_SND_USB_6FIRE is not set @@ -4160,6 +4295,7 @@ CONFIG_SND_USB=y # CONFIG_SND_USB_VARIAX is not set CONFIG_SND_SOC=m CONFIG_SND_SOC_GENERIC_DMAENGINE_PCM=y +# CONFIG_SND_SOC_USB is not set # # Analog Devices @@ -4890,6 +5026,7 @@ CONFIG_USB_G_MULTI_CDC=y # CONFIG_TYPEC is not set CONFIG_USB_ROLE_SWITCH=y +# CONFIG_USB_ROLES_APPLE_S8000 is not set # CONFIG_MMC is not set # CONFIG_MEMSTICK is not set # CONFIG_NEW_LEDS is not set @@ -5192,7 +5329,7 @@ CONFIG_APPLE_DART=y CONFIG_APPLE_MAILBOX=y # CONFIG_APPLE_PMGR_MISC is not set CONFIG_APPLE_RTKIT=y -# CONFIG_APPLE_AOP_T8010 is not set +CONFIG_APPLE_AOP_T8010=m CONFIG_APPLE_PMP_T8015_V1=y CONFIG_APPLE_SIO=m CONFIG_APPLE_T8010_WLAN=y @@ -5312,6 +5449,7 @@ CONFIG_IIO_CONSUMERS_PER_TRIGGER=2 # CONFIG_ADXL372_I2C is not set # CONFIG_ADXL380_SPI is not set # CONFIG_ADXL380_I2C is not set +CONFIG_APPLE_AOP_ACCEL=m # CONFIG_BMA180 is not set # CONFIG_BMA220 is not set # CONFIG_BMA400 is not set @@ -5625,6 +5763,7 @@ CONFIG_IIO_CONSUMERS_PER_TRIGGER=2 # CONFIG_ADIS16260 is not set # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set +CONFIG_APPLE_AOP_GYRO=m # CONFIG_BMG160 is not set # CONFIG_FXAS21002C is not set # CONFIG_MPU3050_I2C is not set @@ -5860,6 +5999,7 @@ CONFIG_TSL2583=m # CONFIG_HP206C is not set # CONFIG_ZPA2326 is not set # CONFIG_ADP810 is not set +CONFIG_APPLE_AOP_PRESSURE=m # end of Pressure sensors # @@ -5998,6 +6138,7 @@ CONFIG_APPLE_M1_CPU_PMU=y # CONFIG_HNS3_PMU is not set # CONFIG_DWC_PCIE_PMU is not set # CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU is not set +# CONFIG_APPLE_AMC_PMU is not set # end of Performance monitor support # CONFIG_RAS is not set @@ -6156,7 +6297,8 @@ CONFIG_VFAT_FS=y CONFIG_FAT_DEFAULT_CODEPAGE=437 CONFIG_FAT_DEFAULT_IOCHARSET="iso8859-1" # CONFIG_FAT_DEFAULT_UTF8 is not set -# CONFIG_EXFAT_FS is not set +CONFIG_EXFAT_FS=m +CONFIG_EXFAT_DEFAULT_IOCHARSET="utf8" # CONFIG_NTFS_FS is not set # CONFIG_NTFS3_FS is not set # end of DOS/FAT/EXFAT/NT Filesystems @@ -6309,7 +6451,7 @@ CONFIG_NLS_ISO8859_1=y # CONFIG_NLS_MAC_INUIT is not set # CONFIG_NLS_MAC_ROMANIAN is not set # CONFIG_NLS_MAC_TURKISH is not set -# CONFIG_NLS_UTF8 is not set +CONFIG_NLS_UTF8=m # CONFIG_DLM is not set # CONFIG_UNICODE is not set CONFIG_IO_WQ=y diff --git a/kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch b/kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch new file mode 100644 index 00000000..02ad3882 --- /dev/null +++ b/kernel/patches/devices/s8000/0007-arm64-dts-apple-select-certified-AOP-reload-through-.patch @@ -0,0 +1,126 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + .../arm64/boot/dts/apple/s800-0-3-common.dtsi | 1 + + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 37 ++++++++++++++++++ + arch/arm64/boot/dts/apple/s8003-j71t.dts | 39 +++++++++++++++++++ + 3 files changed, 77 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +index 354533f26..338182cf5 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3-common.dtsi +@@ -11,6 +11,7 @@ + + / { + aliases { ++ aop = &aop; /* Used by loader */ + ausbphy0 = &ausbphy0; + nvme-hmb = &nvme_hmb; /* Used by loader */ + serial0 = &serial0; +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 3277a73bb..75ec4a403 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -415,6 +415,43 @@ + interrupt-parent = <&aic>; + interrupts = ; + }; ++ ++ aop: aop@210800000 { ++ compatible = "apple,s8000-aop"; ++ reg = <0x2 0x10800000 0x0 0x4000>, ++ <0x2 0x10000600 0x0 0x4>, ++ <0x2 0x10e00000 0x0 0x9b000>; ++ reg-names = "akf", "control", "sram"; ++ mboxes = <&aop_mbox>; ++ mbox-names = "mbox"; ++ status = "disabled"; ++ ++ aop_accel: accel { ++ compatible = "apple,s8000-aop-accel"; ++ }; ++ ++ aop_gyro: gyro { ++ compatible = "apple,s8000-aop-gyro"; ++ }; ++ ++ aop_pressure: pressure { ++ compatible = "apple,s8000-aop-pressure"; ++ }; ++ }; ++ ++ aop_mbox: mbox@210804000 { ++ compatible = "apple,s8000-a7iop-mailbox", "apple,t8010-a7iop-mailbox"; ++ reg = <0x2 0x10804000 0x0 0x1000>; ++ interrupt-parent = <&aic>; ++ interrupts = , ++ , ++ , ++ ; ++ interrupt-names = "send-empty", "send-not-empty", ++ "recv-empty", "recv-not-empty"; ++ #mbox-cells = <0>; ++ status = "disabled"; ++ }; + }; + + timer { +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..2728cbd5f 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -16,6 +16,45 @@ + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; + }; + ++/ { ++ reserved-memory { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* Same tail page as release6, never a firmware DMA grant. */ ++ aop_shutdown: aop-shutdown@8583fc000 { ++ reg = <0x8 0x583fc000 0 0x4000>; ++ no-map; ++ }; ++ }; ++}; ++ ++&aop { ++ apple,certified-reload; ++ memory-region = <&aop_shutdown>; ++ apple,observe-domains = <&ps_aop_gpio &ps_aop &ps_aop_busif ++ &ps_aop_cpu &ps_aop_filter>; ++ apple,spi0-domain = <&ps_spi0>; ++ firmware-name = "apple/aop-ipad6f.bin", "apple/aop-ipad6f-ext.bin"; ++ status = "okay"; ++}; ++ ++/* ++ * The AOP reports the accelerometer in its sensor frame, which is the ++ * device frame (x right, y up, z out of the screen) negated: flat, screen up ++ * reads z = -1 g, upright y = -1 g. iio-sensor-proxy takes upright as ++ * negative y and needs z flipped. ++ */ ++&aop_accel { ++ mount-matrix = "1", "0", "0", ++ "0", "1", "0", ++ "0", "0", "-1"; ++}; ++ ++&aop_mbox { ++ status = "okay"; ++}; ++ + &touchscreen { + firmware-name = "apple/j71t-fwstart.z2fw"; + }; diff --git a/kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..dab3a029 --- /dev/null +++ b/kernel/patches/devices/s8000/0008-arm64-dts-apple-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,32 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..0bc19727d 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -11,6 +11,14 @@ + #include "s8003.dtsi" + #include "s800x-ipad5.dtsi" + ++&pcie0 { ++ apple,ans-kexec-handoff; ++}; ++ ++&pcie0_dart0 { ++ apple,ans-dart-handoff; ++}; ++ + / { + compatible = "apple,j71t", "apple,s8003", "apple,arm-platform"; + model = "Apple iPad 5 (Wi-Fi) (TSMC)"; diff --git a/kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..36c9879f --- /dev/null +++ b/kernel/patches/devices/s8000/0009-arm64-dts-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 3277a73bb..f6449a7e6 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -443,3 +443,7 @@ pmu { + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/ { ++ apple,reserve-pstore; ++}; diff --git a/kernel/patches/devices/s8000/0010-arm64-dts-apple-s800-0-3-Add-the-SoC-performance-sta.patch b/kernel/patches/devices/s8000/0010-arm64-dts-apple-s800-0-3-Add-the-SoC-performance-sta.patch new file mode 100644 index 00000000..021df011 --- /dev/null +++ b/kernel/patches/devices/s8000/0010-arm64-dts-apple-s800-0-3-Add-the-SoC-performance-sta.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 09:49:33 +0000 +Subject: [PATCH] arm64: dts: apple: s800-0-3: Add the SoC performance state + +The PMGR register that selects the SoC and memory performance states, +for apple-pmgr-misc. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 3277a73bb..779118fe3 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -326,6 +326,12 @@ touchclk: clock-controller@78000 { + }; + }; + ++ pmgr_misc: power-management@20e068000 { ++ compatible = "apple,s8000-pmgr-misc"; ++ reg = <0x2 0x0e068000 0x0 0x4>; ++ reg-names = "soc-ps"; ++ }; ++ + aic:interrupt-controller@20e100000 { + compatible = "apple,s8000-aic", "apple,aic"; + reg = <0x2 0x0e100000 0x0 0x100000>; diff --git a/kernel/patches/devices/s8000/0011-arm64-dts-apple-s800-0-3-Add-the-memory-port-monitor.patch b/kernel/patches/devices/s8000/0011-arm64-dts-apple-s800-0-3-Add-the-memory-port-monitor.patch new file mode 100644 index 00000000..3cd328d1 --- /dev/null +++ b/kernel/patches/devices/s8000/0011-arm64-dts-apple-s800-0-3-Add-the-memory-port-monitor.patch @@ -0,0 +1,29 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 10:32:54 +0000 +Subject: [PATCH] arm64: dts: apple: s800-0-3: Add the memory port monitor + +Describe the MCC port-monitor window shared by S8000 and S8003 so the +apple_amc PMU can count CPU and graphics memory traffic. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index 779118fe3..bc8d381e2 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -70,6 +70,11 @@ soc { + nonposted-mmio; + ranges; + ++ amc_pmu: pmu@200000500 { ++ compatible = "apple,s8000-amc-pmu"; ++ reg = <0x2 0x500 0x0 0xa0>; ++ }; ++ + cpufreq: performance-controller@202220000 { + compatible = "apple,s8000-cluster-cpufreq", "apple,t8103-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x02220000 0 0x1000>; diff --git a/kernel/patches/devices/s8000/0012-arm64-dts-apple-s800-0-3-Add-the-USB-role-switch-and.patch b/kernel/patches/devices/s8000/0012-arm64-dts-apple-s800-0-3-Add-the-USB-role-switch-and.patch new file mode 100644 index 00000000..d5129176 --- /dev/null +++ b/kernel/patches/devices/s8000/0012-arm64-dts-apple-s800-0-3-Add-the-USB-role-switch-and.patch @@ -0,0 +1,57 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 3 Nov 2026 14:46:21 +0100 +Subject: [PATCH] arm64: dts: apple: s800-0-3: Add the USB role switch and + EHCI/OHCI + +Add the A9 USB role switch to the USB complex, disabled, with the EHCI +and OHCI it creates in the host role as its children: they share the +AUSB PHY with usbdev, and the switch decides which side has it. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800-0-3.dtsi | 31 +++++++++++++++++++++++++ + 1 file changed, 31 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800-0-3.dtsi b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +index ff1c5c66e..135c9f8e6 100644 +--- a/arch/arm64/boot/dts/apple/s800-0-3.dtsi ++++ b/arch/arm64/boot/dts/apple/s800-0-3.dtsi +@@ -282,6 +282,37 @@ usbdev: usbdev@20c100000 { + resets = <&ps_usbotg>; + dr_mode = "peripheral"; + }; ++ ++ /* Moves the PHY between usbdev and the EHCI/OHCI pair. */ ++ usb_role: usb-role-switch@20e0d8000 { ++ compatible = "apple,s8000-usb-role-switch"; ++ reg = <0x2 0x0e0d8000 0x0 0x1000>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ /* The usb-complex dma-ranges apply to the host pair. */ ++ dma-ranges; ++ nonposted-mmio; ++ phys = <&ausbphy0 PHY_TYPE_USB2>; ++ phy-names = "usb2-phy"; ++ apple,usb-device = <&usbdev>; ++ status = "disabled"; ++ ++ /* Created by the switch in the host role only. */ ++ usb@20c200000 { ++ compatible = "apple,s8000-ehci", "generic-ehci"; ++ reg = <0x2 0x0c200000 0x0 0x10000>; ++ interrupts = ; ++ power-domains = <&ps_usb2host0>; ++ }; ++ ++ usb@20c300000 { ++ compatible = "apple,s8000-ohci", "generic-ohci"; ++ reg = <0x2 0x0c300000 0x0 0x10000>; ++ interrupts = ; ++ power-domains = <&ps_usb2host0_ohci>; ++ }; ++ }; + #if 0 + ehci0: usb-ehci@20c200000 { + compatible = "apple,s8000-ehci", "generic-ehci"; diff --git a/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch b/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch new file mode 100644 index 00000000..70a0484d --- /dev/null +++ b/kernel/patches/devices/s8003/0001-arm64-dts-apple-describe-J71t-tethered-boot.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:20:28 +0100 +Subject: [PATCH] arm64: dts: apple: describe J71t tethered boot + +Record the board fact for the droppable watchdog reset guard. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 733cefc66..0df58a56f 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -24,3 +24,7 @@ &touchscreen { + &wifi { + brcm,board-type = "apple,feara"; + }; ++ ++&wdt { ++ apple,tethered-boot; ++}; diff --git a/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch b/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch new file mode 100644 index 00000000..43ae9e86 --- /dev/null +++ b/kernel/patches/devices/s8003/0002-arm64-dts-apple-j71t-enable-paired-sleep-and-the-s2i.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] arm64: dts: apple: j71t: enable paired sleep and the s2idle + gadget + +Mark the J71t ANS port for paired system sleep and keep the USB gadget +connected over suspend to idle. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 3ee634e79..d03bc4ff6 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -75,3 +75,11 @@ + &wdt { + apple,tethered-boot; + }; ++ ++&pcie0 { ++ apple,paired-system-sleep; ++}; ++ ++&usbdev { ++ apple,retain-gadget-in-s2idle; ++}; diff --git a/kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch b/kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch new file mode 100644 index 00000000..4f9d26c1 --- /dev/null +++ b/kernel/patches/devices/s8003/0003-arm64-dts-apple-j71t-wake-from-sleep-on-the-Smart-Co.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:27:23 +0000 +Subject: [PATCH] arm64: dts: apple: j71t: wake from sleep on the Smart Cover + events + +PMIC GPIO lines 0 and 10 are the Smart Cover hall sensors. Their events, +0x30 and 0x3a, latch while the iPad sleeps; list them as wake-only so +that opening the cover wakes the iPad. The polled switches and their +reserved range are unchanged: lines 0 and 10 are already outside +gpio-reserved-ranges. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index acb62901e..2a62f0d95 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -40,3 +40,8 @@ &pcie0 { + &usbdev { + apple,retain-gadget-in-s2idle; + }; ++ ++/* Smart Cover hall sensors (PMIC GPIO 0 and 10): wake-only. */ ++&pmu_gpio { ++ apple,wake-events = <0x30 0x3a>; ++}; diff --git a/kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch b/kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch new file mode 100644 index 00000000..4b13d168 --- /dev/null +++ b/kernel/patches/devices/s8003/0004-arm64-dts-apple-enable-G5P-on-J71t.patch @@ -0,0 +1,53 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] arm64: dts: apple: enable G5P on J71t + +Use chip compatible matching; only the qualified board enables the node. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s8003-j71t.dts | 4 ++++ + arch/arm64/boot/dts/apple/s8003.dtsi | 17 +++++++++++++++++ + 2 files changed, 21 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s8003-j71t.dts b/arch/arm64/boot/dts/apple/s8003-j71t.dts +index 5b129d695..323c27e27 100644 +--- a/arch/arm64/boot/dts/apple/s8003-j71t.dts ++++ b/arch/arm64/boot/dts/apple/s8003-j71t.dts +@@ -76,6 +76,10 @@ + apple,tethered-boot; + }; + ++&gpu_diag { ++ status = "okay"; ++}; ++ + &pcie0 { + apple,paired-system-sleep; + }; +diff --git a/arch/arm64/boot/dts/apple/s8003.dtsi b/arch/arm64/boot/dts/apple/s8003.dtsi +index b51588941..b7b7a2188 100644 +--- a/arch/arm64/boot/dts/apple/s8003.dtsi ++++ b/arch/arm64/boot/dts/apple/s8003.dtsi +@@ -336,3 +336,20 @@ + * the S8003 (APL1022) on 16nm. There are some minor differences + * such as timing in cpufreq state transistions. + */ ++ ++/* G5P resources; enable only on a board qualified with this driver. */ ++&{/soc} { ++ gpu_diag: gpu@201000000 { ++ status = "disabled"; ++ compatible = "apple,s8003-gpu"; ++ reg = <0x2 0x01000000 0x0 0x100000>, ++ <0x2 0x01e00000 0x0 0x100000>; ++ reg-names = "rgx", "fender"; ++ power-domains = <&ps_gfx>; ++ interrupts = , ++ , ++ , ++ , ++ ; ++ }; ++}; diff --git a/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch b/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch new file mode 100644 index 00000000..ff7c75cc --- /dev/null +++ b/kernel/patches/devices/s800x/0002-arm64-dts-apple-preserve-PMU-and-RTC-interrupt-owner.patch @@ -0,0 +1,42 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] arm64: dts: apple: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +index 2ce40b469..20de6953f 100644 +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -191,11 +191,11 @@ + interrupt-parent = <&pmu_gpio>; + /* + * The PMU line, ADC done, then the events the iPadOS +- * charger subscribes to. ++ * charger subscribes to, except RTC event 16: Linux gives that ++ * event exclusively to rtc@500, not to source identification. + */ + interrupts = <240 IRQ_TYPE_EDGE_RISING>, + <176 IRQ_TYPE_EDGE_RISING>, +- <16 IRQ_TYPE_EDGE_RISING>, + <40 IRQ_TYPE_EDGE_RISING>, + <41 IRQ_TYPE_EDGE_RISING>, + <42 IRQ_TYPE_EDGE_RISING>, +@@ -218,6 +218,9 @@ + reg = <0x500 0x10>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; ++ interrupt-parent = <&pmu_gpio>; ++ interrupts = <16 IRQ_TYPE_EDGE_RISING>; ++ wakeup-source; + }; + + /* D2365 register 0x31c bit 0 */ diff --git a/kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch b/kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch new file mode 100644 index 00000000..9de5ab14 --- /dev/null +++ b/kernel/patches/devices/t8010/0022-arm64-dts-apple-retain-J71t-charger-wake.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:00:00 +0100 +Subject: [PATCH] arm64: dts: apple: retain charger wake after shared PMU data + +Keep the charger marked as a wake source after the shared PMU data has +been applied. Preserve the separate RTC event ownership. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -185,6 +185,7 @@ + }; + + tristar_charger: charger@180 { ++ wakeup-source; + compatible = "apple,j71t-tristar-charger", "apple,tristar-charger"; + reg = <0x180 0x10>, <0xa00 0x04>, <0x4000 0x03>; + reg-names = "status", "control", "adc"; diff --git a/kernel/patches/devices/t8010/0023-arm64-dts-apple-ipad5-Enable-USB-host-on-the-Lightni.patch b/kernel/patches/devices/t8010/0023-arm64-dts-apple-ipad5-Enable-USB-host-on-the-Lightni.patch new file mode 100644 index 00000000..1d53794d --- /dev/null +++ b/kernel/patches/devices/t8010/0023-arm64-dts-apple-ipad5-Enable-USB-host-on-the-Lightni.patch @@ -0,0 +1,47 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 3 Nov 2026 14:46:21 +0100 +Subject: [PATCH] arm64: dts: apple: ipad5: Enable USB host on the Lightning + port + +Enable the S8000 USB role switch on the iPad 5 with the D2365 PMIC +(accessory power LDO) and the Tristar (ACC pin routing), and point the +Lightning charger at it, so that its accessory policy can give the port +to the host controllers while the camera adapter is attached. The +charger keeps setting the device controller's role through its port. + +Tested on an iPad 5 (J71t) with the Lightning to USB camera adapter: +attaching it switches the port to the host role with EHCI and OHCI up; +USB mass storage transfers at about 36 MB/s; a powered USB-C hub +enumerates with its hub, SD card reader and RTL8152 Ethernet; detaching +the adapter returns the port to the device role and restores the USB +gadget. + +Assisted-by: LLM +--- + arch/arm64/boot/dts/apple/s800x-ipad5.dtsi | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +index 8f19a2f0d..8529ac608 100644 +--- a/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi ++++ b/arch/arm64/boot/dts/apple/s800x-ipad5.dtsi +@@ -204,6 +204,7 @@ tristar_charger: charger@180 { + <72 IRQ_TYPE_EDGE_RISING>, + <77 IRQ_TYPE_EDGE_RISING>; + apple,tristar = <&tristar>; ++ apple,usb-role-switch = <&usb_role>; + input-current-limit-microamp = <2400000>; + + /* The charger decides when USB data may come on. */ +@@ -477,3 +478,10 @@ usbdev_role: endpoint { + }; + }; + }; ++ ++/* Lightning host role: accessory power from LDO5, ACC routing by the Tristar. */ ++&usb_role { ++ apple,pmic = <&d2365_pmic>; ++ apple,tristar = <&tristar>; ++ status = "okay"; ++}; diff --git a/kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..6fd6514b --- /dev/null +++ b/kernel/patches/drivers/base/0001-core-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index 3fcdf4b4e..fcc51b699 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4589,6 +4589,7 @@ bool pr_flush(int timeout_ms, bool reset_on_progress) + { + return __pr_flush(NULL, timeout_ms, reset_on_progress); + } ++EXPORT_SYMBOL_GPL(pr_flush); + + /* + * Delayed printk version, for scheduler-internal messages: diff --git a/kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..c3550de8 --- /dev/null +++ b/kernel/patches/drivers/base/0002-core-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,26 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] core: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + kernel/kexec_core.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c +index dc770b9a6..819650bcd 100644 +--- a/kernel/kexec_core.c ++++ b/kernel/kexec_core.c +@@ -53,6 +53,7 @@ atomic_t __kexec_lock = ATOMIC_INIT(0); + + /* Flag to indicate we are going to kexec a new kernel */ + bool kexec_in_progress = false; ++EXPORT_SYMBOL_GPL(kexec_in_progress); + + bool kexec_file_dbg_print; + diff --git a/kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch b/kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch new file mode 100644 index 00000000..054cba10 --- /dev/null +++ b/kernel/patches/drivers/base/0003-printk-export-kmsg_dump_desc-for-module-cold-reset-d.patch @@ -0,0 +1,27 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:54:33 +0100 +Subject: [PATCH] printk: export kmsg_dump_desc for module cold-reset + diagnostics + +The VXD shutdown fail-safe is a GPL module. Allow it to dump emergency +evidence before a cold reset, matching the built-in GPU and PCIe paths. +No printk behavior changes. + +Assisted-by: LLM +--- + kernel/printk/printk.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c +index fcc51b699..a812ec1e5 100644 +--- a/kernel/printk/printk.c ++++ b/kernel/printk/printk.c +@@ -4882,6 +4882,7 @@ void kmsg_dump_desc(enum kmsg_dump_reason reason, const char *desc) + } + rcu_read_unlock(); + } ++EXPORT_SYMBOL_GPL(kmsg_dump_desc); + + /** + * kmsg_dump_get_line - retrieve one kmsg log line diff --git a/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch b/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch new file mode 100644 index 00000000..e510724f --- /dev/null +++ b/kernel/patches/drivers/base/0004-PM-sleep-reserve-driver-admission-before-sync-throug.patch @@ -0,0 +1,205 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:37 +0000 +Subject: [PATCH] PM: sleep: reserve driver admission before sync through + resume or abort + +Some drivers can only enter system sleep while no new work is admitted, +and they have to decide that before the filesystems are synced. Once +sync has run, a refusal from a suspend callback aborts the transition +late, after work that cannot be undone. There is no hook between the +start of a transition and sync for this. + +Add a small admission list to kernel/power/suspend.c. A driver registers +a prepare/complete pair. enter_state() calls every prepare under +system_transition_mutex before sync, then a blocking precheck notifier +chain on which a driver can refuse the transition with -EBUSY. complete +runs in reverse order after resume and after every abort, including a +failed prepare, and the chain is then called with +PM_SUSPEND_PRECHECK_COMPLETE so that notifier users can drop their +state. pm_suspend_admission_reserved() lets a precheck ask whether a +device's admission is held. + +Assisted-by: LLM +--- + include/linux/pm-suspend-admission.h | 29 ++++++++++ + include/linux/suspend.h | 10 ++++ + kernel/power/suspend.c | 82 ++++++++++++++++++++++++++++ + 3 files changed, 121 insertions(+) + create mode 100644 include/linux/pm-suspend-admission.h + +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +new file mode 100644 +index 000000000..f1c92e058 +--- /dev/null ++++ b/include/linux/pm-suspend-admission.h +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _LINUX_PM_SUSPEND_ADMISSION_H ++#define _LINUX_PM_SUSPEND_ADMISSION_H ++#include ++#include ++ ++/* Driver-owned reversible admission gate. Registration must precede exposing ++ * hardware work. Unregister before removing the hardware or freeing this object. ++ * prepare must reserve admission without destructive MMIO; complete releases it ++ * after resume or ANY abort, including a failed prepare. No PM-op inference. ++ */ ++struct pm_suspend_admission { ++ struct list_head node; ++ struct device *dev; ++ int (*prepare)(struct device *dev); ++ void (*complete)(struct device *dev); ++ bool reserved; ++}; ++#ifdef CONFIG_SUSPEND ++int register_pm_suspend_admission(struct pm_suspend_admission *admission); ++void unregister_pm_suspend_admission(struct pm_suspend_admission *admission); ++/* Only callable from a precheck notifier, under system_transition_mutex. */ ++bool pm_suspend_admission_reserved(struct device *dev); ++#else ++static inline int register_pm_suspend_admission(struct pm_suspend_admission *a) { return 0; } ++static inline void unregister_pm_suspend_admission(struct pm_suspend_admission *a) {} ++static inline bool pm_suspend_admission_reserved(struct device *dev) { return false; } ++#endif ++#endif +diff --git a/include/linux/suspend.h b/include/linux/suspend.h +index b02876f1a..93082b990 100644 +--- a/include/linux/suspend.h ++++ b/include/linux/suspend.h +@@ -142,9 +142,19 @@ struct platform_s2idle_ops { + void (*end)(void); + }; + ++#ifdef CONFIG_SUSPEND ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb); ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb); ++#else ++static inline int register_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } ++static inline int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } ++#endif ++#define PM_SUSPEND_PRECHECK_COMPLETE 0 ++ + #ifdef CONFIG_SUSPEND + extern suspend_state_t pm_suspend_target_state; + extern suspend_state_t mem_sleep_current; ++ + extern suspend_state_t mem_sleep_default; + + /** +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 57c442686..307e6d5f3 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -9,6 +9,7 @@ + + #define pr_fmt(fmt) "PM: " fmt + ++#include + #include + #include + #include +@@ -573,8 +574,68 @@ static void suspend_finish(void) + * Fail if that's not the case. Otherwise, prepare for system suspend, make the + * system enter the given sleep state and clean up after wakeup. + */ ++static LIST_HEAD(suspend_admissions); ++ ++int register_pm_suspend_admission(struct pm_suspend_admission *a) ++{ ++ struct pm_suspend_admission *other; ++ ++ if (!a->dev || !a->prepare || !a->complete) ++ return -EINVAL; ++ mutex_lock(&system_transition_mutex); ++ list_for_each_entry(other, &suspend_admissions, node) { ++ if (other->dev == a->dev) { ++ mutex_unlock(&system_transition_mutex); ++ return -EEXIST; ++ } ++ } ++ a->reserved = false; ++ get_device(a->dev); ++ list_add_tail(&a->node, &suspend_admissions); ++ mutex_unlock(&system_transition_mutex); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_admission); ++ ++void unregister_pm_suspend_admission(struct pm_suspend_admission *a) ++{ ++ /* Serializes removal/module unload with reservation through complete. */ ++ mutex_lock(&system_transition_mutex); ++ list_del(&a->node); ++ put_device(a->dev); ++ mutex_unlock(&system_transition_mutex); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_admission); ++ ++bool pm_suspend_admission_reserved(struct device *dev) ++{ ++ struct pm_suspend_admission *a; ++ ++ lockdep_assert_held(&system_transition_mutex); ++ list_for_each_entry(a, &suspend_admissions, node) ++ if (a->dev == dev) ++ return a->reserved; ++ return false; ++} ++EXPORT_SYMBOL_GPL(pm_suspend_admission_reserved); ++ ++static BLOCKING_NOTIFIER_HEAD(suspend_precheck_chain); ++ ++int register_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_register(&suspend_precheck_chain, nb); ++} ++EXPORT_SYMBOL_GPL(register_pm_suspend_precheck_notifier); ++ ++int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) ++{ ++ return blocking_notifier_chain_unregister(&suspend_precheck_chain, nb); ++} ++EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); ++ + static int enter_state(suspend_state_t state) + { ++ struct pm_suspend_admission *admission; + int error; + + trace_suspend_resume(TPS("suspend_enter"), state, true); +@@ -594,6 +655,18 @@ static int enter_state(suspend_state_t state) + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); + ++ list_for_each_entry(admission, &suspend_admissions, node) { ++ /* Also release a provider whose prepare returned an error. */ ++ admission->reserved = true; ++ error = admission->prepare(admission->dev); ++ if (error) ++ goto Unlock; ++ } ++ error = notifier_to_errno(blocking_notifier_call_chain( ++ &suspend_precheck_chain, state, NULL)); ++ if (error) ++ goto Unlock; ++ + if (sync_on_suspend_enabled) { + trace_suspend_resume(TPS("sync_filesystems"), 0, true); + +@@ -622,6 +695,15 @@ static int enter_state(suspend_state_t state) + pm_pr_dbg("Finishing wakeup.\n"); + suspend_finish(); + Unlock: ++ list_for_each_entry_reverse(admission, &suspend_admissions, node) { ++ if (!admission->reserved) ++ continue; ++ admission->complete(admission->dev); ++ admission->reserved = false; ++ } ++ /* Release driver admission reservations on success and every abort. */ ++ blocking_notifier_call_chain(&suspend_precheck_chain, ++ PM_SUSPEND_PRECHECK_COMPLETE, NULL); + mutex_unlock(&system_transition_mutex); + return error; + } diff --git a/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch b/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch new file mode 100644 index 00000000..88392fe4 --- /dev/null +++ b/kernel/patches/drivers/base/0005-PM-sleep-report-suspend-admission-readiness-in-sys-p.patch @@ -0,0 +1,176 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:11:36 +0000 +Subject: [PATCH] PM: sleep: report suspend admission readiness in + /sys/power/suspend_ready + +Driver admission providers and the early suspend prechecks decide inside +enter_state() whether a suspend is admitted, but userspace has no way to +ask before requesting one. A desktop that offers Suspend, or a power +daemon that would suspend on idle, can only try it and see the +transition fail. + +Add an optional non-reserving check callback to struct +pm_suspend_admission and a read-only /sys/power/suspend_ready. Reading +it takes system_transition_mutex without waiting, runs every provider's +check and then every precheck notifier for the current mem_sleep state, +and reports "ready" or "not-ready " for the first refusal. +While the query runs, pm_suspend_admission_reserved() also reports a +provider whose check passed, so prechecks that require a reserved +provider answer as they would during a suspend. A provider without a +check reports not ready. No reservation is taken, nothing is logged by +the query itself, and a later suspend reserves and checks again. + +Assisted-by: LLM +--- + Documentation/ABI/testing/sysfs-power | 18 ++++++++ + include/linux/pm-suspend-admission.h | 10 ++++- + kernel/power/suspend.c | 63 ++++++++++++++++++++++++++- + 3 files changed, 89 insertions(+), 2 deletions(-) + +diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power +index d38da0779..98cc5c2b5 100644 +--- a/Documentation/ABI/testing/sysfs-power ++++ b/Documentation/ABI/testing/sysfs-power +@@ -470,3 +470,21 @@ Description: + + Minimum value: 1 + Default value: 3 ++ ++What: /sys/power/suspend_ready ++Date: November 2026 ++Contact: Linux PM list ++Description: ++ Read-only report of whether a suspend to the current ++ mem_sleep state would be admitted now. Reading runs the ++ admission a suspend runs before syncing filesystems: each ++ registered driver admission provider's non-reserving check, ++ then each early suspend precheck. It takes no reservation ++ and changes no state; a later suspend checks again. ++ ++ The file reads "ready", or one line ++ "not-ready " naming the first refusal: a ++ device name for a provider, the precheck callback's symbol, ++ "transition" while a system transition is in progress, or ++ "state" when the mem_sleep state is not available. ++ is a negative decimal errno value. +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +index f1c92e058..0b1a1c57b 100644 +--- a/include/linux/pm-suspend-admission.h ++++ b/include/linux/pm-suspend-admission.h +@@ -8,18 +8,26 @@ + * hardware work. Unregister before removing the hardware or freeing this object. + * prepare must reserve admission without destructive MMIO; complete releases it + * after resume or ANY abort, including a failed prepare. No PM-op inference. ++ * check, optional, answers /sys/power/suspend_ready: whether prepare would ++ * admit now, without reserving or any side effect. Work prepare would drain ++ * is not a refusal. A provider without check reports not ready. + */ + struct pm_suspend_admission { + struct list_head node; + struct device *dev; + int (*prepare)(struct device *dev); + void (*complete)(struct device *dev); ++ int (*check)(struct device *dev); + bool reserved; ++ bool checked; /* check passed during a readiness query */ + }; + #ifdef CONFIG_SUSPEND + int register_pm_suspend_admission(struct pm_suspend_admission *admission); + void unregister_pm_suspend_admission(struct pm_suspend_admission *admission); +-/* Only callable from a precheck notifier, under system_transition_mutex. */ ++/* ++ * Only callable from a precheck notifier, under system_transition_mutex. ++ * During a readiness query it reports a provider whose check passed. ++ */ + bool pm_suspend_admission_reserved(struct device *dev); + #else + static inline int register_pm_suspend_admission(struct pm_suspend_admission *a) { return 0; } +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 307e6d5f3..670f50ad8 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -590,6 +590,7 @@ int register_pm_suspend_admission(struct pm_suspend_admission *a) + } + } + a->reserved = false; ++ a->checked = false; + get_device(a->dev); + list_add_tail(&a->node, &suspend_admissions); + mutex_unlock(&system_transition_mutex); +@@ -614,7 +615,7 @@ bool pm_suspend_admission_reserved(struct device *dev) + lockdep_assert_held(&system_transition_mutex); + list_for_each_entry(a, &suspend_admissions, node) + if (a->dev == dev) +- return a->reserved; ++ return a->reserved || a->checked; + return false; + } + EXPORT_SYMBOL_GPL(pm_suspend_admission_reserved); +@@ -633,6 +634,66 @@ int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) + } + EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); + ++/* ++ * The admission enter_state() would run, read only: each provider's check ++ * instead of its reservation, then each precheck notifier for the current ++ * mem_sleep state. No reservation is taken and nothing is logged here; a ++ * later suspend still reserves and checks again. ++ */ ++static ssize_t suspend_ready_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ suspend_state_t state = mem_sleep_current; ++ struct pm_suspend_admission *a; ++ struct notifier_block *nb; ++ ssize_t len = 0; ++ int error; ++ ++ if (!mutex_trylock(&system_transition_mutex)) ++ return sysfs_emit(buf, "not-ready transition %d\n", -EBUSY); ++ if (state != PM_SUSPEND_TO_IDLE && !valid_state(state)) { ++ len = sysfs_emit(buf, "not-ready state %d\n", -EINVAL); ++ goto unlock; ++ } ++ list_for_each_entry(a, &suspend_admissions, node) { ++ error = a->check ? a->check(a->dev) : -EOPNOTSUPP; ++ if (error) { ++ len = sysfs_emit(buf, "not-ready %s %d\n", ++ dev_name(a->dev), error); ++ goto clear; ++ } ++ a->checked = true; ++ } ++ /* Walk the chain to name the refusing notifier, not only its errno. */ ++ down_read(&suspend_precheck_chain.rwsem); ++ for (nb = rcu_dereference_raw(suspend_precheck_chain.head); nb; ++ nb = rcu_dereference_raw(nb->next)) { ++ error = notifier_to_errno(nb->notifier_call(nb, state, NULL)); ++ if (error) { ++ len = sysfs_emit(buf, "not-ready %ps %d\n", ++ nb->notifier_call, error); ++ break; ++ } ++ } ++ up_read(&suspend_precheck_chain.rwsem); ++ if (!len) ++ len = sysfs_emit(buf, "ready\n"); ++clear: ++ list_for_each_entry(a, &suspend_admissions, node) ++ a->checked = false; ++unlock: ++ mutex_unlock(&system_transition_mutex); ++ return len; ++} ++ ++power_attr_ro(suspend_ready); ++ ++static int __init suspend_ready_init(void) ++{ ++ return sysfs_create_file(power_kobj, &suspend_ready_attr.attr); ++} ++late_initcall(suspend_ready_init); ++ + static int enter_state(suspend_state_t state) + { + struct pm_suspend_admission *admission; diff --git a/kernel/patches/drivers/base/0006-PM-sleep-let-an-admission-provider-name-its-refusal-.patch b/kernel/patches/drivers/base/0006-PM-sleep-let-an-admission-provider-name-its-refusal-.patch new file mode 100644 index 00000000..6e3ab174 --- /dev/null +++ b/kernel/patches/drivers/base/0006-PM-sleep-let-an-admission-provider-name-its-refusal-.patch @@ -0,0 +1,82 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 19:57:26 +0000 +Subject: [PATCH] PM: sleep: let an admission provider name its refusal in + suspend_ready + +/sys/power/suspend_ready names the refusing provider and its errno only. +-EBUSY from a GPU admission check can mean an unquiescent firmware, a +ring that has not drained or a reservation that is in use, and the +reader cannot tell them apart without a kernel log. + +Add an optional reason callback to struct pm_suspend_admission. It is +called only right after the provider's check refused, under +system_transition_mutex, and returns a short token. The token is joined +to the device name, "not-ready : ", so the line +keeps its three fields and existing parsers keep working. A token that +contains white space or a colon is dropped and the old line is printed. + +Assisted-by: LLM +--- + Documentation/ABI/testing/sysfs-power | 5 +++++ + include/linux/pm-suspend-admission.h | 3 +++ + kernel/power/suspend.c | 12 ++++++++++-- + 3 files changed, 18 insertions(+), 2 deletions(-) + +diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power +index 98cc5c2b5..e53150e75 100644 +--- a/Documentation/ABI/testing/sysfs-power ++++ b/Documentation/ABI/testing/sysfs-power +@@ -488,3 +488,8 @@ Description: + "transition" while a system transition is in progress, or + "state" when the mem_sleep state is not available. + is a negative decimal errno value. ++ ++ A provider may name its reason: is then ++ ":", where contains no white ++ space or colon, e.g. ++ "not-ready 201000000.gpu:never-park/gpc=04 -16". +diff --git a/include/linux/pm-suspend-admission.h b/include/linux/pm-suspend-admission.h +index 0b1a1c57b..c1996d2e5 100644 +--- a/include/linux/pm-suspend-admission.h ++++ b/include/linux/pm-suspend-admission.h +@@ -11,6 +11,8 @@ + * check, optional, answers /sys/power/suspend_ready: whether prepare would + * admit now, without reserving or any side effect. Work prepare would drain + * is not a refusal. A provider without check reports not ready. ++ * reason, optional, is called only right after check refused, under the same ++ * lock: a short token naming why (no white space or colon), or NULL. + */ + struct pm_suspend_admission { + struct list_head node; +@@ -18,6 +20,7 @@ struct pm_suspend_admission { + int (*prepare)(struct device *dev); + void (*complete)(struct device *dev); + int (*check)(struct device *dev); ++ const char *(*reason)(struct device *dev); + bool reserved; + bool checked; /* check passed during a readiness query */ + }; +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 670f50ad8..8bb2c7730 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -658,8 +658,16 @@ static ssize_t suspend_ready_show(struct kobject *kobj, + list_for_each_entry(a, &suspend_admissions, node) { + error = a->check ? a->check(a->dev) : -EOPNOTSUPP; + if (error) { +- len = sysfs_emit(buf, "not-ready %s %d\n", +- dev_name(a->dev), error); ++ const char *why = a->check && a->reason ? ++ a->reason(a->dev) : NULL; ++ ++ /* One token: ":" keeps three fields. */ ++ if (why && *why && !strpbrk(why, " \t\n:")) ++ len = sysfs_emit(buf, "not-ready %s:%s %d\n", ++ dev_name(a->dev), why, error); ++ else ++ len = sysfs_emit(buf, "not-ready %s %d\n", ++ dev_name(a->dev), error); + goto clear; + } + a->checked = true; diff --git a/kernel/patches/drivers/base/0007-PM-sleep-name-every-admission-and-precheck-refusal-i.patch b/kernel/patches/drivers/base/0007-PM-sleep-name-every-admission-and-precheck-refusal-i.patch new file mode 100644 index 00000000..3e20983b --- /dev/null +++ b/kernel/patches/drivers/base/0007-PM-sleep-name-every-admission-and-precheck-refusal-i.patch @@ -0,0 +1,148 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:26:19 +0100 +Subject: [PATCH] PM: sleep: name every admission and precheck refusal in the + log + +enter_state() refused silently: a held system_transition_mutex, a +provider's prepare and a precheck notifier all returned -EBUSY with no +line, and suspend_stats records no step for them. A refused s2idle +attempt then shows only "PM: suspend exit". + +Log each refusal with its errno: the held transition lock, the refusing +provider by device name, and the refusing precheck notifier by symbol. +The readiness query and the attempt now walk the precheck chain through +one helper. During an attempt the notifiers get +PM_SUSPEND_PRECHECK_ATTEMPT as their data (the readiness query still +passes NULL), so a notifier can log its own reason for an attempt +without logging on every poll of /sys/power/suspend_ready. + +Assisted-by: LLM +--- + include/linux/suspend.h | 6 ++++ + kernel/power/suspend.c | 62 ++++++++++++++++++++++++++++++----------- + 2 files changed, 51 insertions(+), 17 deletions(-) + +diff --git a/include/linux/suspend.h b/include/linux/suspend.h +index 93082b990..782bbad59 100644 +--- a/include/linux/suspend.h ++++ b/include/linux/suspend.h +@@ -150,6 +150,12 @@ static inline int register_pm_suspend_precheck_notifier(struct notifier_block *n + static inline int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) { return 0; } + #endif + #define PM_SUSPEND_PRECHECK_COMPLETE 0 ++/* ++ * A precheck notifier's data: this cookie during a suspend attempt, NULL ++ * during a readiness query (/sys/power/suspend_ready). A notifier logs why it ++ * refuses only for an attempt, so polling the query stays quiet. ++ */ ++#define PM_SUSPEND_PRECHECK_ATTEMPT ((void *)1) + + #ifdef CONFIG_SUSPEND + extern suspend_state_t pm_suspend_target_state; +diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c +index 8bb2c7730..cd5b6a703 100644 +--- a/kernel/power/suspend.c ++++ b/kernel/power/suspend.c +@@ -634,6 +634,31 @@ int unregister_pm_suspend_precheck_notifier(struct notifier_block *nb) + } + EXPORT_SYMBOL_GPL(unregister_pm_suspend_precheck_notifier); + ++/* ++ * Run the precheck chain for @state, stopping at the first refusal, and name ++ * the refusing notifier in *who. Shared by the readiness query and the attempt ++ * so that both walk exactly the same chain. ++ */ ++static int suspend_precheck_run(suspend_state_t state, void *data, ++ struct notifier_block **who) ++{ ++ struct notifier_block *nb; ++ int error = 0; ++ ++ *who = NULL; ++ down_read(&suspend_precheck_chain.rwsem); ++ for (nb = rcu_dereference_raw(suspend_precheck_chain.head); nb; ++ nb = rcu_dereference_raw(nb->next)) { ++ error = notifier_to_errno(nb->notifier_call(nb, state, data)); ++ if (error) { ++ *who = nb; ++ break; ++ } ++ } ++ up_read(&suspend_precheck_chain.rwsem); ++ return error; ++} ++ + /* + * The admission enter_state() would run, read only: each provider's check + * instead of its reservation, then each precheck notifier for the current +@@ -672,18 +697,11 @@ static ssize_t suspend_ready_show(struct kobject *kobj, + } + a->checked = true; + } +- /* Walk the chain to name the refusing notifier, not only its errno. */ +- down_read(&suspend_precheck_chain.rwsem); +- for (nb = rcu_dereference_raw(suspend_precheck_chain.head); nb; +- nb = rcu_dereference_raw(nb->next)) { +- error = notifier_to_errno(nb->notifier_call(nb, state, NULL)); +- if (error) { +- len = sysfs_emit(buf, "not-ready %ps %d\n", +- nb->notifier_call, error); +- break; +- } +- } +- up_read(&suspend_precheck_chain.rwsem); ++ /* Name the refusing notifier, not only its errno. */ ++ error = suspend_precheck_run(state, NULL, &nb); ++ if (error) ++ len = sysfs_emit(buf, "not-ready %ps %d\n", nb->notifier_call, ++ error); + if (!len) + len = sysfs_emit(buf, "ready\n"); + clear: +@@ -705,6 +723,7 @@ late_initcall(suspend_ready_init); + static int enter_state(suspend_state_t state) + { + struct pm_suspend_admission *admission; ++ struct notifier_block *nb; + int error; + + trace_suspend_resume(TPS("suspend_enter"), state, true); +@@ -718,23 +737,32 @@ static int enter_state(suspend_state_t state) + } else if (!valid_state(state)) { + return -EINVAL; + } +- if (!mutex_trylock(&system_transition_mutex)) ++ if (!mutex_trylock(&system_transition_mutex)) { ++ pr_info("suspend refused: another system transition holds the lock %d\n", ++ -EBUSY); + return -EBUSY; ++ } + + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); + ++ /* Every admission refusal is named: no silent -EBUSY before the sync. */ + list_for_each_entry(admission, &suspend_admissions, node) { + /* Also release a provider whose prepare returned an error. */ + admission->reserved = true; + error = admission->prepare(admission->dev); +- if (error) ++ if (error) { ++ pr_info("suspend refused: admission %s %d\n", ++ dev_name(admission->dev), error); + goto Unlock; ++ } + } +- error = notifier_to_errno(blocking_notifier_call_chain( +- &suspend_precheck_chain, state, NULL)); +- if (error) ++ error = suspend_precheck_run(state, PM_SUSPEND_PRECHECK_ATTEMPT, &nb); ++ if (error) { ++ pr_info("suspend refused: precheck %ps %d\n", nb->notifier_call, ++ error); + goto Unlock; ++ } + + if (sync_on_suspend_enabled) { + trace_suspend_resume(TPS("sync_filesystems"), 0, true); diff --git a/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..2865725b --- /dev/null +++ b/kernel/patches/drivers/drm/0008-drm-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] drm: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/gpu/drm/adbe/adbe_drv.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..7ed2de3d7 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -1065,6 +1065,9 @@ static void adbe_shutdown(struct platform_device *pdev) + struct adbe *adbe = platform_get_drvdata(pdev); + + drm_atomic_helper_shutdown(&adbe->drm); ++ timer_delete_sync(&adbe->flip_timer); ++ /* release9: restore the boot buffer even if the CRTC was never enabled. */ ++ adbe_park(adbe); + writel(adbe->saved_inten, adbe->regs + ADP_INTEN); + } + diff --git a/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch b/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch new file mode 100644 index 00000000..d0acb8a6 --- /dev/null +++ b/kernel/patches/drivers/drm/0009-dt-bindings-gpu-describe-Apple-S8003-G5P-resources.patch @@ -0,0 +1,46 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] dt-bindings: gpu: describe Apple S8003 G5P resources + +Use chip compatible matching; only the qualified board enables the node. + +Assisted-by: LLM +--- + .../bindings/gpu/apple,s8003-g5p.yaml | 27 +++++++++++++++++++ + 1 file changed, 27 insertions(+) + create mode 100644 Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml + +diff --git a/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +new file mode 100644 +index 000000000..76aebc183 +--- /dev/null ++++ b/Documentation/devicetree/bindings/gpu/apple,s8003-g5p.yaml +@@ -0,0 +1,27 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/gpu/apple,s8003-g5p.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++title: Apple S8003 G5P GPU ++maintainers: ++ - Berkay Kösebay ++description: | ++ G5P GPU with META firmware on S8003. The compatible is Apple's inherited ++ ADT spelling used by the qualified driver; no new vendor prefix is implied. ++properties: ++ compatible: ++ const: apple,s8003-gpu ++ reg: ++ maxItems: 2 ++ reg-names: ++ items: ++ - const: rgx ++ - const: fender ++ interrupts: ++ minItems: 5 ++ maxItems: 5 ++ power-domains: ++ maxItems: 1 ++required: [compatible, reg, reg-names, interrupts, power-domains] ++additionalProperties: false diff --git a/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch b/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch new file mode 100644 index 00000000..4da6e1b6 --- /dev/null +++ b/kernel/patches/drivers/drm/0010-drm-apple-g5p-define-the-render-and-compute-userspac.patch @@ -0,0 +1,651 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: define the render and compute userspace + interface + +Import the sealed warmport ABI, before pipeline stage wait extensions. + +Assisted-by: LLM +--- + include/uapi/drm/apple_g5p_drm.h | 631 +++++++++++++++++++++++++++++++ + 1 file changed, 631 insertions(+) + create mode 100644 include/uapi/drm/apple_g5p_drm.h + +diff --git a/include/uapi/drm/apple_g5p_drm.h b/include/uapi/drm/apple_g5p_drm.h +new file mode 100644 +index 000000000..5e41ab533 +--- /dev/null ++++ b/include/uapi/drm/apple_g5p_drm.h +@@ -0,0 +1,631 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++#ifndef _UAPI_APPLE_G5P_DRM_H_ ++#define _UAPI_APPLE_G5P_DRM_H_ ++ ++#include "drm.h" ++ ++/* G5P merged winsys ABI. Retired ioctl numbers remain reserved. */ ++#define DRM_G5P_UAPI_VERSION 1 ++#define DRM_G5P_CAP_BO_CPU (1ULL << 0) ++#define DRM_G5P_CAP_VM_PREPARE (1ULL << 1) ++#define DRM_G5P_CAP_SYNCOBJ (1ULL << 2) ++#define DRM_G5P_CAP_RENDER (1ULL << 5) ++#define DRM_G5P_CAP_PRIME (1ULL << 6) ++#define DRM_G5P_CAP_CODE_BASE_VERIFIED (1ULL << 7) ++#define DRM_G5P_CAP_VM_LIVE (1ULL << 8) ++#define DRM_G5P_CAP_CONTEXT_LIVE (1ULL << 9) ++#define DRM_G5P_CAP_JOB_STATUS (1ULL << 10) ++#define DRM_G5P_CAP_CONTEXT_STATUS (1ULL << 11) ++#define DRM_G5P_CAP_FAULT_INFO (1ULL << 13) ++ ++/* LIVE means the owned user catalogue has completed firmware attachment. */ ++#define DRM_G5P_CONTEXT_LIVE 3 ++#define DRM_G5P_CONTEXT_HOST_ONLY 0 ++#define DRM_G5P_CONTEXT_LOST 1 ++ ++struct drm_g5p_get_context_status { ++ __u32 context; ++ __u32 flags; ++ __u32 state; ++ __u32 map_count; ++ __u64 mapped_bytes; ++ __u64 reserved; ++}; ++ ++/* First terminal firmware event, retained independently of last_event. */ ++#define DRM_G5P_FAULT_NONE 0 ++#define DRM_G5P_FAULT_RECOVERY 1 ++#define DRM_G5P_FAULT_PAGE_REQUEST 2 ++#define DRM_G5P_FAULT_CHANNEL 3 ++#define DRM_G5P_FAULT_UNKNOWN 4 ++#define DRM_G5P_FAULT_VALID_CHANNEL (1U << 0) ++struct drm_g5p_get_fault_info { ++ __u32 flags; ++ __u32 kind; ++ __u64 event_serial; /* One-based arrival order; zero if no fault. */ ++ __u32 valid; ++ __u32 channel_class; /* Event8: 0 memory failure, 1 killed, others opaque. */ ++ __u32 channel_token; /* Firmware token, not a DRM context handle. */ ++ __u32 channel_stamp; ++ __u32 raw[11]; /* Includes unknown fields; no decoded fault VA is claimed. */ ++ __u32 reserved; ++}; ++ ++struct drm_g5p_get_info { ++ __u32 size; ++ __u32 flags; ++ __u32 version; ++ __u32 gpu_page_shift; ++ __u64 capabilities; ++ __u64 va_start; ++ __u64 va_end; ++ __u64 reserved_start; ++ __u64 reserved_end; ++ __u64 code_base; /* Zero until CODE_BASE_VERIFIED. */ ++ __u64 max_bo_size; ++ __u32 bvnc[4]; /* Static target, not a new MMIO measurement. */ ++ __u64 reserved[2]; ++}; ++ ++/* GPU access roles; CPU initialization/mmap is permitted for every role. */ ++#define DRM_G5P_BO_READ (1U << 0) ++#define DRM_G5P_BO_WRITE (1U << 1) ++#define DRM_G5P_BO_EXEC (1U << 2) ++struct drm_g5p_create_bo { ++ __u64 size; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_bo_mmap_offset { ++ __u32 handle; ++ __u32 flags; ++ __u64 offset; ++}; ++ ++#define DRM_G5P_BO_SYNC_TO_DEVICE 0 ++#define DRM_G5P_BO_SYNC_TO_CPU 1 ++struct drm_g5p_bo_sync { ++ __u32 handle; ++ __u32 op; ++ __u64 offset; ++ __u64 size; ++}; ++ ++struct drm_g5p_create_vm { ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_vm { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++#define DRM_G5P_VM_MAP 0 ++#define DRM_G5P_VM_UNMAP 1 ++struct drm_g5p_vm_bind { ++ __u32 vm; ++ __u32 bo; ++ __u64 va; ++ __u64 size; ++ __u64 offset; ++ __u32 op; ++ __u32 flags; ++}; ++ ++struct drm_g5p_create_context { ++ __u32 vm; ++ __u32 type; ++ __u32 flags; ++ __u32 handle; ++}; ++ ++struct drm_g5p_destroy_context { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* A point of zero uses a binary syncobj; nonzero uses a timeline point. */ ++#define DRM_G5P_SYNC_SIGNAL (1U << 0) ++struct drm_g5p_sync { ++ __u32 handle; ++ __u32 flags; ++ __u64 point; ++}; ++ ++/* Privileged one-shot kernel fixture; all payload/BO fields must be zero. ++ * The empty user context authorizes ownership only; its VM is not attached. ++ * Does not advertise general COMPUTE or attach user-provided GPU addresses. ++ */ ++ ++/* Geometry is all zero/NONE while CODE_BASE_VERIFIED is clear. */ ++#define DRM_G5P_BASE_NONE 0 ++#define DRM_G5P_BASE_DEVICE 1 ++#define DRM_G5P_BASE_VM 2 ++#define DRM_G5P_BASE_CONTEXT 3 ++struct drm_g5p_get_exec_info { ++ __u32 size; ++ __u32 flags; ++ __u32 base_scope; ++ __u32 pad; ++ __u64 usc_base; ++ __u64 pds_base; ++ __u64 window_size; ++ __u64 allocation_alignment; ++ __u32 usc_offset_bits; ++ __u32 pds_offset_bits; ++ __u64 reserved[3]; ++}; ++ ++#define DRM_G5P_JOB_PENDING 0 ++#define DRM_G5P_JOB_COMPLETE 1 ++struct drm_g5p_get_job_status { ++ __u64 job_id; ++ __u32 flags; ++ __u32 state; ++ __s32 error; /* -EINPROGRESS while pending; immutable after completion. */ ++ __u32 pad; ++ __u64 reset_generation; /* Zero: no reset/recovery support. */ ++}; ++ ++#define DRM_G5P_GET_INFO 0x00 ++#define DRM_G5P_CREATE_BO 0x01 ++#define DRM_G5P_BO_MMAP_OFFSET 0x02 ++#define DRM_G5P_BO_SYNC 0x03 ++#define DRM_G5P_CREATE_VM 0x04 ++#define DRM_G5P_DESTROY_VM 0x05 ++#define DRM_G5P_VM_BIND 0x06 ++#define DRM_G5P_CREATE_CONTEXT 0x07 ++#define DRM_G5P_DESTROY_CONTEXT 0x08 ++#define DRM_G5P_GET_EXEC_INFO 0x0b ++#define DRM_G5P_GET_JOB_STATUS 0x0c ++#define DRM_G5P_GET_CONTEXT_STATUS 0x0d ++#define DRM_G5P_GET_FAULT_INFO 0x0e ++ ++#define DRM_IOCTL_G5P_GET_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_INFO, struct drm_g5p_get_info) ++#define DRM_IOCTL_G5P_CREATE_BO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_BO, struct drm_g5p_create_bo) ++#define DRM_IOCTL_G5P_BO_MMAP_OFFSET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_BO_MMAP_OFFSET, \ ++ struct drm_g5p_bo_mmap_offset) ++#define DRM_IOCTL_G5P_BO_SYNC \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_BO_SYNC, struct drm_g5p_bo_sync) ++#define DRM_IOCTL_G5P_CREATE_VM \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_VM, struct drm_g5p_create_vm) ++#define DRM_IOCTL_G5P_DESTROY_VM \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_VM, \ ++ struct drm_g5p_destroy_vm) ++#define DRM_IOCTL_G5P_VM_BIND \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_VM_BIND, struct drm_g5p_vm_bind) ++#define DRM_IOCTL_G5P_CREATE_CONTEXT \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_CONTEXT, \ ++ struct drm_g5p_create_context) ++#define DRM_IOCTL_G5P_DESTROY_CONTEXT \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_CONTEXT, \ ++ struct drm_g5p_destroy_context) ++ ++#define DRM_IOCTL_G5P_GET_EXEC_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_EXEC_INFO, \ ++ struct drm_g5p_get_exec_info) ++#define DRM_IOCTL_G5P_GET_JOB_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_JOB_STATUS, \ ++ struct drm_g5p_get_job_status) ++ ++#define DRM_IOCTL_G5P_GET_CONTEXT_STATUS \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_CONTEXT_STATUS, \ ++ struct drm_g5p_get_context_status) ++ ++#define DRM_IOCTL_G5P_GET_FAULT_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_FAULT_INFO, \ ++ struct drm_g5p_get_fault_info) ++ ++#define DRM_G5P_CAP_IDLE_VM_BIND (1ULL << 18) ++ ++#define DRM_G5P_CAP_BOUNDED_RENDER (1ULL << 19) ++#define DRM_G5P_CAP_TYPED_RENDER (1ULL << 20) ++#define DRM_G5P_CAP_RENDER_USER_GEOMETRY (1ULL << 21) ++#define DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY (1ULL << 22) ++#define DRM_G5P_CAP_TYPED_RENDER_V2 (1ULL << 23) ++/* Bit 24 is reserved for COMPUTE_BLIT_V1 and bit 25 is FIXED_DATA_B1 on ++ * a9/release, so the next bit free in every branch is 26. ++ */ ++#define DRM_G5P_CAP_TYPED_RENDER_V3 (1ULL << 26) ++ ++/** ++ * DRM_G5P_CAP_RENDER_MULTI_CONTEXT - several render contexts share the GPU ++ * ++ * Reported together with TYPED_RENDER_V3. Any number of render contexts, in any ++ * number of files, may submit TYPED_V3 jobs. Each context has its own VM and its ++ * own firmware slot pair, and the jobs of all contexts run one at a time in ++ * submission order. Without this bit, the first context to submit owns the GPU ++ * until reboot and every other context is refused. The other render profiles ++ * stay single-context. ++ */ ++#define DRM_G5P_CAP_RENDER_MULTI_CONTEXT (1ULL << 27) ++ ++/** ++ * DRM_G5P_CAP_RENDER_UNPRIVILEGED - TYPED_V3 render needs no CAP_SYS_RAWIO ++ * ++ * Reported together with RENDER_MULTI_CONTEXT. Any opener of the render node may ++ * create render contexts and submit TYPED_V3 jobs. A render VM then maps none of ++ * the firmware's memory beyond the kernel-built window the GPU's own requesters ++ * need, and the contexts close cleanly: a closed context's firmware binding is ++ * removed and its memory freed. Pinned memory is limited device-wide. The other ++ * render profiles, compute and the diagnostic ioctls still need CAP_SYS_RAWIO. ++ */ ++#define DRM_G5P_CAP_RENDER_UNPRIVILEGED (1ULL << 28) ++ ++/** ++ * DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH - TYPED_V3 carries pixel_output_width ++ * ++ * Reported together with TYPED_RENDER_V3. struct drm_g5p_render_pass_v3 has ++ * pixel_output_width, so a TYPED_V3 pass in the V1 dialect (all-zero ext) can ++ * write several colour targets. Without this bit the field is reserved and ++ * must be zero. ++ */ ++#define DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH (1ULL << 29) ++ ++/* drm_g5p_create_context.type; flags must be zero. */ ++#define DRM_G5P_CONTEXT_RENDER 1 ++ ++#define DRM_G5P_RENDER_PROFILE_TYPED_V3 4 ++ ++/* LINEAR64_V1: kind is also the resource's position in the canonical array. */ ++ ++#define DRM_G5P_RENDER_ACCESS_READ (1U << 0) ++#define DRM_G5P_RENDER_ACCESS_WRITE (1U << 1) ++#define DRM_G5P_RENDER_ACCESS_IMPLICIT (1U << 2) ++#define DRM_G5P_RENDER_ACCESS_CACHE (1U << 3) ++ ++#define DRM_G5P_RENDER_MAX_RESOURCES 256 ++#define DRM_G5P_RENDER_MAX_SYNCS 17 ++#define DRM_G5P_RENDER_MAX_VDM_BYTES 0x10000 ++ ++/* A GPU VA range [va, va + size) inside one existing mapping of handle's BO ++ * in the context VM, at BO byte offset `offset`. Never a CPU pointer, PA or ++ * META address. LINEAR64_V1 requires offset 0 and the exact whole BO. ++ */ ++struct drm_g5p_render_resource { ++ __u32 handle; ++ __u32 kind; ++ __u64 offset; ++ __u64 size; ++ __u64 va; ++}; ++ ++/** ++ * DRM_G5P_CAP_RENDER_MSAA - 4x multisampled render targets and passes ++ * ++ * Reported with TYPED_RENDER_V3 and RENDER_KERNEL_GEOMETRY. ++ * CREATE_RENDER_TARGET accepts samples = 4 (kernel-derived geometry; with ++ * DRM_G5P_RT_USER_GEOMETRY userspace may only restate it), and a TYPED_V3 ++ * pass carries samples, msaa_flags and sample_positions in struct ++ * drm_g5p_render_pass. Without this bit those fields are reserved. ++ */ ++#define DRM_G5P_CAP_RENDER_MSAA (1ULL << 31) ++ ++/** ++ * DRM_G5P_CAP_RENDER_OCCLUSION_QUERY - TYPED_V3 carries ISP query storage ++ * ++ * The V2 extension names the query-result allocation used by ISP visibility ++ * tests. The kernel validates the complete range and writes its base into the ++ * two per-partition ISP_OCLQRY_BASE work-record slots. A zero VA and size ++ * disable visibility-result storage. ++ */ ++#define DRM_G5P_CAP_RENDER_OCCLUSION_QUERY (1ULL << 33) ++ ++/* drm_g5p_render_pass.msaa_flags: the end of tile averages each pixel's ++ * samples into the (one-sample) surface the EOT's PBE words name. ++ */ ++#define DRM_G5P_RENDER_MSAA_RESOLVE (1U << 0) ++ ++struct drm_g5p_render_pass { ++ __u32 size; /* sizeof(struct drm_g5p_render_pass) */ ++ __u32 flags; /* zero */ ++ __u64 vdm_va; /* VDM control stream, READ, 4-byte aligned */ ++ __u32 vdm_size; /* bytes, multiple of 4, <= MAX_VDM_BYTES */ ++ __u32 vdm_control; /* V1/V2: TA W+0x88 value, multiple of 16; V3: zero */ ++ __u64 scratch_va; /* TA scratch block, WRITE, 16-byte aligned */ ++ __u64 scratch_size; ++ __u64 scratch_ptr[6]; /* TA scratch/marker pointers, in a WRITE range */ ++ __u64 pool7_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool7_size; ++ __u64 pool8_va; /* 3D allocator pool, WRITE, page aligned */ ++ __u64 pool8_size; ++ __u32 context_pds[2][6]; /* TA context store/load PDS descriptors */ ++ __u32 resume_pds[2][4]; /* TA resume PDS descriptors */ ++ __u32 ta_pds[4]; /* TA state PDS descriptor; word 3 zero */ ++ __u32 background[8]; /* 3D background (load/clear) object */ ++ __u32 eot[4]; /* 3D end-of-tile object; word 3 zero */ ++ __u32 eot_resume[4]; /* 3D partial-render EOT; word 3 zero */ ++ __u32 restore[4]; /* 3D partial-render restore words */ ++ /* DRM_G5P_CAP_RENDER_MSAA, TYPED_V3 only; zero in every other profile. ++ * samples: 0 or 1 with a one-sample target, 4 with a four-sample one. ++ * sample_positions: one byte per sample, x in [3:0] and y in [7:4] in ++ * 1/16 px (PPP_MULTISAMPLECTL); 0 takes the target's standard pattern. ++ * The kernel writes the AA mode, ISP_CTL and output-partition words ++ * for the sample count. The draws' ISPCTL.upass (0x7 at four samples, ++ * 0xf at one), the PBE/EOT words and the fragment programs are ++ * userspace's, as for one sample; see docs/gpu/experiments/msaa.md. ++ */ ++ __u32 samples; ++ __u32 msaa_flags; /* DRM_G5P_RENDER_MSAA_* */ ++ __u64 sample_positions; ++ __u64 reserved[6]; ++}; ++ ++/* ++ * TYPED_V2 additions. Each field is a 3D register image that the firmware ++ * uploads from the normal-3D work object. The CR offsets are the firmware ++ * store targets (its 3D upload puts W3c0 on ROGUE_CR_ISP_CTL 0xf38); the ++ * Rogue names are inferred, graded in docs/gpu/render-uapi.md. ++ */ ++#define DRM_G5P_RENDER_MAX_TABLE_ENTRIES 65536 /* ISPDBSC 16-bit indices */ ++#define DRM_G5P_RENDER_ZLS_REGS 6 ++ ++struct drm_g5p_render_pass_ext { ++ /* ISP_SCISSOR_BASE (CR fb0): IPF_SCISSOR_WORD_0/1 pairs, 8 bytes each, ++ * indexed by TA_STATE_ISPDBSC.scindex. READ, 4-byte aligned. ++ */ ++ __u64 scissor_va; ++ __u32 scissor_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ __u32 dbias_count; /* entries, 1..MAX_TABLE_ENTRIES */ ++ /* ISP_DBIAS_BASE (CR fb8): {constant, slope, clamp} f32 triples, ++ * 12 bytes each, indexed by ISPDBSC.dbindex. READ, 4-byte aligned. ++ */ ++ __u64 dbias_va; ++ /* PDS_BGRND1_BASE (CR 6a8): background object texture/varying data ++ * (B-relative, like background[]); LOAD_OP_LOAD's texture state. ++ */ ++ __u64 background_texture; ++ __u32 bgobj_depth; /* ISP_BGOBJDEPTH (CR f80): depth clear, ZLS format */ ++ __u32 bgobj_vals; /* ISP_BGOBJVALS (CR f88): stencil [7:0], mask [8], bgtag [9] */ ++ /* Pixel output width (2, 4 or 8): the 6ca7ae0 inputs to ISP_CTL's ++ * partition fields and the six per-core output words. 2 for one ++ * 32-bpp colour attachment (the captured value). ++ */ ++ __u32 pixel_output_width; ++ /* Bit i: zls[i] is an address ([39:4]; [63:40] zero; [3:0] flags) ++ * inside a listed READ|WRITE range. The kernel does not decode ZLS. ++ */ ++ __u32 zls_address_mask; ++ /* Opaque ZLS register images, CR f48, f50, f58, f60, f68 and fd0: ++ * Rogue ISP_ZLSCTL, ZLOAD/ZSTORE and STENCIL_LOAD/STORE bases plus ++ * one more; positions proven, order unqualified. Zero: no ZLS. ++ */ ++ __u64 zls[DRM_G5P_RENDER_ZLS_REGS]; ++ __u64 occlusion_query_va; /* ISP_OCLQRY_BASE, 16-byte aligned, READ|WRITE */ ++ __u64 occlusion_query_size; /* bytes in the listed allocation */ ++ __u64 reserved[2]; ++}; ++ ++struct drm_g5p_render_pass_v2 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++}; ++ ++#define DRM_G5P_RENDER_MAX_PROGRAM_BYTES 0x1000 ++#define DRM_G5P_RENDER_STREAM_TERMINATE 0xc0000000U ++ ++struct drm_g5p_render_pass_v3 { ++ struct drm_g5p_render_pass base; ++ struct drm_g5p_render_pass_ext ext; ++ __u64 ta_program_va; /* -> W+0x88 / R0+0x470 as an offset from pds_base */ ++ __u32 ta_program_size; /* bytes, 16..MAX_PROGRAM_BYTES, multiple of 16 */ ++ __u32 vdm_end; /* bytes from vdm_va past STREAM_TERMINATE, <= vdm_size */ ++ __u32 pixel_output_width; /* V1 dialect: 0 (= 2), 2, 4 or 8 */ ++ __u32 pad; /* zero */ ++ __u64 reserved[3]; ++}; ++ ++struct drm_g5p_submit_render_typed { ++ __u32 context; ++ __u32 flags; /* zero */ ++ __u32 profile; ++ __u32 render_target; /* handle from CREATE_RENDER_TARGET */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[] */ ++ __u64 pass; /* struct drm_g5p_render_pass */ ++ __u32 pass_size; ++ __u32 pad; ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; ++}; ++ ++/* ++ * Render-target memory geometry: the AGXRenderTargetMemoryG5 arithmetic ++ * outputs (firmware round16 section 3, R-offsets in comments). The kernel ++ * computes them for its verified table (64x64x1), or with ++ * RENDER_KERNEL_GEOMETRY for any size; with RENDER_USER_GEOMETRY userspace ++ * supplies them and the kernel bounds the memory they imply (and, with ++ * RENDER_KERNEL_GEOMETRY, requires its own result). ++ */ ++struct drm_g5p_rt_geometry { ++ __u32 header_bytes; /* R10 */ ++ __u32 region_bytes; /* R20 */ ++ __u64 packed_state; /* Rb0 */ ++ __u32 state_word; /* Rb8 */ ++ __u32 padded_dims; /* Rbc */ ++ __u64 scale[3]; /* Rc0, Rc8, Rd0 packed float pairs */ ++ __u64 ta_state[2]; /* Rf0, Rf8 */ ++ __u64 ta_tiling[2]; /* R100, R108 */ ++ __u32 ta_tiling_word; /* R110 */ ++ __u32 tile_count; /* R118 */ ++ __u64 secondary_bytes; /* R138 */ ++ __u64 secondary_align; /* R178 */ ++ __u64 reserved[2]; ++}; ++ ++#define DRM_G5P_RT_USER_GEOMETRY (1U << 0) ++struct drm_g5p_create_render_target { ++ __u32 context; /* render context; RT memory lives in its VM */ ++ __u32 flags; ++ __u32 width; ++ __u32 height; ++ __u32 samples; /* 1, or 4 with DRM_G5P_CAP_RENDER_MSAA */ ++ __u32 layers; /* 1 */ ++ __u32 handle; /* output */ ++ __u32 pad; ++ __u64 main_va; /* output: kernel-private GPU VA */ ++ __u64 main_bytes; /* output */ ++ __u64 secondary_va; /* output */ ++ __u64 secondary_bytes; /* output: mapped bytes */ ++ struct drm_g5p_rt_geometry geometry; /* input if USER, else output */ ++ __u32 pm_blocks; /* output with RENDER_KERNEL_GEOMETRY: context PM pool */ ++ __u32 pad2; ++ __u64 reserved; ++}; ++ ++struct drm_g5p_destroy_render_target { ++ __u32 handle; ++ __u32 flags; ++}; ++ ++/* GET_RENDER_INFO.flags, output only (input must be zero). ++ * VARIANT_ACTIVE: a render_user_* hypothesis knob (stage, cache/mmu/timestamp ++ * variant, 3d_after_ta, pm_policy, cache_repeat, wait_ms, pm/memory/rt/slot ++ * variant, profile1_vdm_control) differs from its default, so a context ++ * created now latches a non-default job behaviour. GET_RENDER_INFO has no ++ * context: it reports what CREATE_CONTEXT would latch at this moment. ++ */ ++#define DRM_G5P_RENDER_INFO_VARIANT_ACTIVE (1U << 0) ++ ++struct drm_g5p_get_render_info { ++ __u32 size; /* sizeof, exact */ ++ __u32 flags; /* in: zero; out: RENDER_INFO_* */ ++ __u32 profiles; /* bit N: profile N accepted */ ++ __u32 max_render_targets; ++ __u64 private_va_start; /* kernel window in render VMs */ ++ __u64 private_va_end; ++ __u64 user_va_min; /* lowest mappable VA in a render VM */ ++ __u64 usc_base; /* common USC/PDS base, 32-bit offsets */ ++ __u64 pds_base; ++ __u64 code_window; /* bytes above usc_base usable for code */ ++ __u32 pm_blocks; /* parameter buffer blocks per context */ ++ __u32 pm_block_bytes; ++ __u32 max_queued_jobs; ++ __u32 max_vdm_bytes; ++ __u32 job_timeout_ms; ++ __u32 retained_results; ++ /* RENDER_KERNEL_GEOMETRY only, else zero (64x64 only). */ ++ __u32 max_render_width; ++ __u32 max_render_height; ++ __u32 max_pm_blocks; /* PM pool growth bound per context */ ++ __u32 pad; ++ union { ++ __u64 reserved[2]; /* in: zero */ ++ struct { ++ /* TYPED_V3: every GPU VA the TA/3D read ends at or below ++ * this; 0 makes no statement. Out only. ++ */ ++ __u64 hw_va_limit; ++ __u64 reserved1; ++ }; ++ }; ++}; ++ ++#define DRM_G5P_CREATE_RENDER_TARGET 0x11 ++#define DRM_G5P_DESTROY_RENDER_TARGET 0x12 ++#define DRM_G5P_GET_RENDER_INFO 0x13 ++#define DRM_G5P_SUBMIT_RENDER_TYPED 0x14 ++ ++#define DRM_IOCTL_G5P_SUBMIT_RENDER_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_RENDER_TYPED, \ ++ struct drm_g5p_submit_render_typed) ++#define DRM_IOCTL_G5P_CREATE_RENDER_TARGET \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_CREATE_RENDER_TARGET, \ ++ struct drm_g5p_create_render_target) ++#define DRM_IOCTL_G5P_DESTROY_RENDER_TARGET \ ++ DRM_IOW(DRM_COMMAND_BASE + DRM_G5P_DESTROY_RENDER_TARGET, \ ++ struct drm_g5p_destroy_render_target) ++#define DRM_IOCTL_G5P_GET_RENDER_INFO \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_GET_RENDER_INFO, \ ++ struct drm_g5p_get_render_info) ++ ++/** ++ * DRM_G5P_CAP_TYPED_COMPUTE - compute jobs on render contexts (SUBMIT_COMPUTE_TYPED) ++ * ++ * Reported together with TYPED_RENDER_V3. A ++ * render context (DRM_G5P_CONTEXT_RENDER) runs compute jobs beside its TYPED_V3 render jobs, ++ * in the same VM, in submission order, serialized with every other context's jobs. The ++ * kernel builds the firmware's compute work (W, microsequence, stamps, channel, slot); the ++ * job carries the CDM control stream and the register images iOS's userspace hands its ++ * kernel, which the kernel bounds but does not decode. Compute needs an epoch a render job ++ * brought up: on a device no render job has started, SUBMIT_COMPUTE_TYPED fails with EAGAIN. ++ */ ++#define DRM_G5P_CAP_TYPED_COMPUTE (1ULL << 30) ++ ++#define DRM_G5P_COMPUTE_MAX_STREAM_BYTES 0x10000 ++#define DRM_G5P_COMPUTE_STREAM_TERMINATE 0x80000000U ++ ++/* A PDS program reference: code and data offsets from GET_RENDER_INFO.pds_base (16-byte ++ * aligned; code in a READ resource whose BO carries DRM_G5P_BO_EXEC, data in a READ resource) ++ * and its opaque control word (Rogue's PDS1 position: sizes and temps). ++ */ ++struct drm_g5p_compute_pds { ++ __u32 code; ++ __u32 data; ++ __u32 control; ++ __u32 pad; /* zero */ ++}; ++ ++/* ++ * One compute job. Every *_va is a GPU VA inside a listed resource with the stated access. ++ * cdm_va/cdm_size the CDM control stream (W+0x70, R0+0x480): READ, 4-byte aligned, ++ * cdm_size <= MAX_STREAM_BYTES, in a native (not PRIME-imported) BO ++ * cdm_end bytes from cdm_va past the stream's terminate word (the dword at ++ * cdm_va + cdm_end - 4 is STREAM_TERMINATE), <= cdm_size ++ * state_va[4] CDM context-state words (W+0x1ec..0x204), READ|WRITE, 4-byte aligned ++ * state_tail_va CDM context-state tail (W+0x60, R0+0x498; W+0x20c), READ|WRITE, 16 bytes, ++ * 16-byte aligned ++ * config[2] CDM configuration words (W+0x10/0x14, R0+0x4c0/0x4c8), opaque ++ * store context-store program (W+0x21c/0x224, R0+0x4a8/0x4b0) ++ * resume context-resume program (W+0x22c/0x234) ++ * terminate terminate program (W+0x68, R0+0x4b8); control must be zero ++ */ ++struct drm_g5p_compute_pass { ++ __u32 size; /* sizeof(struct drm_g5p_compute_pass) */ ++ __u32 flags; /* zero */ ++ __u64 cdm_va; ++ __u32 cdm_size; ++ __u32 cdm_end; ++ __u64 state_va[4]; ++ __u64 state_tail_va; ++ __u32 config[2]; ++ struct drm_g5p_compute_pds store; ++ struct drm_g5p_compute_pds resume; ++ struct drm_g5p_compute_pds terminate; ++ __u64 reserved[4]; /* zero */ ++}; ++ ++/* Command 0x15. resources lists every BO the job references (1..RENDER_MAX_RESOURCES, ++ * TYPED_V1 access-mask kinds, IMPLICIT and CACHE included); pass_size is exactly ++ * sizeof(struct drm_g5p_compute_pass). Syncs, job ids, results and the error contract are ++ * SUBMIT_RENDER_TYPED's: every error other than EINTR/EFAULT on copy-out precedes fence ++ * replacement and hardware publication. ++ */ ++struct drm_g5p_submit_compute_typed { ++ __u32 context; /* a DRM_G5P_CONTEXT_RENDER context */ ++ __u32 flags; /* zero */ ++ __u64 resources; /* struct drm_g5p_render_resource[] */ ++ __u32 resource_count; ++ __u32 sync_count; ++ __u64 syncs; /* struct drm_g5p_sync[]: exactly one SIGNAL, at most 16 inputs */ ++ __u64 pass; /* struct drm_g5p_compute_pass */ ++ __u32 pass_size; ++ __u32 pad; /* zero */ ++ __u64 job_id; /* output */ ++ __u64 reserved[2]; /* zero */ ++}; ++ ++#define DRM_G5P_SUBMIT_COMPUTE_TYPED 0x15 ++ ++#define DRM_IOCTL_G5P_SUBMIT_COMPUTE_TYPED \ ++ DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_COMPUTE_TYPED, \ ++ struct drm_g5p_submit_compute_typed) ++ ++#endif diff --git a/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch b/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch new file mode 100644 index 00000000..5e33a414 --- /dev/null +++ b/kernel/patches/drivers/drm/0011-drm-apple-g5p-add-firmware-transport-and-recovery-co.patch @@ -0,0 +1,14822 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add firmware transport and recovery core + +[Orchard: from brkosebay/linux d67f133590a; fixed default policy and +removed idle experiments.] Load the original user-supplied META firmware +and retain bounded recovery. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/bits-compat.h | 20 + + drivers/gpu/apple-g5p/boot-layout.h | 306 ++++ + drivers/gpu/apple-g5p/boot-r73b.inc | 1179 +++++++++++++ + drivers/gpu/apple-g5p/boot-r74f.inc | 532 ++++++ + drivers/gpu/apple-g5p/boot-r74g.inc | 288 +++ + drivers/gpu/apple-g5p/boot.inc | 1665 ++++++++++++++++++ + drivers/gpu/apple-g5p/compute-user-layout.h | 207 +++ + drivers/gpu/apple-g5p/core.inc | 567 ++++++ + drivers/gpu/apple-g5p/diagnostics.inc | 181 ++ + drivers/gpu/apple-g5p/dirtystart108.h | 454 +++++ + drivers/gpu/apple-g5p/dirtystart108.inc | 280 +++ + drivers/gpu/apple-g5p/drm-validate.h | 40 + + drivers/gpu/apple-g5p/fender.h | 87 + + drivers/gpu/apple-g5p/gpu-oc.h | 149 ++ + drivers/gpu/apple-g5p/heartbeat116.h | 175 ++ + drivers/gpu/apple-g5p/idle-power.inc | 38 + + drivers/gpu/apple-g5p/kick.inc | 27 + + drivers/gpu/apple-g5p/latency123.h | 63 + + drivers/gpu/apple-g5p/ldr.h | 154 ++ + drivers/gpu/apple-g5p/lifecycle.inc | 1032 +++++++++++ + drivers/gpu/apple-g5p/meta71c114.h | 65 + + drivers/gpu/apple-g5p/mmio-licence.h | 148 ++ + drivers/gpu/apple-g5p/mmio.inc | 295 ++++ + drivers/gpu/apple-g5p/mmu-attrs.inc | 54 + + drivers/gpu/apple-g5p/mmu-tables.inc | 57 + + drivers/gpu/apple-g5p/pack.h | 89 + + drivers/gpu/apple-g5p/perf-envelope.h | 107 ++ + drivers/gpu/apple-g5p/power-r33.inc | 70 + + drivers/gpu/apple-g5p/power-r41.inc | 135 ++ + drivers/gpu/apple-g5p/r74e112.h | 137 ++ + drivers/gpu/apple-g5p/r87b-attach.h | 34 + + drivers/gpu/apple-g5p/readback.h | 40 + + drivers/gpu/apple-g5p/recover.inc | 101 ++ + drivers/gpu/apple-g5p/render-layout-r22.h | 54 + + drivers/gpu/apple-g5p/render-layout.h | 59 + + drivers/gpu/apple-g5p/render-user-bind.h | 285 +++ + drivers/gpu/apple-g5p/render-user-close.h | 91 + + drivers/gpu/apple-g5p/render-user-evwait.h | 52 + + drivers/gpu/apple-g5p/render-user-fault.h | 60 + + drivers/gpu/apple-g5p/render-user-faultreg.h | 124 ++ + drivers/gpu/apple-g5p/render-user-geometry.h | 144 ++ + drivers/gpu/apple-g5p/render-user-layout.h | 651 +++++++ + drivers/gpu/apple-g5p/render-user-loss.h | 120 ++ + drivers/gpu/apple-g5p/render-user-mmu.h | 60 + + drivers/gpu/apple-g5p/render-user-msaa.h | 88 + + drivers/gpu/apple-g5p/render-user-pmset.h | 77 + + drivers/gpu/apple-g5p/render-user-quiesce.h | 145 ++ + drivers/gpu/apple-g5p/render-user-recover.h | 107 ++ + drivers/gpu/apple-g5p/render-user-retire.h | 248 +++ + drivers/gpu/apple-g5p/render-user-sched.h | 155 ++ + drivers/gpu/apple-g5p/render-user-serial.h | 122 ++ + drivers/gpu/apple-g5p/render-user-sync.h | 50 + + drivers/gpu/apple-g5p/render-user-template.h | 24 + + drivers/gpu/apple-g5p/render-user-terminal.h | 103 ++ + drivers/gpu/apple-g5p/render-user-touch.h | 102 ++ + drivers/gpu/apple-g5p/render-user-wait.h | 266 +++ + drivers/gpu/apple-g5p/render-user-wake.h | 273 +++ + drivers/gpu/apple-g5p/render-user-window.h | 74 + + drivers/gpu/apple-g5p/render-variants-r25.h | 48 + + drivers/gpu/apple-g5p/retire108.h | 434 +++++ + drivers/gpu/apple-g5p/teardown.h | 740 ++++++++ + drivers/gpu/apple-g5p/teardown.inc | 694 ++++++++ + drivers/gpu/apple-g5p/transport-budget.inc | 6 + + drivers/gpu/apple-g5p/vm-mutation.h | 36 + + drivers/gpu/apple-g5p/work-layout.h | 21 + + 65 files changed, 14289 insertions(+) + create mode 100644 drivers/gpu/apple-g5p/bits-compat.h + create mode 100644 drivers/gpu/apple-g5p/boot-layout.h + create mode 100644 drivers/gpu/apple-g5p/boot-r73b.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74f.inc + create mode 100644 drivers/gpu/apple-g5p/boot-r74g.inc + create mode 100644 drivers/gpu/apple-g5p/boot.inc + create mode 100644 drivers/gpu/apple-g5p/compute-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/core.inc + create mode 100644 drivers/gpu/apple-g5p/diagnostics.inc + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.h + create mode 100644 drivers/gpu/apple-g5p/dirtystart108.inc + create mode 100644 drivers/gpu/apple-g5p/drm-validate.h + create mode 100644 drivers/gpu/apple-g5p/fender.h + create mode 100644 drivers/gpu/apple-g5p/gpu-oc.h + create mode 100644 drivers/gpu/apple-g5p/heartbeat116.h + create mode 100644 drivers/gpu/apple-g5p/idle-power.inc + create mode 100644 drivers/gpu/apple-g5p/kick.inc + create mode 100644 drivers/gpu/apple-g5p/latency123.h + create mode 100644 drivers/gpu/apple-g5p/ldr.h + create mode 100644 drivers/gpu/apple-g5p/lifecycle.inc + create mode 100644 drivers/gpu/apple-g5p/meta71c114.h + create mode 100644 drivers/gpu/apple-g5p/mmio-licence.h + create mode 100644 drivers/gpu/apple-g5p/mmio.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-attrs.inc + create mode 100644 drivers/gpu/apple-g5p/mmu-tables.inc + create mode 100644 drivers/gpu/apple-g5p/pack.h + create mode 100644 drivers/gpu/apple-g5p/perf-envelope.h + create mode 100644 drivers/gpu/apple-g5p/power-r33.inc + create mode 100644 drivers/gpu/apple-g5p/power-r41.inc + create mode 100644 drivers/gpu/apple-g5p/r74e112.h + create mode 100644 drivers/gpu/apple-g5p/r87b-attach.h + create mode 100644 drivers/gpu/apple-g5p/readback.h + create mode 100644 drivers/gpu/apple-g5p/recover.inc + create mode 100644 drivers/gpu/apple-g5p/render-layout-r22.h + create mode 100644 drivers/gpu/apple-g5p/render-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-bind.h + create mode 100644 drivers/gpu/apple-g5p/render-user-close.h + create mode 100644 drivers/gpu/apple-g5p/render-user-evwait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-fault.h + create mode 100644 drivers/gpu/apple-g5p/render-user-faultreg.h + create mode 100644 drivers/gpu/apple-g5p/render-user-geometry.h + create mode 100644 drivers/gpu/apple-g5p/render-user-layout.h + create mode 100644 drivers/gpu/apple-g5p/render-user-loss.h + create mode 100644 drivers/gpu/apple-g5p/render-user-mmu.h + create mode 100644 drivers/gpu/apple-g5p/render-user-msaa.h + create mode 100644 drivers/gpu/apple-g5p/render-user-pmset.h + create mode 100644 drivers/gpu/apple-g5p/render-user-quiesce.h + create mode 100644 drivers/gpu/apple-g5p/render-user-recover.h + create mode 100644 drivers/gpu/apple-g5p/render-user-retire.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sched.h + create mode 100644 drivers/gpu/apple-g5p/render-user-serial.h + create mode 100644 drivers/gpu/apple-g5p/render-user-sync.h + create mode 100644 drivers/gpu/apple-g5p/render-user-template.h + create mode 100644 drivers/gpu/apple-g5p/render-user-terminal.h + create mode 100644 drivers/gpu/apple-g5p/render-user-touch.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wait.h + create mode 100644 drivers/gpu/apple-g5p/render-user-wake.h + create mode 100644 drivers/gpu/apple-g5p/render-user-window.h + create mode 100644 drivers/gpu/apple-g5p/render-variants-r25.h + create mode 100644 drivers/gpu/apple-g5p/retire108.h + create mode 100644 drivers/gpu/apple-g5p/teardown.h + create mode 100644 drivers/gpu/apple-g5p/teardown.inc + create mode 100644 drivers/gpu/apple-g5p/transport-budget.inc + create mode 100644 drivers/gpu/apple-g5p/vm-mutation.h + create mode 100644 drivers/gpu/apple-g5p/work-layout.h + +diff --git a/drivers/gpu/apple-g5p/bits-compat.h b/drivers/gpu/apple-g5p/bits-compat.h +new file mode 100644 +index 000000000..608e9b8ec +--- /dev/null ++++ b/drivers/gpu/apple-g5p/bits-compat.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_BITS_COMPAT_H ++#define G5P_BITS_COMPAT_H ++ ++#ifdef __KERNEL__ ++#include ++#include ++#else ++#ifndef BIT ++#define BIT(nr) (1UL << (nr)) ++#endif ++#ifndef BIT_ULL ++#define BIT_ULL(nr) (1ULL << (nr)) ++#endif ++#ifndef ARRAY_SIZE ++#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) ++#endif ++#endif ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-layout.h b/drivers/gpu/apple-g5p/boot-layout.h +new file mode 100644 +index 000000000..752e6873f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-layout.h +@@ -0,0 +1,306 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_BOOT_LAYOUT_H ++#define APPLE_G5P_BOOT_LAYOUT_H ++ ++#include "bits-compat.h" ++#include "pack.h" ++ ++#define G5P_GPU_PAGE 0x4000U ++#define G5P_ROOT_SIZE 0x8000U ++#define G5P_SHARED_SIZE 0x60000U ++#define G5P_IMAGE_VA 0xe1c0000000ULL ++#define G5P_SHARED_VA 0xe1c0200000ULL ++#define G5P_ROOT_VA 0xe1c3200000ULL ++#define G5P_PIO_VA 0xe1c4200000ULL ++#define G5P_PIO_SIZE 0x10000U ++ ++/* c95da0 descriptors, sizes at 5940b90; c97b6c packs on 64-byte boundaries. */ ++struct g5p_boot_layout { ++ u32 offset[0x90]; ++ u8 group[0x90]; ++ u32 used[2]; ++}; ++ ++static inline void g5p_put32(u8 *p, u32 value) ++{ ++ p[0] = value; ++ p[1] = value >> 8; ++ p[2] = value >> 16; ++ p[3] = value >> 24; ++} ++ ++static inline void g5p_put64(u8 *p, u64 value) ++{ ++ g5p_put32(p, value); ++ g5p_put32(p + 4, value >> 32); ++} ++ ++static inline u32 g5p_meta_address(u64 va) ++{ ++ /* ca0744: root range 8 bypasses META caches; other shared RAM is global. */ ++ u32 prefix = va >= G5P_ROOT_VA && va < G5P_ROOT_VA + 0x1000000 ? ++ 0x40000000 : ++ 0xc0000000; ++ ++ return ((u32)va + 0x40000000) | prefix; ++} ++ ++static inline int g5p_plan_layout(struct g5p_boot_layout *p) ++{ ++ static const struct { ++ u8 cpu, gpu, group; ++ u32 size; ++ } objects[] = { ++ { 0x44, 0x50, 0, 0x1490 }, ++ { 0x47, 0x51, 0, 0x990 }, ++ { 0x48, 0x52, 0, 0x8e0 }, ++ { 0x49, 0x53, 0, 0x8c0 }, ++ { 0x4a, 0x54, 0, 0x938 }, ++ { 0x4b, 0x55, 0, 0x28 }, ++ { 0x4c, 0x56, 0, 8 }, ++ { 0x4d, 0x57, 0, 0x78c }, ++ { 0x4e, 0x58, 0, 0x84 }, ++ { 0x5e, 0x5f, 1, 0x2058 }, ++ { 0x70, 0x78, 1, 0x20 }, ++ { 0x71, 0x79, 1, 0x80 }, ++ { 0x72, 0x7a, 1, 0x20 }, ++ { 0x73, 0x7b, 1, 0x20 }, ++ { 0x74, 0x7c, 1, 0x2c00 }, ++ { 0x75, 0x7d, 1, 0xb000 }, ++ { 0x4f, 0x59, 1, 0x36000 }, ++ { 0x76, 0x7e, 1, 0x2c00 }, ++ { 0x77, 0x7f, 1, 0x2400 }, ++ { 0x6c, 0x6e, 1, 0x20 }, ++ { 0x6d, 0x6f, 1, 0x1400 }, ++ { 0x60, 0x62, 1, 0x20 }, ++ { 0x64, 0x66, 1, 0x20 }, ++ { 0x68, 0x6a, 1, 0x20 }, ++ { 0x61, 0x63, 1, 0x1400 }, ++ { 0x65, 0x67, 1, 0x1400 }, ++ { 0x69, 0x6b, 1, 0x1400 }, ++ /* Separate c95da0 allocations, placed after the recovered group. */ ++ { 0x5c, 0x5d, 1, 8 }, ++ { 0x85, 0x88, 1, 0x400 }, ++ { 0x84, 0x87, 1, 0x400 }, ++ { 0x86, 0x89, 1, 0x800 }, ++ }; ++ unsigned int i; ++ ++ memset(p, 0, sizeof(*p)); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) { ++ u32 group = objects[i].group; ++ u32 off = (p->used[group] + 63) & ~63U; ++ ++ p->offset[objects[i].cpu] = off; ++ p->offset[objects[i].gpu] = off; ++ p->group[objects[i].cpu] = group; ++ p->group[objects[i].gpu] = group; ++ p->used[group] = off + objects[i].size; ++ } ++ return p->used[0] <= G5P_ROOT_SIZE && p->used[1] <= G5P_SHARED_SIZE ? ++ 0 : ++ -EINVAL; ++} ++ ++static inline u64 g5p_object_va(const struct g5p_boot_layout *p, u32 slot) ++{ ++ return (p->group[slot] ? G5P_SHARED_VA : G5P_ROOT_VA) + p->offset[slot]; ++} ++ ++static inline u8 *g5p_object(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 slot) ++{ ++ return (p->group[slot] ? shared : root) + p->offset[slot]; ++} ++ ++/* ca9b34 + cab44c + cac314: 40-bit PA, 16 KiB GPU pages. */ ++static inline u32 g5p_catalogue_entry(u64 pa) ++{ ++ return (u32)(pa >> 8) | 1; ++} ++ ++static inline u64 g5p_directory_entry(u64 pa) ++{ ++ /* Geometry bytes 3c..3f are 08 05 05 15: size, entry/PA/VA shifts. */ ++ return pa | 3; ++} ++ ++static inline u64 g5p_leaf_entry(u64 pa, u32 attributes) ++{ ++ /* caa6e4 range attributes -> ca9a60: META protect, CC / SLC bypass. */ ++ return (pa & 0xfffffff000ULL) | (attributes & 0xc) | ++ ((u64)(attributes & 0x10) << 58) | 1; ++} ++ ++static inline void g5p_make_boot_config(struct g5p_ldr_image *im, ++ const struct g5p_boot_layout *p) ++{ ++ static const u32 bases[] = { 0x10000000, 0x20000000, 0x40000000, ++ 0x60001000 }; ++ static const u32 sizes[] = { 0x18000, 0x4000, 0x4000, 0x4000 }; ++ static const u32 offsets[] = { 0x4000, 0x1c000, 0, 0x20000 }; ++ u8 *out = im->data + 0x200; ++ unsigned int seg, i; ++ ++ for (seg = 0; seg < 4; seg++) { ++ u64 va = G5P_IMAGE_VA + offsets[seg]; ++ u32 reg = 0x04850000 + seg * 16; ++ u32 values[] = { bases[seg] | 0xf02, sizes[seg] - 1, (u32)va, ++ (u32)(va >> 32) | 0x300000 }; ++ ++ if (seg == 2) { ++ g5p_put32(out, reg + 12); ++ g5p_put32(out + 4, (u32)(va >> 32) | 0x100); ++ out += 8; ++ continue; ++ } ++ for (i = 0; i < 4; i++, out += 8) { ++ g5p_put32(out, reg + i * 4); ++ g5p_put32(out + 4, values[i]); ++ } ++ } ++ for (i = 0; i < im->config_count; i++, out += 8) { ++ g5p_put32(out, im->config[i][0]); ++ g5p_put32(out + 4, im->config[i][1]); ++ } ++ g5p_put64(out, 0); ++ g5p_put32(out + 8, g5p_meta_address(G5P_IMAGE_VA + 0x3ff8)); ++ g5p_put32(im->data + 0x3ff8, g5p_meta_address(g5p_object_va(p, 0x50))); ++ g5p_put32(im->data + 0x3ffc, g5p_meta_address(g5p_object_va(p, 0x5f))); ++} ++ ++/* c96be8/c9edcc/ca3ccc. Unknown platform policy fields remain zero. */ ++static inline void g5p_make_init(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, const struct g5p_ldr_image *im, ++ u64 catalogue, u64 timestamp) ++{ ++ static const u8 roots[] = { ++ 0x62, 0x63, 0x66, 0x67, 0x6a, 0x6b, 0x6e, 0x6f, ++ 0x78, 0x7c, 0x79, 0x7d, 0x7a, 0x7e, 0x7b, 0x7f, ++ }; ++ static const u8 objects[] = { 0x51, 0x52, 0x53, 0x54, 0x55 }; ++ static const u8 tail[] = { 0x58, 0x56, 0x57, 0x88, 0x59 }; ++ static const u64 pio_pa[] = { 0x201e00000ULL, 0x20e100000ULL, ++ 0x20e104000ULL, 0x201000000ULL }; ++ static const u32 pio_size[] = { 0x5000, 0x4000, 0x4000, 0x20000 }; ++ static const u32 pio_offset[] = { 0, 0x8000, 0xc000, 0 }; ++ u8 *init = g5p_object(p, root, shared, 0x44); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *ctrl = g5p_object(p, root, shared, 0x5e); ++ unsigned int i; ++ ++ memset(root, 0, G5P_ROOT_SIZE); ++ memset(shared, 0, G5P_SHARED_SIZE); ++ for (i = 0; i < ARRAY_SIZE(roots); i++) ++ g5p_put32(init + i * 4, ++ g5p_meta_address(g5p_object_va(p, roots[i]))); ++ for (i = 0; i < ARRAY_SIZE(objects); i++) ++ g5p_put32(init + 0x64 + i * 4, ++ g5p_meta_address(g5p_object_va(p, objects[i]))); ++ for (i = 0; i < ARRAY_SIZE(tail); i++) ++ g5p_put32(init + 0x44c + i * 4, ++ g5p_meta_address(g5p_object_va(p, tail[i]))); ++ for (i = 0; i < 3; i++) { ++ u8 *channel = g5p_object(p, root, shared, 0x47 + i); ++ u32 ptr = g5p_meta_address(g5p_object_va(p, 0x51 + i)); ++ ++ g5p_put32(channel + 4, ptr); ++ g5p_put32(channel + 12, ptr + 8); ++ } ++ g5p_put32(g5p_object(p, root, shared, 0x48) + 0x778, 0xffffffff); ++ g5p_put32(init + 0x480, ++ ((u32)(G5P_IMAGE_VA + 0x24000) + 0x40000000) | 0x40000000); ++ g5p_put64(init + 0x484, G5P_IMAGE_VA + 0x24000); ++ g5p_put32(init + 0x48c, 0x55dc); ++ g5p_put64(meta, 0x600000000ULL); ++ g5p_put64(meta + 8, 0xffc00000); ++ g5p_put64(meta + 0x10, 0x300000000ULL); ++ g5p_put64(meta + 0x18, 0x300000000ULL); ++ for (i = 0; i < ARRAY_SIZE(pio_pa); i++) { ++ u8 *entry = meta + 0x630 + i * 16; ++ ++ g5p_put64(entry, pio_pa[i]); ++ g5p_put32(entry + 8, i == 3 ? 0 : ++ g5p_meta_address(G5P_PIO_VA + ++ pio_offset[i])); ++ g5p_put32(entry + 12, pio_size[i]); ++ } ++ g5p_put32(meta + 0x6c0, 1); ++ g5p_put32(meta + 0x6c8, 1); ++ g5p_put64(meta + 0x6d8, 0x100000000ULL); ++ g5p_put32(meta + 0x6e0, 1); ++ g5p_put32(meta + 0x6e4, 1); ++ /* readChipInfo version 15 -> tables 5942650/5942674 -> 1,5; N=6. */ ++ g5p_put32(meta + 0x704, 1); ++ g5p_put32(meta + 0x708, 5); ++ g5p_put32(meta + 0x70c, 6); ++ g5p_put32(meta + 0x758, 0x14); ++ g5p_put32(meta + 0x75c, 0x4000); ++ g5p_put32(meta + 0x760, 0x1020); ++ g5p_put32(meta + 0x764, im->t1_pc); ++ g5p_put32(meta + 0x768, im->t1_sp); ++ g5p_put64(meta + 0x774, G5P_IMAGE_VA); ++ g5p_put64(meta + 0x77c, 0x300000); ++ g5p_put64(meta + 0x784, timestamp); ++ /* Fixed portion of c974f0; dynamic power-policy fields remain zero. */ ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 8, 0x400000000ULL); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x10, 0x3f800000); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x28, 0x100000001ULL); ++ g5p_put32(g5p_object(p, root, shared, 0x4a) + 0x38, 100); ++ g5p_put64(g5p_object(p, root, shared, 0x4a) + 0x8f8, timestamp); ++ g5p_put32(ctrl + 0x24, 0x78); ++ g5p_put32(ctrl + 0x30, 1); ++ g5p_put32(ctrl + 0x1044, g5p_meta_address(g5p_object_va(p, 0x87))); ++ g5p_put32(ctrl + 0x1048, g5p_meta_address(g5p_object_va(p, 0x89))); ++ g5p_put32(ctrl + 0x104c, 1); ++ g5p_put64(g5p_object(p, root, shared, 0x86), catalogue); ++} ++ ++/* Explicit boot-time hypotheses; never modify a live root table. */ ++static inline void g5p_dc_init_variant(const struct g5p_boot_layout *p, ++ u8 *root, u8 *shared, u32 variant) ++{ ++ /* ca0744 uses 40 aliases for cache bypass. Applying that alias to DC ++ * is an unverified hypothesis, not recovered normal host behavior. ++ */ ++ if (variant == 1) ++ g5p_put32(root + 0x18, g5p_le32(root + 0x18) & ~0x80000000U); ++ if (variant == 1 || variant == 6) ++ g5p_put32(root + 0x1c, g5p_le32(root + 0x1c) & ~0x80000000U); ++ /* c96be8 chooses this policy from host+1af8 bits; both values exist. ++ * Whether zero is appropriate for our incomplete policy is unknown. ++ */ ++ if (variant == 5) ++ g5p_put32(shared + p->offset[0x5e] + 0x30, 0); ++ /* MINIM 10001ac8 enters policy1000c0d8 only when meta6c0/6c8 ++ * are both nonzero. Experimental pre-MTS bypass; no live length writes. ++ */ ++ if (variant == 7) ++ g5p_put32(g5p_object(p, root, shared, 0x4d) + 0x6c8, 0); ++ ++ /* Round13: recovered iOS inputs; variants 8/9 retain meta6c8=1. ++ * Neither is a complete DVFS policy (table+3c remains absent). ++ */ ++ if (variant == 8 || variant == 9) { ++ u8 *policy = g5p_object(p, root, shared, 0x4a); ++ u8 *meta = g5p_object(p, root, shared, 0x4d); ++ u8 *control = g5p_object(p, root, shared, 0x5e); ++ static const u32 mhz[] = { 0, 340, 474, 550, 723 }; ++ ++ g5p_put32(policy + 4, 16); ++ if (variant == 9) { ++ g5p_put32(meta + 0x710, 4); ++ g5p_put32(meta + 0x714, 5); ++ for (u32 i = 0; i < 16; i++) ++ g5p_put32(meta + 0x718 + 4 * i, ++ i < ARRAY_SIZE(mhz) ? mhz[i] : 0); ++ g5p_put32(control + 0x1028, 3348); ++ g5p_put32(control + 0x102c, 400); ++ g5p_put32(control + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++ } ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +new file mode 100644 +index 000000000..df2560f5e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -0,0 +1,1179 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R73b crash-first IRQ174 diagnostics. Included by boot.inc after struct ++ * g5p_boot. Case zero allocates nothing and keeps the r73 IRQ order. ++ * Rows live in CPU RAM only; they are formatted after the epoch, never in the ++ * IRQ service loop. The first 128 rows are retained, the last 128 roll. ++ */ ++#include ++#include ++#include ++#include ++#include "r87b-attach.h" ++#include "r74e112.h" ++#include "heartbeat116.h" ++ ++#define G5P_R73B_ROWS 256 ++#define G5P_R73B_HEAD 128 ++#define G5P_R73B_IRQ_CAP 8192 ++#define G5P_R73B_PANIC_ROWS 32 ++#define G5P_R73B_TRACE_ROWS 128 ++#define G5P_R73B_TRACE_WORDS 9 ++#define G5P_R74_IRQ_CAP 262144 ++#define G5P_R74_IRQ_CAP_MAX 1048576 ++#define G5P_R74_EVENT_FLOOR (256 + 4 * 65536) ++#define G5P_R74_STAGES 8 ++#define G5P_R74_MILESTONES 64 ++#define G5P_R74_MAX_TICKS 16 ++#define G5P_R74_RGX 9 ++#define G5P_R74_WORD1 8 ++#define G5P_R74F_KINDS 3 ++#define G5P_R74F_OFF_US 1000 ++#define G5P_R74F_ON_US 1800 ++#define G5P_R74F_BELL_MS 250 ++#define G5P_R74F_MMU_SLOTS 2 ++#define G5P_R74F_MMU_US 1000 ++#define G5P_R74G_GRACE_ALIVES 4 ++#define G5P_R74G_GRACE_US 20000 ++#define G5P_R74G_COMPANION 0x3980 ++#define G5P_R74G_RECOVERIES 0x3994 ++#define G5P_R74G_CLASS 0x3998 ++#define G5P_R74G_BAD_RING BIT(0) ++#define G5P_R74G_BAD_RECORD BIT(1) ++#define G5P_R74G_BAD_PA BIT(2) ++#define G5P_R74G_BAD_HOST BIT(3) ++#define G5P_R74G_BAD_CACHED BIT(4) ++#define G5P_R74G_BAD_FAILED BIT(5) ++#define G5P_R74G_BAD_COMPANION BIT(6) ++#define G5P_R74G_BAD_MUTATION BIT(7) ++#define G5P_R74H_TAIL 128 ++#define G5P_R74H_PHYS_MAX 1024 ++#define G5P_R74H_PTES 10 ++ ++/* First-ALIVE submit: wait (reads only) for a GPC request in flight to finish. */ ++static const unsigned int boot_alive_settle_us = 2000; ++MODULE_PARM_DESC( ++ boot_alive_settle_us, ++ "default0 (single read, as before); 1..5000: when the first-ALIVE submit reads " ++ "R1+0x1000 with bit 4 set (a GPC performance-state request in flight), re-read " ++ "it every 10 us for up to this long before the unchanged state==1 check, as " ++ "Apple's setGPCPerformanceState waits for bit 4; R1 reads only; latched at map"); ++ ++enum g5p_r73b_phase { ++ G5P_R73B_RELEASE_ENTRY = 1, ++ G5P_R73B_RELEASE_FIFO, ++ G5P_R73B_RELEASE_GPC, ++ G5P_R73B_RELEASE_EXIT, ++ G5P_R73B_IRQ, ++ G5P_R73B_DRAIN_BEFORE, ++ G5P_R73B_DRAIN_AFTER, ++ G5P_R73B_DRAIN_BUSY, ++ G5P_R73B_CAP, ++ G5P_R73B_EVENTS_EXIT, ++ G5P_R73B_DC_PUBLISHED, ++ G5P_R73B_DC_KICKED, ++ G5P_R73B_WAIT_EXIT, ++ G5P_R73B_EXPORT, ++ G5P_R73B_PHASES, ++}; ++ ++static const char *const g5p_r73b_phase_names[G5P_R73B_PHASES] = { ++ "unknown", "release-entry", "release-fifo", "release-gpc", ++ "release-exit", "irq", "drain-before", "drain-after", ++ "drain-busy", "cap", "events-exit", "dc-published", ++ "dc-kicked", "wait-exit", "export", ++}; ++ ++enum g5p_r73b_owner { ++ G5P_R73B_NONE, ++ G5P_R73B_OWNER_IRQ, ++ G5P_R73B_OWNER_EVENTS, ++ G5P_R73B_OWNER_PREPARE, ++ G5P_R73B_OWNER_PUBLISH, ++ G5P_R73B_OWNER_KICK, ++ G5P_R73B_OWNER_WAIT, ++}; ++ ++/* Row columns: event, log lane 0, statistics, trace, device control. */ ++static const u8 g5p_r73b_row_slots[5] = { 0x70, 0x71, 0x72, 0x73, 0x6c }; ++ ++struct g5p_r73b_row { ++ u32 seq; /* sequence + 1, published last; zero means empty/torn */ ++ u32 phase; ++ u64 t_ns; ++ u32 irq, ack, released, owner, cap_seen; ++ s32 result; ++ u32 cp[5][2]; ++}; ++ ++enum g5p_r74_step { ++ G5P_R74_IRQ_TEST_REFUSED, ++ G5P_R74_PREPARE_BEGIN, ++ G5P_R74_PREPARE_END, ++ G5P_R74_ATTACH_BEGIN, ++ G5P_R74_ATTACH_PUBLISHED, ++ G5P_R74_ATTACH_DOORBELL_BEGIN, ++ G5P_R74_ATTACH_DOORBELL_END, ++ G5P_R74_ATTACH_CONSUMED, ++ G5P_R74_ATTACH_END, ++ G5P_R74_TA_BEGIN, ++ G5P_R74_TA_PUBLISH, ++ G5P_R74_TA_DOORBELL_RETURNED, ++ G5P_R74_OBSERVE, ++ G5P_R74_WAIT_TICK, ++ G5P_R74_WAIT_EXIT, ++ G5P_R74_REPORT_END, ++ G5P_R74_CMD8_CONSUMED, ++ G5P_R74_TA_CONSUMED, ++ G5P_R74_TA_RAM_COMPLETE, ++ G5P_R74_TA_RAM_TIMEOUT, ++ G5P_R74F_ATTACH_ARMED, ++ G5P_R74F_ATTACH_REACTIVATED, ++ G5P_R74F_ATTACH_BELL, ++ G5P_R74F_MMU_CLOSED, ++ G5P_R74F_TA_ARMED, ++ G5P_R74F_TA_REACTIVATED, ++ G5P_R74F_TA_BELL, ++ G5P_R74F_STOP, ++ G5P_R74G_FW_MMU, ++ G5P_R74G_ACK_SKIPPED, ++ G5P_R74G_TA_GRACE, ++ G5P_R74G_TA_FALLBACK, ++ G5P_R74_STEPS, ++}; ++ ++static const char *const g5p_r74_steps[G5P_R74_STEPS] = { ++ "irq-test-refused", ++ "prepare-begin", ++ "prepare-end", ++ "attach-begin", ++ "attach-dc3-published", ++ "attach-doorbell-begin", ++ "attach-doorbell-end", ++ "attach-consumed", ++ "attach-end", ++ "ta-begin", ++ "ta-publish", ++ "ta-doorbell-returned", ++ "observe", ++ "wait-tick", ++ "wait-exit", ++ "report-end", ++ "cmd8-consumed", ++ "ta-consumed", ++ "ta-ram-complete", ++ "ta-ram-timeout", ++ "attach-bell-armed", ++ "attach-reactivated", ++ "attach-bell", ++ "first-mmu-window-closed", ++ "ta-bell-armed", ++ "ta-reactivated", ++ "ta-bell", ++ "r74f-stop", ++ "firmware-mmu-complete", ++ "r31-ack-skipped", ++ "ta-grace-end", ++ "ta-fallback", ++}; ++ ++/* R74 read-only witnesses; every address is already read elsewhere in-tree. */ ++ ++struct g5p_r74_milestone { ++ u64 t_ns; ++ u32 step, irqs, events; ++ s32 ret; ++ u32 dc[2], ta[2]; ++}; ++ ++enum g5p_r74f_kind { ++ G5P_R74F_DC3, ++ G5P_R74F_TA, ++ G5P_R74F_3D, /* r96: ring 0x64, doorbell 0x11, owned by the ALIVE decoder like the TA's */ ++}; ++ ++/* idle -> published (gate closed) -> open (reactivated) -> firing -> ++ * rang; published/open -> cancelled. FIRING lives under event_lock. ++ */ ++enum g5p_r74f_state { ++ G5P_R74F_IDLE, ++ G5P_R74F_PUBLISHED, ++ G5P_R74F_OPEN, ++ G5P_R74F_FIRING, ++ G5P_R74F_RANG, ++ G5P_R74F_CANCELLED, ++ G5P_R74F_STATES, ++}; ++ ++enum g5p_r74f_cancel { ++ G5P_R74F_CANCEL_NONE, ++ G5P_R74F_CANCEL_REACTIVATION, ++ G5P_R74F_CANCEL_NO_ALIVE, ++ G5P_R74F_CANCEL_CONSUMED, ++ G5P_R74F_CANCEL_PROTOCOL, ++ G5P_R74F_CANCEL_ALIVE_ADVANCED, ++ G5P_R74F_CANCEL_SELF_DRAIN, ++ G5P_R74F_CANCELS, ++}; ++ ++enum g5p_r74f_stop { ++ G5P_R74F_STOP_NONE, ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ G5P_R74F_STOP_MMU_CLOSED, ++ G5P_R74F_STOP_TA_REACTIVATION, ++ G5P_R74F_STOP_TA_NOT_KICKED, ++ G5P_R74G_STOP_DC3_TAGS, ++ G5P_R74G_STOP_DC3_RAM, ++ G5P_R74G_STOP_COMPANION, ++ G5P_R74G_STOP_TA_NOT_CONSUMED, ++ G5P_R74G_STOP_TA_WINDOW_MISSED, ++ G5P_R74F_STOPS, ++}; ++ ++enum g5p_r74f_path { ++ G5P_R74F_PATH_NONE, ++ G5P_R74F_PATH_IMMEDIATE, ++ G5P_R74F_PATH_DEFERRED, ++ G5P_R74F_PATHS, ++}; ++ ++/* One generation. Its owner writes the receipt, then releases state. */ ++struct g5p_r74f_bell { ++ atomic_t state; ++ u32 kind, generation, ring, start, target, path, cancel; ++ u32 arm_alive, open_alive, bell_alive; ++ u32 first_fence, first_state, fence, bell_state; ++ s32 first_ret, bell_ret, power_ret; ++ u32 reactivated, core_fence, gpc_fence, off_raw, on_raw; ++ u32 attempted, kicked, complete; ++ u32 consumer_before, consumer_after, producer_after; ++ u32 alive_done; ++ u64 arm_ns, power_start_ns, off_ns, on_ns, open_ns; ++ u64 bell_start_ns, bell_ns, done_ns; ++}; ++ ++enum g5p_r74g_grace { ++ G5P_R74G_GRACE_NONE, ++ G5P_R74G_GRACE_CONSUMED, ++ G5P_R74G_GRACE_ALIVES_END, ++ G5P_R74G_GRACE_WALL, ++ G5P_R74G_GRACE_FAULT, ++ G5P_R74G_GRACES, ++}; ++ ++/* The RAM-visible results of the firmware DC3 handler: phase 0 at the DC3 ++ * consume, phase 1 again just before the TA producer moves. ++ */ ++struct g5p_r74g_dc3 { ++ u64 pa[2], expected_pa, t_ns; ++ u32 slot[2], host[2], cached[2], rec[2][5]; ++ u32 start, target, consumer, producer, tag; ++ u32 failed, event_failed, companion, recoveries, classification, bad; ++}; ++ ++struct g5p_r73b { ++ struct device *dev; ++ struct g5p_boot *boot; ++ u32 case_id, watchdog_ms, dc_variant, meta6c8, r73e_case; ++ bool ack_first, aux, fast_enabled, fast_window; ++ bool early_release_ready, early_power_abort; ++ atomic_t early_once, early_busy; ++ u64 early_irq_ns, early_power_start_ns; ++ u64 early_off_ns, early_on_ns; ++ u32 early_state, early_after_state, early_power_raw; ++ s32 early_ret, early_power_ret; ++ /* wake112 (r74e112.h): the R74E proof deferred to the first-ALIVE owner. */ ++ atomic_t early_proof; ++ u32 early_proof_gen, early_proof_gen_now, early_proof_why; ++ s32 early_defer_ret; ++ /* boot_alive_settle_us: limit, first in-flight value seen, re-reads, time spent. */ ++ u32 settle_us, settle_first, settle_samples; ++ u64 settle_ns; ++ u64 fast_release_ns, fast_alive_ns, fast_submit_ns; ++ u32 fast_state; ++ s32 fast_ret; ++ /* nopatch: Apple's unmodified image; policy record +0/+4/+8/+c/+34 taken before the first bell. */ ++ bool nopatch; ++ u32 np_pre[5]; ++ atomic_t seq, acks, cap_seen, busy, drains, drain_errors, exported, ++ trace_saved; ++ atomic_t watchdog_state; /* 0 off, 1 armed, 2 disarmed, 3 fired, 4 terminal */ ++ int watchdog_cpu; ++ u32 owner; ++ struct mutex drain_lock; /* single auxiliary-ring owner */ ++ struct hrtimer watchdog; ++ struct g5p_r73b_row rows[G5P_R73B_ROWS]; ++ u32 trace_rows[G5P_R73B_TRACE_ROWS][G5P_R73B_TRACE_WORDS]; ++ u32 irq_cap, r74_case, r74_mask, r74_ticks, r74_word1_count; ++ u32 r74_word1[G5P_R74_WORD1]; ++ atomic_t r74_seq, r74_irq_refused; ++ struct g5p_r74_milestone r74_milestones[G5P_R74_MILESTONES]; ++ ++ /* r74f_alive and FIRING are written under event_lock only. */ ++ u32 r74f_case, r74f_alive, r74f_active, r74f_generation, r74f_stop; ++ ++ u32 r87b_attach; ++ s32 r74f_stop_value; ++ atomic_t r74f_bells; ++ struct g5p_r74f_bell r74f_bell[G5P_R74F_KINDS]; ++ ++ /* R74g: no host MMU pass, publish-only TA grace, one fallback bell. */ ++ u32 r74g_case, r74g_ack_skipped, r74g_grace_end, r74g_opened; ++ u32 r74g_grace_alive; ++ u64 r74g_grace_ns; ++ struct g5p_r74g_dc3 r74g_dc3[2]; ++ /* R74h: RAM-only trace tail and checkpoints; NULL when off. */ ++ u32 r74h_case; ++ ++ ++ ++ /* Heartbeat116 (heartbeat116.h): the R74 window detector; IRQ thread only. */ ++ bool hb_on; ++ u32 hb_seed, hb_empty, hb_cross; ++ atomic_t hb_late; ++ struct g5p_hb_cfg hb_cfg; ++ struct g5p_hb hb; ++}; ++ ++/* The ctrl+0 word installed before release. */ ++static u32 g5p_boot_ctrl0(struct g5p_boot *b) ++{ ++ return b->r78_ctrl0; ++} ++ ++ ++ ++/* render-r96.inc: the receipts at export (RAM only) */ ++ ++/* render-r94.inc: the RAM state of the record and the code BO at export, read only */ ++ ++/* boot-r80.inc: the two IRQ-owned plateau banks, read-only */ ++ ++/* render-r84b.inc: the late raw dump and the CPU visibility step, RAM only */ ++ ++static int g5p_boot_drain(struct g5p_diag *g); ++static u32 g5p_transport_event_limit(void); ++static int g5p_r73c_fast_control(struct g5p_diag *g); ++static int g5p_r73e_prepare(struct g5p_diag *g); ++static int g5p_r73e_irq_submit(struct g5p_diag *g); ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner); ++ ++static const char *g5p_r73b_name(u32 id) ++{ ++ return id == 1 ? "meta1-bounded-all-rings" : ++ id == 2 ? "meta0-bounded-all-rings-control" : ++ id == 3 ? "meta1-event-only-reference" : ++ "off"; ++} ++ ++static bool g5p_r73b_sampled(u32 n) ++{ ++ return n <= 16 || is_power_of_2(n); ++} ++ ++static u32 *g5p_r73b_ptr(struct g5p_boot *b, u32 off) ++{ ++ return (u32 *)(b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r73b_word(struct g5p_boot *b, u32 off) ++{ ++ return READ_ONCE(*g5p_r73b_ptr(b, off)); ++} ++ ++static u32 g5p_r73b_slot(u32 seq) ++{ ++ if (seq < G5P_R73B_HEAD) ++ return seq; ++ return G5P_R73B_HEAD + ++ (seq - G5P_R73B_HEAD) % (G5P_R73B_ROWS - G5P_R73B_HEAD); ++} ++ ++/* Lockless: a writer owns its slot by sequence; seq is published last. */ ++static void g5p_r73b_crumb(struct g5p_boot *b, u32 phase, u32 owner, int result) ++{ ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r73b_row *row; ++ u32 seq; ++ ++ if (!r) ++ return; ++ seq = atomic_inc_return(&r->seq) - 1; ++ row = &r->rows[g5p_r73b_slot(seq)]; ++ WRITE_ONCE(row->seq, 0); ++ /* Invalidate before rewriting so a reader never pairs old seq/new data. */ ++ smp_wmb(); ++ row->phase = phase; ++ row->t_ns = ktime_get_mono_fast_ns(); ++ row->irq = atomic_read(&b->irqs); ++ row->ack = atomic_read(&r->acks); ++ row->released = READ_ONCE(b->released); ++ row->owner = owner; ++ row->result = result; ++ row->cap_seen = atomic_read(&r->cap_seen); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r73b_row_slots); i++) { ++ u32 ctl = b->layout.offset[g5p_r73b_row_slots[i]]; ++ ++ row->cp[i][0] = g5p_r73b_word(b, ctl); ++ row->cp[i][1] = g5p_r73b_word(b, ctl + 0x10); ++ } ++ /* Publish the row contents before its sequence. */ ++ smp_wmb(); ++ WRITE_ONCE(row->seq, seq + 1); ++} ++ ++static const char *g5p_r74_name(u32 id) ++{ ++ return id == 1 ? "forced-ready-meta1-natural-ta" : ++ id == 2 ? "meta0-unpatched-natural-ta-control" : ++ "off"; ++} ++ ++static const char *g5p_r74_step_name(u32 step) ++{ ++ return step < G5P_R74_STEPS ? g5p_r74_steps[step] : "unknown"; ++} ++ ++static u32 g5p_r74_case(struct g5p_boot *b) ++{ ++ return b && b->r73b ? 1 : 0; ++} ++ ++/* R74b case 1: after cmd8 retirement, optional host MMIO must not race the ++ * firmware policy loop power-gating RGX. Functional setup/kick sites opt in ++ * explicitly; this predicate gates observations only. ++ */ ++static bool g5p_r74b_ram_only(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ ++ return g5p_r74_case(b) == 1 && b && READ_ONCE(b->released) && ++ READ_ONCE(b->r72_meta6c8); ++} ++ ++/* Process context only, never per IRQ: one console line per r74 step, so a ++ * whole-SoC freeze is localized by the last line netconsole delivered. ++ */ ++static void g5p_r74_mark(struct g5p_diag *g, u32 step, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_r74_milestone m; ++ u32 seq; ++ ++ if (!r || !1) ++ return; ++ m.t_ns = ktime_get_ns(); ++ m.step = step; ++ m.ret = ret; ++ m.irqs = atomic_read(&b->irqs); ++ m.events = READ_ONCE(b->events); ++ m.dc[0] = g5p_r73b_word(b, b->layout.offset[0x6c]); ++ m.dc[1] = g5p_r73b_word(b, b->layout.offset[0x6c] + 0x10); ++ m.ta[0] = g5p_r73b_word(b, b->layout.offset[0x60]); ++ m.ta[1] = g5p_r73b_word(b, b->layout.offset[0x60] + 0x10); ++ seq = atomic_inc_return(&r->r74_seq) - 1; ++ if (seq < G5P_R74_MILESTONES) ++ r->r74_milestones[seq] = m; ++ dev_emerg( ++ g->dev, ++ "R74 milestone seq=%u step=%s ret=%d irqs=%u events=%u dc=%u/%u ta=%u/%u " ++ "t_ns=%llu end=r74\n", ++ seq, g5p_r74_step_name(step), ret, m.irqs, m.events, m.dc[0], ++ m.dc[1], m.ta[0], m.ta[1], m.t_ns); ++} ++ ++/* R74 keeps repeated ALIVEs quiet whatever their word1: each new value is ++ * printed once (eight at most), so a moved loop cannot flood the console. ++ */ ++static bool g5p_r74_word1(struct g5p_boot *b, u32 word1) ++{ ++ struct g5p_r73b *r = b->r73b; ++ ++ ; ++ ; ++ for (u32 i = 0; i < r->r74_word1_count; i++) ++ if (r->r74_word1[i] == word1) ++ return false; ++ if (r->r74_word1_count == G5P_R74_WORD1) ++ return false; ++ r->r74_word1[r->r74_word1_count++] = word1; ++ return true; ++} ++ ++static bool g5p_r73b_copy_row(struct g5p_r73b *r, u32 seq, ++ struct g5p_r73b_row *out) ++{ ++ struct g5p_r73b_row *row = &r->rows[g5p_r73b_slot(seq)]; ++ ++ if (READ_ONCE(row->seq) != seq + 1) ++ return false; ++ /* Pairs with the writer's publication barrier. */ ++ smp_rmb(); ++ *out = *row; ++ /* Re-check that the slot was not recycled while copying. */ ++ smp_rmb(); ++ return READ_ONCE(row->seq) == seq + 1; ++} ++ ++static int g5p_r73b_format(char *buf, size_t len, ++ const struct g5p_r73b_row *row) ++{ ++ u32 phase = row->phase < G5P_R73B_PHASES ? row->phase : 0; ++ ++ return scnprintf( ++ buf, len, ++ "R73B crumb seq=%u t_ns=%llu phase=%s irq=%u ack=%u released=%u event=%u/%u " ++ "log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d cap=%u end=r73b\n", ++ row->seq - 1, row->t_ns, g5p_r73b_phase_names[phase], row->irq, ++ row->ack, row->released, row->cp[0][0], row->cp[0][1], ++ row->cp[1][0], row->cp[1][1], row->cp[2][0], row->cp[2][1], ++ row->cp[3][0], row->cp[3][1], row->cp[4][0], row->cp[4][1], ++ row->owner, row->result, row->cap_seen); ++} ++ ++/* Snapshot the producer once and consume at most that modular distance ++ * (<= 255). One consumer store returns the whole batch; a producer that ++ * advances meanwhile is left for the next pass, never chased. ++ */ ++static int g5p_r73b_drain_ring(struct g5p_boot *b, u32 ctl_slot, u32 lane, ++ u32 data_slot, u32 stride, u32 ring, ++ u32 *consumed) ++{ ++ u32 ctl = b->layout.offset[ctl_slot] + lane * 0x20; ++ u32 data = b->layout.offset[data_slot] + ++ (ctl_slot == 0x71 ? lane * 0x2c00 : 0); ++ u32 consumer = g5p_r73b_word(b, ctl); ++ u32 producer = g5p_r73b_word(b, ctl + 0x10); ++ u32 count, hash = b->r73_aux_hash[ring] ?: 2166136261U; ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ count = (producer - consumer) & 255; ++ if (!count) ++ return 0; ++ /* Producer index before the slot payloads it covers. */ ++ dma_rmb(); ++ for (u32 n = 0; n < count; n++) { ++ u32 words[G5P_R73B_TRACE_WORDS] = { 0 }; ++ u32 off = data + consumer * stride; ++ ++ for (u32 i = 0; i < stride / 4; i++) { ++ u32 value = g5p_r73b_word(b, off + i * 4); ++ ++ if (i < ARRAY_SIZE(words)) ++ words[i] = value; ++ hash = (hash ^ value) * 16777619U; ++ } ++ if (ring == 5 && ++ atomic_read(&b->r73b->trace_saved) < G5P_R73B_TRACE_ROWS) { ++ u32 saved = atomic_read(&b->r73b->trace_saved); ++ ++ memcpy(b->r73b->trace_rows[saved], words, ++ sizeof(words)); ++ /* Publish this complete row before increasing the visible count. */ ++ smp_wmb(); ++ atomic_inc(&b->r73b->trace_saved); ++ }; ++ ; ++ if (!b->r73_aux_entries[ring]) ++ memcpy(b->r73_aux_first[ring], words, ++ sizeof(b->r73_aux_first[ring])); ++ memcpy(b->r73_aux_last[ring], words, ++ sizeof(b->r73_aux_last[ring])); ++ b->r73_aux_entries[ring]++; ++ consumer = (consumer + 1) & 255; ++ } ++ b->r73_aux_hash[ring] = hash; ++ /* dma_wmb() ordering plus the slot loads above: Apple returns the ++ * consumer only after its reads, with a full barrier. ++ */ ++ dma_mb(); ++ WRITE_ONCE(*g5p_r73b_ptr(b, ctl), consumer); ++ *consumed += count; ++ return 0; ++} ++ ++/* Apple order after the ACK: four log lanes, events, statistics, trace. */ ++static int g5p_r73b_aux_pass(struct g5p_diag *g, u32 *consumed) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret = 0; ++ ++ for (u32 lane = 0; lane < 4 && !ret; lane++) ++ ret = g5p_r73b_drain_ring(b, 0x71, lane, 0x75, 44, lane, ++ consumed); ++ if (!ret) ++ ret = g5p_boot_drain(g); ++ /* R80: ring 4 has one consumer, the drain thread (retains every record). */ ++ if (!ret && !false) ++ ret = g5p_r73b_drain_ring(b, 0x72, 0, 0x76, 44, 4, consumed); ++ if (!ret) ++ ret = g5p_r73b_drain_ring(b, 0x73, 0, 0x77, 36, 5, consumed); ++ return ret; ++} ++ ++static bool g5p_r73b_pending(struct g5p_boot *b, bool aux) ++{ ++ static const u8 slots[] = { 0x70, 0x71, 0x72, 0x73 }; ++ ++ for (u32 i = 0; i < (aux ? ARRAY_SIZE(slots) : 1); i++) { ++ for (u32 lane = 0; lane < (slots[i] == 0x71 ? 4 : 1); lane++) { ++ u32 ctl = b->layout.offset[slots[i]] + lane * 0x20; ++ ++ if (g5p_r73b_word(b, ctl) != ++ g5p_r73b_word(b, ctl + 0x10)) ++ return true; ++ } ++ } ++ return false; ++} ++ ++/* The IRQ thread only trylocks the auxiliary rings; a miss still runs the ++ * spinlock-protected event decoder. Process-context paths lock normally. ++ * Drains with work, errors or sampled entries record before/after rows. ++ */ ++static int g5p_r73b_service(struct g5p_diag *g, u32 owner, bool sampled) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 consumed = 0, before; ++ bool rows, aux; ++ int ret; ++ ++ if (!r) ++ return g5p_boot_drain(g); ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* The atomic r73c window services only the event ring. Auxiliary ++ * traffic is retained until cmd8 is either published or refused. */ ++ aux = 1 && !READ_ONCE(r->fast_window); ++ if (aux && owner == G5P_R73B_OWNER_IRQ && ++ !mutex_trylock(&r->drain_lock)) { ++ atomic_inc(&r->busy); ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BUSY, READ_ONCE(r->owner), ++ -EBUSY); ++ ret = g5p_boot_drain(g); ++ goto out; ++ } ++ if (aux && owner != G5P_R73B_OWNER_IRQ) ++ mutex_lock(&r->drain_lock); ++ if (aux) ++ WRITE_ONCE(r->owner, owner); ++ atomic_inc(&r->drains); ++ rows = sampled || g5p_r73b_pending(b, aux); ++ if (rows) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_BEFORE, owner, 0); ++ before = READ_ONCE(b->events); ++ ret = aux ? g5p_r73b_aux_pass(g, &consumed) : g5p_boot_drain(g); ++ consumed += READ_ONCE(b->events) - before; ++ if (rows || ret) ++ g5p_r73b_crumb(b, G5P_R73B_DRAIN_AFTER, owner, ret ?: consumed); ++ if (aux) { ++ WRITE_ONCE(r->owner, G5P_R73B_NONE); ++ mutex_unlock(&r->drain_lock); ++ } ++out: ++ if (ret) { ++ atomic_inc(&r->drain_errors); ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return ret; ++} ++ ++/* Heartbeat116: an R74 epoch bounds IRQ174 and its ring entries per window. */ ++static bool g5p_hb116_on(struct g5p_boot *b) ++{ ++ return b->r73b && b->r73b->hb_on; ++} ++ ++/* Auxiliary-ring entries consumed so far (RAM, monotonic, may wrap). */ ++static u32 g5p_hb116_aux(struct g5p_boot *b) ++{ ++ u32 sum = 0; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(b->r73_aux_entries); i++) ++ sum += READ_ONCE(b->r73_aux_entries[i]); ++ return sum; ++} ++ ++static u32 g5p_hb116_progress(struct g5p_boot *b) ++{ ++ return READ_ONCE(b->events) + g5p_hb116_aux(b); ++} ++ ++/* The lifetime cap's terminal stop, once: final row, cap_seen, failed, one disable. */ ++static void g5p_hb116_trip(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb *h = &r->hb; ++ u32 prior = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (atomic_cmpxchg(&r->cap_seen, 0, 1)) ++ return; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "HB116 trip why=%s prior_failed=%u windows=%u stalled=%u ms=%llu irqs=%u " ++ "events=%u acks=%u empty=%u aux=%u total_irqs=%u total_events=%u t_ns=%llu " ++ "stop=masked late=ack-only end=hb116\n", ++ g5p_hb_why_name[h->why < G5P_HBW_N ? h->why : 0], prior, ++ h->windows, h->stalled, h->trip.ns / NSEC_PER_MSEC, ++ h->trip.irqs, h->trip.events, h->trip.acks, h->trip.empty, ++ h->trip.aux, n, READ_ONCE(b->events), h->trip_ns); ++} ++ ++/* Receipts only: the first window past each Wake112 lifetime bound; nothing trips here. */ ++static void g5p_hb116_cross(struct g5p_diag *g, u32 irqs, u32 events) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ u32 limit = g5p_transport_event_limit(); ++ ++ if (!(r->hb_cross & 1) && irqs > r->irq_cap) { ++ r->hb_cross |= 1; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=irq-cap value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ r->irq_cap, irqs, events, r->hb.windows); ++ } ++ if (!(r->hb_cross & 2) && events >= limit) { ++ r->hb_cross |= 2; ++ dev_info( ++ g->dev, ++ "HB116 cross bound=event-limit value=%u irqs=%u events=%u windows=%u " ++ "verdict=continue end=hb116\n", ++ limit, irqs, events, r->hb.windows); ++ } ++} ++ ++/* After the entry's ACK and service: account it, close a window, or trip. */ ++static void g5p_hb116_irq(struct g5p_diag *g, int irq, u32 n, u32 before) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ const struct g5p_hb_win *w = &r->hb.last; ++ u32 seed = r->hb_seed, events = READ_ONCE(b->events); ++ struct g5p_hb_in in; ++ ++ if (g5p_hb116_progress(b) == before) ++ r->hb_empty++; ++ in.now_ns = ktime_get_mono_fast_ns(); ++ in.irqs = n + seed; ++ in.events = events + seed; ++ in.acks = atomic_read(&r->acks) + seed; ++ in.empty = r->hb_empty + seed; ++ in.aux = g5p_hb116_aux(b) + seed; ++ switch (g5p_hb_step(&r->hb, &r->hb_cfg, &in)) { ++ case G5P_HBR_OPEN: ++ case G5P_HBR_CLOSED: ++ break; ++ case G5P_HBR_TRIP: ++ g5p_hb116_trip(g, irq, n); ++ return; ++ } ++ g5p_hb116_cross(g, n, events); ++ if (g5p_hb_report(&r->hb)) ++ g5p_job_printk( ++ g->dev, ++ "HB116 window n=%u t_ns=%llu ms=%llu irqs=%u events=%u acks=%u empty=%u aux=%u " ++ "stalled=%u peak_irqs=%u peak_events=%u total_irqs=%u total_events=%u " ++ "total_acks=%d end=hb116\n", ++ r->hb.windows, in.now_ns, w->ns / NSEC_PER_MSEC, ++ w->irqs, w->events, w->acks, w->empty, w->aux, ++ r->hb.stalled, r->hb.peak_irqs, r->hb.peak_events, n, ++ events, atomic_read(&r->acks)); ++} ++ ++/* Every entry ACKs. Over the cap: ACK, final row, cap_seen, one disable; ++ * the kernel stays alive so the runner can export the record. ++ * Heartbeat116: an R74 epoch has no lifetime cap. The window detector runs ++ * after the ACK and the service; after its trip an entry only ACKs. ++ */ ++static irqreturn_t g5p_r73b_irq(struct g5p_diag *g, int irq, u32 n) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ bool sampled = g5p_r73b_sampled(n); ++ u32 before = 0; ++ ++ if (r->hb_on && atomic_read(&r->cap_seen)) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ atomic_inc(&r->hb_late); ++ return IRQ_HANDLED; ++ } ++ if (!r->hb_on && n > r->irq_cap) { ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ if (!atomic_read(&r->cap_seen)) { ++ g5p_r73b_crumb(b, G5P_R73B_CAP, G5P_R73B_OWNER_IRQ, n); ++ if (!atomic_cmpxchg(&r->cap_seen, 0, 1)) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ } ++ } ++ return IRQ_HANDLED; ++ } ++ { ++ /* KPF level6G: Apple ACKs the AIC meta interrupt before ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ atomic_inc(&r->acks); ++ } ++ if (sampled) ++ g5p_r73b_crumb(b, G5P_R73B_IRQ, G5P_R73B_OWNER_IRQ, n); ++ if (r->hb_on) ++ before = g5p_hb116_progress(b); ++ if (READ_ONCE(b->released)) ++ g5p_r73b_service(g, G5P_R73B_OWNER_IRQ, sampled); ++ if (1 && READ_ONCE(b->alive)) ++ g5p_r73e_alive_submit(g, G5P_R73B_OWNER_IRQ); ++ ; ++ ; ++ ; ++ ; ++ ; ++ if (r->hb_on) ++ g5p_hb116_irq(g, irq, n, before); ++ return IRQ_HANDLED; ++} ++ ++static void g5p_r73b_disarm(struct g5p_r73b *r) ++{ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 2) == 1) ++ hrtimer_cancel(&r->watchdog); ++} ++ ++static void g5p_r73b_stop(void *data) ++{ ++ g5p_r73b_disarm(data); ++} ++ ++/* Cached CPU RAM only: no MMIO, no locks, then a deliberate panic so a ++ * silent hang becomes a pstore/ramoops record. ++ */ ++static enum hrtimer_restart g5p_r73b_watchdog_fire(struct hrtimer *timer) ++{ ++ struct g5p_r73b *r = container_of(timer, struct g5p_r73b, watchdog); ++ struct g5p_boot *b = r->boot; ++ struct g5p_r73b_row row; ++ u32 total = atomic_read(&r->seq), first, last; ++ char line[320]; ++ bool found = false; ++ ++ if (atomic_cmpxchg(&r->watchdog_state, 1, 3) != 1) ++ return HRTIMER_NORESTART; ++ /* A contained cap or a latched failure is terminal: the IRQ only ACKs, ++ * service returns -EIO and the runner sees failed=1. Keep that host ++ * alive for the export instead of turning it into a deliberate panic. ++ */ ++ if (atomic_read(&r->cap_seen) || READ_ONCE(b->failed)) { ++ atomic_set(&r->watchdog_state, 4); ++ dev_emerg( ++ r->dev, ++ "R73B watchdog terminal case=%u irqs=%d acks=%d cap_seen=%d failed=%u rows=%u " ++ "no-panic end=r73b\n", ++ r->case_id, atomic_read(&b->irqs), ++ atomic_read(&r->acks), atomic_read(&r->cap_seen), ++ READ_ONCE(b->failed), total); ++ return HRTIMER_NORESTART; ++ } ++ dev_emerg( ++ r->dev, ++ "R73B receipt case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u release=%s end=r73b\n", ++ r->case_id, g5p_r73b_name(r->case_id), r->ack_first, r->aux, ++ r->irq_cap, r->watchdog_ms, r->dc_variant, r->meta6c8, ++ init_utsname()->release); ++ first = total > G5P_R73B_PANIC_ROWS ? total - G5P_R73B_PANIC_ROWS : 0; ++ for (u32 seq = first; seq < total; seq++) { ++ if (!g5p_r73b_copy_row(r, seq, &row)) ++ continue; ++ g5p_r73b_format(line, sizeof(line), &row); ++ dev_emerg(r->dev, "%s", line); ++ } ++ for (last = total; last > first && !found; last--) ++ found = g5p_r73b_copy_row(r, last - 1, &row); ++ if (!found) ++ memset(&row, 0, sizeof(row)); ++ panic("G5P R73B watchdog: release epoch incomplete after %ums case=%u irqs=%d acks=%d " ++ "cap_seen=%d busy=%d rows=%u last seq=%u phase=%s irq=%u ack=%u released=%u " ++ "event=%u/%u log0=%u/%u other=%u/%u trace=%u/%u dc=%u/%u owner=%u result=%d " ++ "cap=%u end=r73b\n", ++ 0, 1, atomic_read(&b->irqs), atomic_read(&r->acks), ++ atomic_read(&r->cap_seen), atomic_read(&r->busy), total, ++ row.seq ? row.seq - 1 : 0, ++ g5p_r73b_phase_names[row.phase < G5P_R73B_PHASES ? row.phase : 0], ++ row.irq, row.ack, row.released, row.cp[0][0], row.cp[0][1], ++ row.cp[1][0], row.cp[1][1], row.cp[2][0], row.cp[2][1], ++ row.cp[3][0], row.cp[3][1], row.cp[4][0], row.cp[4][1], row.owner, ++ row.result, row.cap_seen); ++ return HRTIMER_NORESTART; ++} ++ ++static void g5p_r73b_release_entry(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ ++ if (!r) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_ENTRY, G5P_R73B_NONE, 0); ++ { ++ return; ++ } ++} ++ ++static void g5p_r73b_release_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_EXIT, G5P_R73B_NONE, ret); ++ if (ret) ++ g5p_r73b_disarm(b->r73b); ++} ++ ++/* The release epoch ends when the host has returned from the ALIVE wait. */ ++static void g5p_r73b_events_exit(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ if (!b->r73b) ++ return; ++ g5p_r73b_crumb(b, G5P_R73B_EVENTS_EXIT, G5P_R73B_OWNER_EVENTS, ret); ++ g5p_r73b_disarm(b->r73b); ++} ++ ++static int g5p_r73b_latch(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const u32 id = (1), ms = (0); ++ const u32 fast = (1); ++ const u32 early = (1); ++ const u32 r74 = (1), cap = (1048576); ++ const u32 r74f = (1); ++ const u32 r74g = (1); ++ const u32 r74h = (0); ++ ++ struct g5p_r73b *r; ++ int ret; ++ ++ ; ++ ; ++ ; ++ ; ++ ; ++ ; ++ /* nopatch replaces r73f case 1 under the whole r74/r74f/r74g path, and nothing else. */ ++ /* nopatch's receipts and its r74 gate are the TYPED_V3 path's; any other path (r74 0) runs ++ * Apple's image anyway (only r73f patches it) and gets neither. ++ */ ++ if (1 != g5p_fw_unpatched_want((1), 0, 0, 0) || ++ (1 && r74 == 1 && (0 || r74f != 1 || r74g != 1))) { ++ dev_err(g->dev, ++ "NOPATCH reject phase=map unpatched=%u/%u r73f_case=%u path=%u/%u/%u end=nopatch\n", ++ 1, (1), 0, r74, r74f, r74g); ++ return -EINVAL; ++ } ++ /* R74 rides only on an exact r73f case 1 (or nopatch) or 4 with the raised limits. */ ++ if (r74 && (!id || r74 > 2 || 0 != (r74 == 1 ? (1 ? 0 : 1) : 4) || ++ 1 != (r74 == 1) || !(1) || cap < G5P_R74_IRQ_CAP || ++ cap > G5P_R74_IRQ_CAP_MAX || ++ g5p_transport_event_limit() < G5P_R74_EVENT_FLOOR)) { ++ dev_err(g->dev, ++ "R74 reject phase=map case=%u r73b_case=%u r73f_case=%u meta6c8=%u " ++ "render_engine=%u irq_cap=%u event_limit=%u end=r74\n", ++ r74, id, 0, b->r72_meta6c8, (1), cap, ++ g5p_transport_event_limit()); ++ return -EINVAL; ++ } ++ ++ /* R74f rides only on the exact r74 case 1 forced-ready epoch. */ ++ ; ++ ++ /* R74g rides only on the exact r74f case 1 epoch. */ ++ ; ++ ++ /* R74h observes only the exact r74g case 1 epoch. */ ++ ; ++ ++ /* The bounded serialized drain supersedes r73's IRQ-context drain. */ ++ ; ++ ; ++ ; ++ ; ++ if (id > 3 || fast > 1 || (fast && id == 3) || early > 2 || ++ (early && (!fast || id != early)) || b->r73_aux_drain || 7 != 7 || ++ 1 != (id == 2 ? 0 : 1) || (ms && (ms < 1000 || ms > 30000))) ++ return -EINVAL; ++ r = devm_kzalloc(g->dev, sizeof(*r), GFP_KERNEL); ++ if (!r) ++ return -ENOMEM; ++ r->dev = g->dev; ++ r->boot = b; ++ r->case_id = id; ++ r->watchdog_ms = ms; ++ r->dc_variant = 7; ++ r->meta6c8 = 1; ++ r->aux = id != 3; ++ r->ack_first = true; ++ r->fast_enabled = fast; ++ r->r73e_case = early; ++ r->r74_case = r74; ++ r->r74f_case = r74f; ++ r->r74g_case = r74g; ++ r->r74h_case = r74h; ++ r->settle_us = min_t(u32, READ_ONCE(boot_alive_settle_us), 5000); ++ r->nopatch = 1 && r74 == 1; ++ r->irq_cap = r74 ? cap : G5P_R73B_IRQ_CAP; ++ /* Heartbeat116: in an R74 epoch the window detector replaces both lifetime bounds. */ ++ r->hb_on = r74 != 0; ++ r->hb_seed = r74 ? (0) : 0; ++ g5p_hb_cfg_default(&r->hb_cfg); ++ atomic_set(&r->early_once, 0); ++ atomic_set(&r->early_busy, 0); ++ atomic_set(&r->early_proof, G5P_PX_NONE); ++ r->watchdog_cpu = -1; ++ mutex_init(&r->drain_lock); ++ hrtimer_setup(&r->watchdog, g5p_r73b_watchdog_fire, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL_PINNED_HARD); ++ ret = devm_add_action_or_reset(g->dev, g5p_r73b_stop, r); ++ if (ret) ++ return ret; ++ b->r73_aux_drain = 1; ++ b->r73b = r; ++ dev_info( ++ g->dev, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), r->ack_first, r->aux, r->irq_cap, ms, ++ r->dc_variant, r->meta6c8, G5P_R73B_ROWS, ++ init_utsname()->release); ++ g5p_blob_printf( ++ g, ++ "R73B latched case=%u name=%s ack_first=%u aux=%u cap=%u watchdog_ms=%u " ++ "dc_variant=%u meta6c8=%u rows=%u release=%s end=r73b\n", ++ id, g5p_r73b_name(id), 1, 1, r->irq_cap, ms, 7, 1, ++ G5P_R73B_ROWS, init_utsname()->release); ++ g5p_blob_printf(g, "R73C latched fast=%u case=%u meta6c8=%u end=r73c\n", ++ 1, id, 1); ++ dev_info(g->dev, "R73E latched case=%u meta6c8=%u fast=%u end=r73e\n", ++ early, r->meta6c8, r->fast_enabled); ++ ; ++ ; ++ if (r->nopatch) ++ dev_info( ++ g->dev, ++ "NOPATCH latched fw=apple-original r73f_case=0 meta6c8=%u r73e_case=%u r74=%u " ++ "r74f=%u r74g=%u force_policy_ready=0 end=nopatch\n", ++ r->meta6c8, early, r74, r74f, r74g); ++ { ++ dev_info( ++ g->dev, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, r->meta6c8, ++ r->irq_cap, g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "R74 latched case=%u name=%s r73f_case=%u r73e_case=%u meta6c8=%u irq_cap=%u " ++ "event_limit=%u render_engine=typed-v3 irq_test=refused render_irq_tested=0 " ++ "end=r74\n", ++ r74, g5p_r74_name(r74), 0, early, 1, r->irq_cap, ++ g5p_transport_event_limit()); ++ dev_info( ++ g->dev, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ g5p_blob_printf( ++ g, ++ "HB116 latched on=%u window_ms=%llu irq_ceiling=%u event_ceiling=%u ratio=%u " ++ "slack=%u stall_windows=%u seed=%u legacy_irq_cap=%u legacy_event_limit=%u " ++ "lifetime_bound=none end=hb116\n", ++ r->hb_on, r->hb_cfg.window_ns / NSEC_PER_MSEC, ++ r->hb_cfg.irq_ceiling, r->hb_cfg.event_ceiling, ++ r->hb_cfg.ratio, r->hb_cfg.slack, ++ r->hb_cfg.stall_windows, r->hb_seed, r->irq_cap, ++ g5p_transport_event_limit()); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ g5p_blob_printf( ++ g, ++ "R74F latched case=%u off_us=%u on_us=%u bell_ms=%u kick=10 " ++ "owner=first-type0-alive-after-reactivation retained_failed=0 end=r74f\n", ++ r74f, G5P_R74F_OFF_US, G5P_R74F_ON_US, ++ G5P_R74F_BELL_MS); ++ } ++ { ++ dev_info( ++ g->dev, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ g5p_blob_printf( ++ g, ++ "R74G latched case=%u grace_alives=%u grace_us=%u fallback_bells_max=1 " ++ "host_mmu=firmware-dc3 ack_ac8=suppressed companion=root+3980 end=r74g\n", ++ r74g, G5P_R74G_GRACE_ALIVES, G5P_R74G_GRACE_US); ++ }; ++ /* TYPED_V3 production path: the self-drain DC3 attach policy of r87b rides on r74f + r74g ++ * alone, with none of the r74h / r80 instrumentation. r74g has the firmware own the first ++ * MMU pass, which is what makes self-drain safe (r87b-attach.h); r74h only observes. Every ++ * other r8x / r9x knob is still refused below without boot_r80_case. ++ */ ++ { ++ r->r87b_attach = G5P_R87B_ATTACH_SELF_DRAIN; ++ dev_info( ++ g->dev, ++ "R87B latched policy=1 name=self-drain r74f=%u r74g=%u r74h=%u r80=0 " ++ "production_path=1 end=r87b\n", ++ r74f, r74g, r74h); ++ } ++ { ++ { ++ { ++ { ++ { ++ if ((1) && !r->r87b_attach) { ++ dev_err(g->dev, ++ "R87B reject phase=map r87b knob without boot_r80_case policy=%u end=r87b\n", ++ (1)); ++ return -EBADMSG; ++ } else { ++ { ++ { ++ ; ++ ; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ }; ++ } ++ return 0; ++} ++ ++/* Process context, cached CPU RAM only; never waits for the command lock. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +new file mode 100644 +index 000000000..71cf79dbe +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -0,0 +1,532 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74f deferred doorbell. Included by boot.inc before the event decoder. ++ * Diagnostic bypass on the r74 forced-ready epoch only; not power policy. ++ * ++ * A DC3 pair or a natural TA is published once, append-only, before this ++ * code runs; its producer is never rewound, republished or torn down. If ++ * the immediate functional gate finds GPC inactive, that generation stays ++ * pending (never b->failed), the host repeats the r74e release transition ++ * once, and the first type-0 ALIVE decoded afterwards owns one doorbell ++ * attempt. The owner runs under event_lock: producer confirm, R1+14 fence, ++ * active check, dma_wmb(), doorbell 0x10. Receipts stay in RAM; console ++ * lines are printed afterwards from process context. ++ */ ++ ++static const char *const g5p_r74f_kinds[G5P_R74F_KINDS] = { "dc3", "ta", "3d" }; ++ ++static const char *const g5p_r74f_states[G5P_R74F_STATES] = { ++ "idle", "published", "open", "firing", "rang", "cancelled", ++}; ++ ++static const char *const g5p_r74f_cancels[G5P_R74F_CANCELS] = { ++ "none", "reactivation-failed", ++ "no-alive", "consumed-before-bell", ++ "protocol", "alive-during-reactivation", ++ "self-drain", ++}; ++ ++static const char *const g5p_r74f_stops[G5P_R74F_STOPS] = { ++ "none", ++ "attach-reactivation-failed", ++ "attach-published-not-kicked", ++ "attach-kicked-not-consumed", ++ "first-mmu-window-closed", ++ "ta-reactivation-failed", ++ "ta-published-not-kicked", ++ "firmware-dc3-tag-mismatch", ++ "dc3-ram-validation-refused", ++ "firmware-companion-recovery", ++ "ta-published-not-consumed", ++ "ta-window-missed", ++}; ++ ++static const char *const g5p_r74f_paths[G5P_R74F_PATHS] = { ++ "none", ++ "immediate", ++ "deferred", ++}; ++ ++static bool g5p_r74f_on(struct g5p_boot *b) ++{ ++ return b && b->r73b && 1; ++} ++ ++static u32 *g5p_r74f_ctl(struct g5p_boot *b, u32 kind) ++{ ++ return (u32 *)g5p_boot_shared(b, kind == G5P_R74F_TA ? 0x60 : ++ kind == G5P_R74F_3D ? 0x64 : ++ 0x6c); ++} ++ ++/* Owner only, event_lock held with interrupts off. Between the active check ++ * and the doorbell there is only dma_wmb(): no console, no other MMIO. ++ */ ++static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ u32 *fence, u32 *state) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 consumer = READ_ONCE(ctl[0]); ++ u32 producer = smp_load_acquire(&ctl[4]); ++ u32 queued = (f->target - f->start) & 255; ++ u32 progressed = (consumer - f->start) & 255; ++ u32 expected = f->kind == G5P_R74F_DC3 ? 2 : 1; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ /* Do not turn corrupted modulo-256 ring state into a doorbell. */ ++ if (consumer > 255 || producer > 255 || queued != expected || ++ progressed > queued) ++ return -EPROTO; ++ /* Acquire: pairs with the producer store published before the state. */ ++ if (producer != f->target) ++ return -EPROTO; ++ if (consumer == f->target) ++ return -EALREADY; ++ writel(7, g->window[1] + 0x14); ++ *fence = readl(g->window[1] + 0x14); ++ if (*fence & ~7U) ++ return -EPROTO; ++ *state = readl(g->window[1] + 0x1000); ++ /* perf60 (gpc_any_pstate): a GPC request in flight or a not-yet-awake GPC after the wake ++ * above; re-read for up to 2 ms, as Apple's setGPCPerformanceState waits for bit 4. Reads only. ++ */ ++ { ++ u64 t0 = ktime_get_mono_fast_ns(); ++ ++ while (!g5p_gpc_ok(*state) && ktime_get_mono_fast_ns() - t0 < ++ 2000ULL * NSEC_PER_USEC) { ++ udelay(10); ++ *state = readl(g->window[1] + 0x1000); ++ } ++ } ++ if (!g5p_gpc_ok(*state)) ++ return -EHOSTDOWN; ++ dma_wmb(); ++ writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); ++ return 0; ++} ++ ++/* Owner only: record the attempt, then publish it with the state release. */ ++static void g5p_r74f_rang(struct g5p_r73b *r, struct g5p_r74f_bell *f, int ret, ++ u32 fence, u32 state, u32 path) ++{ ++ f->bell_ns = ktime_get_mono_fast_ns(); ++ f->path = path; ++ f->fence = fence; ++ f->bell_state = state; ++ f->bell_ret = ret; ++ if (!ret) { ++ f->kicked = 1; ++ atomic_inc(&r->r74f_bells); ++ } else if (ret == -EALREADY) { ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ } else if (ret != -EHOSTDOWN) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ } ++ atomic_set_release(&f->state, G5P_R74F_RANG); ++} ++ ++/* g5p_boot_drain(), event_lock held: one type-0 ALIVE was just decoded. */ ++static void g5p_r74f_alive(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ struct g5p_r74f_bell *f; ++ u32 kind, fence = U32_MAX, state = U32_MAX; ++ int ret; ++ ++ WRITE_ONCE(r->r74f_alive, r->r74f_alive + 1); ++ kind = READ_ONCE(r->r74f_active); ++ if (!kind) ++ return; ++ f = &r->r74f_bell[kind - 1]; ++ if (atomic_read_acquire(&f->state) != G5P_R74F_OPEN || ++ r->r74f_alive <= f->open_alive) ++ return; ++ /* Sole owner: exactly one decoder moves OPEN to FIRING. */ ++ if (atomic_cmpxchg(&f->state, G5P_R74F_OPEN, G5P_R74F_FIRING) != ++ G5P_R74F_OPEN) ++ return; ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_DEFERRED); ++} ++ ++static int g5p_r74f_poll(struct g5p_diag *g, u32 alive, u32 expected, ++ u32 timeout_us, u64 *elapsed_ns, u32 *raw) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u64 start = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ *raw = U32_MAX; ++ for (;;) { ++ if (READ_ONCE(g->boot->failed)) { ++ ret = -EIO; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ /* Serialize the ordinal check and exactly one R1 read with the ++ * decoder. Once ALIVE advances, no further R1 MMIO is licensed. ++ */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ break; ++ } ++ *raw = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if ((*raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ return ret; ++} ++ ++/* Absolute freshness budget from the DC3 doorbell. The first MMU pass may ++ * touch R0 only while the same active window is still demonstrably fresh. ++ */ ++ ++/* The r74e release transition, once per generation, on r74f receipts and ++ * without console output: core request clear, GPC 0 within 1.0 ms, GPC 1 ++ * within 1.8 ms. Every R1 access is serialized against ALIVE decoding; ++ * ordinal advance cancels the generation before any further R1 MMIO. ++ */ ++static int g5p_r74f_reactivate(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags; ++ u32 alive = f->arm_alive; ++ int ret; ++ ++ f->reactivated = 1; ++ f->power_start_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ f->power_ret = -EHOSTDOWN; ++ return -EHOSTDOWN; ++ } ++ writel(0, g->window[1] + 0x20000); ++ f->core_fence = readl(g->window[1] + 0x20000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, 0, G5P_R74F_OFF_US, &f->off_ns, ++ &f->off_raw); ++ if (!ret) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (READ_ONCE(r->r74f_alive) != alive) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = -EHOSTDOWN; ++ goto out; ++ } ++ /* boot_gpu_perf_seed: the firmware believes pstate q; never drop the GPC under it */ ++ writel(0x10 | (b->perf_poweron ?: ++ b->perf_seed ?: ++ 1), ++ g->window[1] + 0x1000); ++ f->gpc_fence = readl(g->window[1] + 0x1000); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ ret = g5p_r74f_poll(g, alive, ++ b->perf_poweron ?: ++ b->perf_seed ?: ++ 1, ++ G5P_R74F_ON_US, &f->on_ns, &f->on_raw); ++ } ++out: ++ f->power_ret = ret; ++ return ret; ++} ++ ++/* Process context, before any publication: no overlap with a pending ring. */ ++static int g5p_r74f_admit(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *dc = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ ++ if (!g5p_r74b_ram_only(g) || READ_ONCE(b->failed) || ++ READ_ONCE(r->r74f_stop) || ++ atomic_read(&r->r74f_bell[kind].state) != G5P_R74F_IDLE) ++ return -EPERM; ++ if (kind == G5P_R74F_DC3) ++ return READ_ONCE(r->r74f_active) ? -EBUSY : 0; ++ /* M7: the TA wakeup never shares a doorbell with a pending DC3 pair. */ ++ dma_rmb(); ++ if (READ_ONCE(r->r74f_active) != G5P_R74F_DC3 + 1 || ++ !r->r74f_bell[G5P_R74F_DC3].complete || ++ READ_ONCE(dc[0]) != READ_ONCE(dc[4])) ++ return -EBUSY; ++ return 0; ++} ++ ++static void g5p_r74f_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = g5p_r74f_ctl(b, f->kind); ++ u32 state = atomic_read_acquire(&f->state); ++ char line[3][512]; ++ ++ dma_rmb(); ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74F bell kind=%s generation=%u ring=%02x start=%u target=%u state=%s path=%s " ++ "first_ret=%d first_fence=%08x first_state=%08x attempted=%u fence=%08x " ++ "bell_state=%08x bell_ret=%d kicked=%u bells=%d cancel=%s complete=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->ring, f->start, ++ f->target, g5p_r74f_states[state < G5P_R74F_STATES ? state : 0], ++ g5p_r74f_paths[f->path], f->first_ret, f->first_fence, ++ f->first_state, f->attempted, f->fence, f->bell_state, ++ f->bell_ret, f->kicked, atomic_read(&b->r73b->r74f_bells), ++ g5p_r74f_cancels[f->cancel], f->complete); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74F ring kind=%s generation=%u arm_alive=%u open_alive=%u bell_alive=%u " ++ "alive_now=%u consumer_before=%u consumer_after=%u producer_after=%u " ++ "consumer_now=%u producer_now=%u failed=%u event_failed=%u end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->arm_alive, ++ f->open_alive, f->bell_alive, READ_ONCE(b->r73b->r74f_alive), ++ f->consumer_before, f->consumer_after, f->producer_after, ++ READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), READ_ONCE(b->failed), ++ READ_ONCE(b->event_failed)); ++ scnprintf( ++ line[2], sizeof(line[2]), ++ "R74F power kind=%s generation=%u reactivated=%u core_fence=%08x gpc_fence=%08x " ++ "off_ns=%llu on_ns=%llu off_raw=%08x on_raw=%08x power_ret=%d arm_ns=%llu " ++ "power_start_ns=%llu open_ns=%llu bell_start_ns=%llu bell_ns=%llu done_ns=%llu " ++ "end=r74f\n", ++ g5p_r74f_kinds[f->kind], f->generation, f->reactivated, ++ f->core_fence, f->gpc_fence, f->off_ns, f->on_ns, f->off_raw, ++ f->on_raw, f->power_ret, f->arm_ns, f->power_start_ns, ++ f->open_ns, f->bell_start_ns, f->bell_ns, f->done_ns); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++/* r87b receipt: the DC3 attach policy and what the host did after the publish. ++ * RAM reads only; one line per DC3 generation (console at the consume/timeout ++ * point, blob at export). ++ */ ++static void g5p_r87b_line(struct g5p_diag *g, const struct g5p_r74f_bell *f, ++ bool blob) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 policy = READ_ONCE(b->r73b->r87b_attach); ++ char line[640]; ++ ++ dma_rmb(); ++ scnprintf( ++ line, sizeof(line), ++ "R87B attach policy=%u name=%s generation=%u gate_first_ret=%d " ++ "gate_first_state=%08x gate_closed=%u path=%s cancel=%s bell_rung=%u bells=%d " ++ "reactivated=%u cons_before=%u prod_before=%u prod_pub=%u cons_after=%u " ++ "prod_after=%u consumed=%u arm_alive=%u alive_done=%u alive_delta=%u " ++ "pub_to_done_ns=%llu irqs=%d events=%u end=r87b\n", ++ policy, policy ? "self-drain" : "original", f->generation, ++ f->first_ret, f->first_state, f->first_ret == -EHOSTDOWN, ++ g5p_r74f_paths[f->path], g5p_r74f_cancels[f->cancel], f->kicked, ++ atomic_read(&b->r73b->r74f_bells), f->reactivated, ++ f->consumer_before, f->start, f->target, f->consumer_after, ++ f->producer_after, f->complete, f->arm_alive, f->alive_done, ++ f->alive_done - f->arm_alive, ++ f->done_ns >= f->arm_ns ? f->done_ns - f->arm_ns : 0, ++ atomic_read(&b->irqs), READ_ONCE(b->events)); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* A retained boundary: the publication stays, nothing is failed. */ ++static int g5p_r74f_stop(struct g5p_diag *g, u32 stop, int value) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ ++ r->r74f_stop_value = value; ++ WRITE_ONCE(r->r74f_stop, stop); ++ g5p_r74_mark(g, G5P_R74F_STOP, stop); ++ dev_emerg(g->dev, "R74F stop reason=%s value=%d failed=%u end=r74f\n", ++ g5p_r74f_stops[stop], value, READ_ONCE(g->boot->failed)); ++ return -EINPROGRESS; ++} ++ ++/* After the reactivation: open the generation, wait (bounded) for the ++ * ALIVE owner while the event drain stays alive, then cancel if unrung. ++ */ ++static int g5p_r74f_open(struct g5p_diag *g, struct g5p_r74f_bell *f) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ unsigned long flags, deadline; ++ int ret; ++ ++ ret = g5p_r74f_reactivate(g, f); ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (ret) { ++ f->cancel = ret == -EHOSTDOWN ? G5P_R74F_CANCEL_ALIVE_ADVANCED : ++ G5P_R74F_CANCEL_REACTIVATION; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ f->open_alive = r->r74f_alive; ++ f->open_ns = ktime_get_mono_fast_ns(); ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (ret) ++ return 0; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ /* The IRQ path is the sole event consumer and bell owner. */ ++ if (READ_ONCE(b->failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* Process context, command mutex held, producer already published. Returns ++ * 0 after one doorbell, 1 if consumed without one, -EINPROGRESS when the ++ * publication is retained unrung, else a fatal protocol error. ++ */ ++static int g5p_r74f_bell(struct g5p_diag *g, u32 kind, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(b, kind); ++ u32 fence = U32_MAX, state = U32_MAX, now; ++ unsigned long flags; ++ int ret, drained = 0; ++ bool ta = kind == G5P_R74F_TA; ++ bool self_drain; ++ ++ if (kind == G5P_R74F_3D) ++ return -EINVAL; /* the 3D bell is armed by g5p_r96_3d_arm(): never the DC3 / self-drain path */ ++ f->kind = kind; ++ f->ring = ta ? 0x60 : 0x6c; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ /* The immediate functional gate r74e used, without console output. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, kind + 1); ++ atomic_set(&f->state, G5P_R74F_FIRING); ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_r74f_ring(g, f, &fence, &state); ++ f->first_ret = ret; ++ f->first_fence = fence; ++ f->first_state = state; ++ /* r87b self-drain: a closed DC3 gate is only a phase sample of the ++ * firmware's GPC window. The host never repairs it: no R1+20000 clear, no ++ * GPC off/on, no bell. The caller waits RAM-only for the firmware to ++ * consume the pair. Only the DC3 kind; an active gate rang above. ++ */ ++ self_drain = ret == -EHOSTDOWN && kind == G5P_R74F_DC3 && ++ READ_ONCE(r->r87b_attach) == G5P_R87B_ATTACH_SELF_DRAIN; ++ if (self_drain) { ++ f->cancel = G5P_R74F_CANCEL_SELF_DRAIN; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (ret == -EHOSTDOWN) { ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ } else { ++ g5p_r74f_rang(r, f, ret, fence, state, G5P_R74F_PATH_IMMEDIATE); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (self_drain) { ++ drained = 1; ++ g5p_r74_mark(g, G5P_R74F_ATTACH_ARMED, state); ++ } else if (ret == -EHOSTDOWN) { ++ drained = g5p_r74f_open(g, f); ++ /* Functional reactivation precedes all console/milestone output. */ ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_ARMED : G5P_R74F_ATTACH_ARMED, ++ state); ++ g5p_r74_mark(g, ++ ta ? G5P_R74F_TA_REACTIVATED : ++ G5P_R74F_ATTACH_REACTIVATED, ++ f->power_ret); ++ } ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, ta ? G5P_R74F_TA_BELL : G5P_R74F_ATTACH_BELL, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ if (drained) ++ return drained; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_REACTIVATION || ++ f->cancel == G5P_R74F_CANCEL_ALIVE_ADVANCED) ++ return g5p_r74f_stop( ++ g, ++ ta ? G5P_R74F_STOP_TA_REACTIVATION : ++ G5P_R74F_STOP_ATTACH_REACTIVATION, ++ f->power_ret); ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, ++ ta ? G5P_R74F_STOP_TA_NOT_KICKED : ++ G5P_R74F_STOP_ATTACH_NOT_KICKED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* RAM only: the consumer this generation reached, at consume or timeout. */ ++static void g5p_r74f_consumed(struct g5p_diag *g, u32 kind) ++{ ++ struct g5p_r74f_bell *f = &g->boot->r73b->r74f_bell[kind]; ++ u32 *ctl = g5p_r74f_ctl(g->boot, kind); ++ ++ dma_rmb(); ++ f->consumer_after = READ_ONCE(ctl[0]); ++ f->producer_after = READ_ONCE(ctl[4]); ++ f->complete = f->consumer_after == f->target; ++ f->done_ns = ktime_get_mono_fast_ns(); ++ f->alive_done = READ_ONCE(g->boot->r73b->r74f_alive); ++ if (kind == G5P_R74F_DC3) ++ g5p_r87b_line(g, f, false); ++} ++ ++/* g5p_r22_mmu(): attach's first pass only; records the gate it already read. */ ++ ++/* g5p_r22_command(): an r74f boundary keeps the epoch, drain and backing. */ ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot-r74g.inc b/drivers/gpu/apple-g5p/boot-r74g.inc +new file mode 100644 +index 000000000..6ff269317 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot-r74g.inc +@@ -0,0 +1,288 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R74g. Included by boot.inc after boot-r74f.inc; r74f case 1 only. It is a ++ * diagnostic bypass on the forced-ready epoch, not power policy. ++ * ++ * The firmware DC3 handler (0x1000218c..0x10002218) writes e140/e148 for ++ * slot N, copies host_generation[N] to cached_generation[N], then issues ++ * the e138 (N << 3) | 4 catalogue invalidation and waits for it. The DC ++ * dispatcher advances the consumer only after the handler returns, so a ++ * consumed DC3 pair is the completion receipt. r74g checks the RAM-visible ++ * results and issues no host MMU MMIO. The natural TA is then published ++ * without a doorbell. A RAM-only grace (four type-0 ALIVEs, 20 ms cap) ++ * lets the firmware drain it on its own. If it stays pending, the r74f ++ * ALIVE owner gets one fenced fallback doorbell. Receipts print afterwards. ++ */ ++ ++/* perf60: with the policy held in state 0 the firmware is quiet (no ALIVE stream, no self-drain), so ++ * job 1's TA bell, owned by "the first ALIVE after the grace", would never ring. ++ */ ++/* release8 default */ ++ ++static const char *const g5p_r74g_graces[G5P_R74G_GRACES] = { ++ "none", "consumed", "alives", "wall", "fault", ++}; ++ ++static bool g5p_r74g_on(struct g5p_boot *b) ++{ ++ return g5p_r74f_on(b) && 1; ++} ++ ++static u32 g5p_r74g_root(struct g5p_boot *b, u32 off) ++{ ++ return g5p_le32(b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++static u32 g5p_r74g_shared(struct g5p_boot *b, u32 at) ++{ ++ return g5p_le32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE); ++} ++ ++/* RAM only: the DC3 ring, both records in attach order (3D, then TA), the ++ * PA[N] entries, host and cached tags, both failure latches and the ++ * root+3980 companion word, whose value 2 is firmware recovery/escape. ++ */ ++static u32 g5p_r74g_check(struct g5p_diag *g, u32 phase, const u32 *slot, ++ u64 pa, u32 tag, u32 start) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r74g_dc3 *d = &b->r73b->r74g_dc3[phase]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_DC3); ++ u32 bad = 0; ++ ++ dma_rmb(); ++ d->start = start; ++ d->target = (start + 2) & 255; ++ d->tag = tag; ++ d->expected_pa = pa; ++ d->consumer = READ_ONCE(ctl[0]); ++ d->producer = READ_ONCE(ctl[4]); ++ if (d->consumer != d->target || d->producer != d->target) ++ bad |= G5P_R74G_BAD_RING; ++ for (u32 i = 0; i < 2; i++) { ++ u32 at = b->layout.offset[0x6d] + ((start + i) & 255) * 20; ++ ++ for (u32 j = 0; j < 5; j++) { ++ d->rec[i][j] = g5p_r74g_shared(b, at + j * 4); ++ if (d->rec[i][j] != (j == 0 ? 3 : ++ j == 1 ? slot[1 - i] : ++ 0)) ++ bad |= G5P_R74G_BAD_RECORD; ++ } ++ } ++ for (u32 e = 0; e < 2; e++) { ++ u8 *p = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ ++ d->slot[e] = slot[e]; ++ d->pa[e] = (u64)g5p_le32(p + 4) << 32 | g5p_le32(p); ++ d->host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + slot[e] * 4); ++ d->cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + slot[e] * 4); ++ if (d->pa[e] != pa) ++ bad |= G5P_R74G_BAD_PA; ++ if (d->host[e] != tag) ++ bad |= G5P_R74G_BAD_HOST; ++ if (d->cached[e] != tag) ++ bad |= G5P_R74G_BAD_CACHED; ++ } ++ d->failed = READ_ONCE(b->failed); ++ d->event_failed = READ_ONCE(b->event_failed); ++ if (d->failed || d->event_failed) ++ bad |= G5P_R74G_BAD_FAILED; ++ d->companion = g5p_r74g_root(b, G5P_R74G_COMPANION); ++ d->recoveries = g5p_r74g_root(b, G5P_R74G_RECOVERIES); ++ d->classification = g5p_r74g_root(b, G5P_R74G_CLASS); ++ if (d->companion == 2) ++ bad |= G5P_R74G_BAD_COMPANION; ++ d->bad = bad; ++ d->t_ns = ktime_get_mono_fast_ns(); ++ return bad; ++} ++ ++/* A refused check is retained, never b->failed: companion 2 is firmware ++ * recovery, a cached-tag-only mismatch is the firmware tag copy, and ++ * anything else (a host tag included) is a refused RAM validation. ++ */ ++static int g5p_r74g_refuse(struct g5p_diag *g, u32 bad) ++{ ++ if (bad & G5P_R74G_BAD_COMPANION) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_COMPANION, bad); ++ if (bad == G5P_R74G_BAD_CACHED) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_TAGS, bad); ++ return g5p_r74f_stop(g, G5P_R74G_STOP_DC3_RAM, bad); ++} ++ ++/* Two bounded lines: every field at full width still ends in end=r74g. */ ++static void g5p_r74g_dc3_line(struct g5p_diag *g, u32 phase, bool blob) ++{ ++ const struct g5p_r74g_dc3 *d = &g->boot->r73b->r74g_dc3[phase]; ++ const char *name = phase ? "ta-preflight" : "dc3-consume"; ++ char line[2][512]; ++ ++ scnprintf( ++ line[0], sizeof(line[0]), ++ "R74G dc3 phase=%s bad=%02x start=%u target=%u consumer=%u producer=%u tag=%08x " ++ "slot=%u/%u host=%08x/%08x cached=%08x/%08x failed=%u event_failed=%u " ++ "companion=%u recoveries=%u class=%u host_mmu=skipped t_ns=%llu end=r74g\n", ++ name, d->bad, d->start, d->target, d->consumer, d->producer, ++ d->tag, d->slot[0], d->slot[1], d->host[0], d->host[1], ++ d->cached[0], d->cached[1], d->failed, d->event_failed, ++ d->companion, d->recoveries, d->classification, d->t_ns); ++ scnprintf( ++ line[1], sizeof(line[1]), ++ "R74G dc3-ram phase=%s pa=%llx/%llx expected_pa=%llx rec0=%x,%x,%x,%x,%x " ++ "rec1=%x,%x,%x,%x,%x end=r74g\n", ++ name, d->pa[0], d->pa[1], d->expected_pa, d->rec[0][0], ++ d->rec[0][1], d->rec[0][2], d->rec[0][3], d->rec[0][4], ++ d->rec[1][0], d->rec[1][1], d->rec[1][2], d->rec[1][3], ++ d->rec[1][4]); ++ for (u32 i = 0; i < ARRAY_SIZE(line); i++) { ++ if (blob) ++ g5p_blob_printf(g, "%s", line[i]); ++ else ++ dev_emerg(g->dev, "%s", line[i]); ++ } ++} ++ ++static void g5p_r74g_ta_line(struct g5p_diag *g, bool blob) ++{ ++ struct g5p_r73b *r = g->boot->r73b; ++ const struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 end = r->r74g_grace_end; ++ char line[256]; ++ ++ scnprintf( ++ line, sizeof(line), ++ "R74G ta grace_end=%s arm_alive=%u grace_alive=%u grace_ns=%llu opened=%u " ++ "open_alive=%u ack_skipped=%u end=r74g\n", ++ g5p_r74g_graces[end < G5P_R74G_GRACES ? end : 0], f->arm_alive, ++ r->r74g_grace_alive, r->r74g_grace_ns - f->arm_ns, ++ r->r74g_opened, f->open_alive, r->r74g_ack_skipped); ++ if (blob) ++ g5p_blob_printf(g, "%s", line); ++ else ++ dev_emerg(g->dev, "%s", line); ++} ++ ++/* Process context, command mutex held, TA producer already published once. ++ * Grace: RAM reads and sleeps only, no MMIO and no doorbell, while the IRQ ++ * thread keeps decoding. It ends at the consume, four type-0 ALIVEs after ++ * the arm, or the 20 ms wall cap. If the TA is still pending, the ++ * generation opens, and the first ALIVE decoded after that owns the single ++ * r74f fallback attempt. Returns 0 after one doorbell, 1 if consumed without ++ * one, -EINPROGRESS for a retained boundary, else a fatal error. ++ */ ++static int g5p_r74g_ta(struct g5p_diag *g, u32 start, u32 target) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_r74f_bell *f = &r->r74f_bell[G5P_R74F_TA]; ++ u32 *ctl = g5p_r74f_ctl(b, G5P_R74F_TA); ++ unsigned long flags, deadline; ++ u32 end = G5P_R74G_GRACE_NONE, now; ++ int ret = 0; ++ ++ f->kind = G5P_R74F_TA; ++ f->ring = 0x60; ++ f->start = start; ++ f->target = target; ++ f->generation = ++r->r74f_generation; ++ f->consumer_before = READ_ONCE(ctl[0]); ++ f->arm_ns = ktime_get_mono_fast_ns(); ++ spin_lock_irqsave(&b->event_lock, flags); ++ f->arm_alive = r->r74f_alive; ++ WRITE_ONCE(r->r74f_active, G5P_R74F_TA + 1); ++ atomic_set_release(&f->state, G5P_R74F_PUBLISHED); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ while (end == G5P_R74G_GRACE_NONE) { ++ ; ++ dma_rmb(); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) ++ end = G5P_R74G_GRACE_FAULT; ++ else if (READ_ONCE(ctl[0]) == target) ++ end = G5P_R74G_GRACE_CONSUMED; ++ else if (READ_ONCE(r->r74f_alive) - f->arm_alive >= ++ G5P_R74G_GRACE_ALIVES) ++ end = G5P_R74G_GRACE_ALIVES_END; ++ else if (ktime_get_mono_fast_ns() - f->arm_ns >= ++ (u64)G5P_R74G_GRACE_US * 1000) ++ end = G5P_R74G_GRACE_WALL; ++ else ++ usleep_range(100, 200); ++ } ++ spin_lock_irqsave(&b->event_lock, flags); ++ r->r74g_grace_end = end; ++ r->r74g_grace_alive = r->r74f_alive; ++ r->r74g_grace_ns = ktime_get_mono_fast_ns(); ++ dma_rmb(); ++ if (end == G5P_R74G_GRACE_FAULT) { ++ f->cancel = G5P_R74F_CANCEL_PROTOCOL; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else if (READ_ONCE(ctl[0]) == target) { ++ /* Drained by the firmware policy loop: never ring. */ ++ f->cancel = G5P_R74F_CANCEL_CONSUMED; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } else { ++ r->r74g_opened = 1; ++ f->open_alive = r->r74f_alive; ++ f->open_ns = r->r74g_grace_ns; ++ /* The next decoded type-0 ALIVE may now take ownership. */ ++ atomic_set_release(&f->state, G5P_R74F_OPEN); ++ /* ta1_host_bell: the host owns it instead, in the same lock section a decoder would. */ ++ if ((1) && atomic_cmpxchg(&f->state, G5P_R74F_OPEN, ++ G5P_R74F_FIRING) == G5P_R74F_OPEN) { ++ u32 fence = U32_MAX, state = U32_MAX; ++ int rung; ++ ++ f->bell_alive = r->r74f_alive; ++ f->attempted = 1; ++ f->bell_start_ns = ktime_get_mono_fast_ns(); ++ rung = g5p_r74f_ring(g, f, &fence, &state); ++ g5p_r74f_rang(r, f, rung, fence, state, ++ G5P_R74F_PATH_IMMEDIATE); ++ } ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (end == G5P_R74G_GRACE_FAULT) ++ ret = -EIO; ++ deadline = jiffies + msecs_to_jiffies(G5P_R74F_BELL_MS); ++ while (atomic_read_acquire(&f->state) == G5P_R74F_OPEN && ++ time_before(jiffies, deadline)) { ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) { ++ ret = -EIO; ++ break; ++ } ++ usleep_range(100, 200); ++ } ++ /* FIRING exists only inside the decoder's event_lock section. */ ++ spin_lock_irqsave(&b->event_lock, flags); ++ if (atomic_read(&f->state) == G5P_R74F_OPEN) { ++ f->cancel = ret ? G5P_R74F_CANCEL_PROTOCOL : ++ G5P_R74F_CANCEL_NO_ALIVE; ++ atomic_set_release(&f->state, G5P_R74F_CANCELLED); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ now = atomic_read_acquire(&f->state); ++ g5p_r74_mark(g, G5P_R74G_TA_GRACE, end); ++ if (r->r74g_opened) ++ g5p_r74_mark(g, G5P_R74G_TA_FALLBACK, ++ now == G5P_R74F_RANG ? f->bell_ret : -ECANCELED); ++ g5p_r74f_line(g, f, false); ++ g5p_r74g_ta_line(g, false); ++ if (ret) ++ return ret; ++ if (now == G5P_R74F_CANCELLED) { ++ if (f->cancel == G5P_R74F_CANCEL_CONSUMED) ++ return 1; ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_NOT_CONSUMED, ++ -ETIMEDOUT); ++ } ++ if (f->kicked) ++ return 0; ++ if (f->bell_ret == -EALREADY) ++ return 1; ++ if (f->bell_ret == -EHOSTDOWN) ++ return g5p_r74f_stop(g, G5P_R74G_STOP_TA_WINDOW_MISSED, ++ -EHOSTDOWN); ++ return f->bell_ret; ++} ++ ++/* r73b-export, process context, RAM only. */ +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +new file mode 100644 +index 000000000..6a62f2eb0 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -0,0 +1,1665 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Experimental G2 stages. Included after the bounded META slave primitives. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "boot-layout.h" ++#include "render-user-pmset.h" ++#include "render-user-recover.h" ++#include "fender.h" ++#include "readback.h" ++#include "meta71c114.h" ++#include "latency123.h" ++#include "perf-envelope.h" ++ ++/* Boot deltas are selected before map; no live render-time reconfiguration. */ ++ ++ ++ ++/* R46 (diag-fable51 T7): the first SOFT_RESET write of the host reset FIFO ++ * also asserts bits 56-59, VBS/VCE/TE3/TILING_CORE in the PBE2_IN_XE ++ * layout. Apple's mask 00e7fffffffffc1d is exactly the other layout's ++ * MASKFULL, in which those bits do not exist and TE/GPP/UVS/TLA/PM/VDM are ++ * already asserted; expected to be a no-op. The unchanged second write ++ * releases them. The FIFO forbids a readback here. ++ */ ++#define G5P_BOOT_RESET1 0x00e7fffffffffc1dULL ++#define G5P_BOOT_R46_RESET_BITS (0xfULL << 56) ++ ++/* R72b: isolate Apple meta+0x6c8 while retaining dc7 transport. */ ++ ++/* The event ring's stamp events (type 1) dump once per epoch, then a count at each power of two. */ ++ ++/* R73: mirror Apple's auxiliary firmware-ring returns on IRQ174. */ ++ ++/* Latency114: the pstate-1 frequency word meta+0x71c (meta71c114.h). */ ++ ++/* Latency123 (latency123.h): arm A boot words, and the live ctl+0x04 tick-service mask. */ ++ ++/* perf60 clock (perf-envelope.h): the firmware's own pstate request, pinned before release. */ ++/* release8 default */ ++/* GPU overclock (gpu-oc.h, docs/power/ds-gpu-dvfs.md §9 R5): PSINFO record 5 = stock record 4 ++ * with only the PLL multiplier moved (no voltage change), written before release while the GPC is at ++ * state 1, the firmware's table extended to state 5, the release powering on at 5. Needs ++ * boot_gpu_perf 500 and boot_gpu_perf_poweron 1. A thermal stop (drm-render-v3.inc) reboots the ++ * firmware at state 4 when a SoC zone reaches boot_gpu_oc_temp_mc. Record 5 is volatile PMGR ++ * state (LLB rewrites the table only at SoC reset; iBoot reads it once to build the ADT before us) ++ * and is cleared on every boot that does not overclock. ++ */ ++ ++static bool ++ g5p_oc_tripped; /* the thermal stop fired: every later epoch of this boot runs state 4 */ ++ ++static const char *const g5p_oc_zone_names[] = { "apple-a9-soc0", ++ "apple-a9-soc1", ++ "apple-a9-tcc2" }; ++ ++/* The hottest of the three SoC zones (millidegrees); the number of zones read. */ ++static u32 g5p_oc_temp(int *max_mc, const char **zone) ++{ ++ u32 n = 0; ++ ++ *max_mc = INT_MIN; ++ *zone = "none"; ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_zone_names); i++) { ++ struct thermal_zone_device *tz = ++ thermal_zone_get_zone_by_name(g5p_oc_zone_names[i]); ++ int t; ++ ++ if (IS_ERR_OR_NULL(tz) || thermal_zone_get_temp(tz, &t)) ++ continue; ++ n++; ++ if (t > *max_mc) { ++ *max_mc = t; ++ *zone = g5p_oc_zone_names[i]; ++ } ++ } ++ return n; ++} ++ ++static int g5p_oc_read(struct g5p_diag *g, u32 n, u32 w[4]) ++{ ++ for (u32 i = 0; i < 4; i++) ++ if (regmap_read(g->pmgr, G5P_PSINFO_REC(n) + 4 * i, &w[i])) ++ return -EIO; ++ return 0; ++} ++ ++/* Write record 5 from the live record 4 and read it back; the GPC is at state 1 (map time). ++ * Slot 5 is read first: nonzero and not a record this driver writes -> refused, left untouched. ++ */ ++ ++/* A boot without the overclock clears a record 5 this driver wrote; anything else stays. */ ++static void g5p_oc_clear(struct g5p_diag *g) ++{ ++ u32 r4[4], r5[4]; ++ ++ if (!g->pmgr || g5p_oc_read(g, 4, r4) || ++ g5p_oc_read(g, G5P_OC_STATE, r5) || ++ !(r5[0] || r5[1] || r5[2] || r5[3])) ++ return; ++ if (!g5p_oc_is_ours(r4, r5)) { ++ dev_info( ++ g->dev, ++ "GPUPERF oc slot 5 not ours, left: %08x %08x %08x %08x end=gpuperf\n", ++ r5[0], r5[1], r5[2], r5[3]); ++ return; ++ } ++ for (u32 i = 0; i < 4; i++) ++ regmap_write(g->pmgr, G5P_PSINFO_REC(G5P_OC_STATE) + 4 * i, 0); ++ g5p_oc_read(g, G5P_OC_STATE, r5); ++ dev_info( ++ g->dev, ++ "GPUPERF oc record 5 cleared (no overclock this epoch) word0=%08x end=gpuperf\n", ++ r5[0]); ++} ++ ++/* perf60 pin >100, the host's own route: power the GPC on at q (R1+0x1000 = 0x10 | q) in the release, ++ * as the GPC1 request the kext and we already make, with q in place of 1; the policy record carries ++ * the seed-1 words (+0x28 = +0x2c = q) and stays "never sampled" (+0 = 0), so the boot sampler only ++ * initialises; it derives +0x34 and its clock entry priv+0x2c4 from Fender (0x1001c838). ++ */ ++/* release8 default */ ++/* perf60 pin >100 (perf-envelope.h g5p_perf_seed): the boot sampler applies the pin before ALIVE. */ ++ ++/* perf60 stage 4 (render-user-pmset.h): firmware-GiB pages for per-context PM records. */ ++/* release8 default */ ++/* perf60 A/B (docs/gpu/experiments/perf60-clock.md): the firmware's per-tick full flush alone. */ ++ ++/* Latched by map, before firmware sees any pointers. Zero preserves round 9. */ ++ ++/* Explicit decoded VA input only: zero is a valid address when valid=true. ++ * Never infer an address or context from revision-dependent e150 bits. ++ */ ++static unsigned long long mmu_fault_va; ++ ++static const bool boot_experiment = 1; ++ ++struct g5p_dma_page { ++ u8 *cpu; ++ dma_addr_t dma; ++}; ++ ++struct g5p_boot { ++ u32 r27_variant, r27_mode_writes, r46_reset, r50_mode, r50_writes; ++ ++ bool table_clean, preinstall_context, compute_boot_vm; ++ u32 compute_slot; ++ ++ bool compute_numerical, user_compute_owned; ++ ++ struct g5p_boot_layout layout; ++ struct g5p_dma_page image[G5P_IMAGE_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page root[G5P_ROOT_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page shared[G5P_SHARED_SIZE / G5P_GPU_PAGE]; ++ struct g5p_dma_page render_meta[2]; ++ /* perf60 stage 4 (render-user-pmset.h): boot_pm_sets per-context PM sets, 3 pages each */ ++ u32 pm_sets; ++ struct g5p_dma_page pm_set[G5P_PMSET_MAX * G5P_PMSET_PAGES]; ++ struct g5p_dma_page catalogue, directory, zero, leaf[4]; ++ struct g5p_dma_page sink_directory, sink_leaf, sink_page; ++ u32 setup_variant; ++ bool sink_ready, sink_dumped; ++ u32 leaf_index[4]; ++ u32 leaves, pages, mapped, fifo, events, alive_word; ++ u32 fender_raw[ARRAY_SIZE(g5p_fender_regs)]; ++ u32 rgx_raw[ARRAY_SIZE(g5p_rgx_regs)]; ++ const char *snapshot_stage; ++ bool snapshot_valid, rgx_valid; ++ u32 rgx_count; ++ int fifo_error; ++ int irq; ++ atomic_t irqs; ++ spinlock_t event_lock; ++ void __iomem *swirq; ++ bool maps_ready, armed, copied, released, alive, failed, irq_ready; ++ bool release_ack, release_ack_attempted; ++ bool fault_va_valid; ++ u64 fault_va; ++ bool work_ready, work_submitted, work_complete; ++ u32 work_target, control_target; ++ u32 dc_variant, work_variant; ++ u32 r72_meta6c8; ++ u32 meta71c; ++ u32 fw_power_words; ++ u32 gpu_perf; ++ u32 perf_seed; /* boot_gpu_perf_seed: the seeded pstate q, 0 = not seeded */ ++ u32 perf_seed_mode; /* boot_gpu_perf_seed as latched (1..5) */ ++ u32 perf_poweron; /* boot_gpu_perf_poweron: the pstate q the release requests, 0 = 1 as before */ ++ u32 gpu_oc_mhz; /* boot_gpu_oc_mhz as applied (state 5), 0 = no overclock */ ++ /* record 5's voltage codes */ ++ u64 seed_stamp, ++ seed_gpc1; /* seed 3/4: the stamp and the GPC1 write (arch timer) */ ++ u32 meta6d4; ++ u32 r78_ctrl0; ++ u32 r73_aux_drain; ++ u32 r73_aux_entries[6]; ++ u32 r73_aux_hash[6]; ++ u32 r73_aux_first[6][3]; ++ u32 r73_aux_last[6][3]; ++ struct g5p_r73b *r73b; ++ bool irq_capped; ++ bool dc_observe; ++ ++ bool control_ready, control_submitted, control_complete; ++ bool irq_tested, stopped, parked; ++ bool control_timed_out, work_timed_out, event_failed; ++ bool context_submitted, context_complete, context_timed_out; ++ u32 context_target; ++ /* The epoch's last recovery packet (render-user-recover.h), under event_lock. */ ++ struct g5p_fw_recovery fw_recovery; ++ u32 fw_recoveries; ++ u32 event_n[10]; /* events drained this epoch by type (9: 9 and above) */ ++}; ++ ++#include "boot-r73b.inc" ++ ++/* The fault-status subset is read-only, including while META reset is held. ++ * Never access R0 with GPC off/busy; no slave-port transaction is involved. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage); ++ ++static int g5p_boot_alloc_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ p->cpu = dmam_alloc_coherent(g->dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ /* There is no GPU bus translation in the recovered GART path. */ ++ if ((p->dma & (G5P_GPU_PAGE - 1)) || p->dma >= BIT_ULL(40) || ++ dma_to_phys(g->dev, p->dma) != p->dma) { ++ dev_err(g->dev, ++ "unsupported DMA translation/alignment at %pad\n", ++ &p->dma); ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ g->boot->pages++; ++ return 0; ++} ++ ++/* Deliberately map one16KiB page only, not the iOS all-address fallback. ++ * Own distinct directory/leaf/backing; do not alias firmware tables or e160. ++ * No live table surgery: this helper runs only during first map construction. ++ */ ++static int g5p_boot_sink_init(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ; ++ ; ++ if (!b->catalogue.cpu || b->maps_ready || b->armed || b->released || ++ b->sink_ready || b->sink_directory.cpu || b->sink_leaf.cpu || ++ b->sink_page.cpu) ++ return -EPERM; ++ if (g5p_le32(b->catalogue.cpu)) ++ return -EEXIST; ++ ret = g5p_boot_alloc_page(g, &b->sink_directory); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_leaf); ++ if (ret) ++ return ret; ++ ret = g5p_boot_alloc_page(g, &b->sink_page); ++ if (ret) ++ return ret; ++ /* Round12 iOS optional dummy-tree leaf flags0: valid writable PA|1. ++ * Only cat[0]/dir[0]/leaf[0] are installed; every other hole stays absent. ++ */ ++ g5p_put64(b->sink_leaf.cpu, g5p_leaf_entry(b->sink_page.dma, 0)); ++ g5p_put64(b->sink_directory.cpu, g5p_directory_entry(b->sink_leaf.dma)); ++ dma_wmb(); ++ g5p_put32(b->catalogue.cpu, g5p_catalogue_entry(b->sink_directory.dma)); ++ dma_wmb(); ++ b->mapped++; ++ b->sink_ready = true; ++ g5p_mmu_record( ++ g, ++ "MMU sink-installed VA=0 bytes=4000 context=0 leaf_flags=0; pre-release " ++ "containment experiment\n"); ++ g5p_blob_printf( ++ g, ++ "MMU sink PC=%pad cat0=%08x dir=%pad dir0=%016llx leaf=%pad pte0=%016llx " ++ "backing=%pad; initially zero\n", ++ &b->catalogue.dma, g5p_catalogue_entry(b->sink_directory.dma), ++ &b->sink_directory.dma, g5p_directory_entry(b->sink_leaf.dma), ++ &b->sink_leaf.dma, g5p_leaf_entry(b->sink_page.dma, 0), ++ &b->sink_page.dma); ++ return 0; ++} ++ ++static int g5p_boot_map_page(struct g5p_diag *g, u64 va, u64 pa, u32 attributes) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ u32 ti = (va >> 14) & 127, i; ++ int ret; ++ ++ if ((va | pa) & (G5P_GPU_PAGE - 1)) ++ return -EINVAL; ++ /* This experiment only owns the recovered firmware heap's catalogue slot. */ ++ if (ci != ((G5P_IMAGE_VA >> 30) & 1023)) ++ return -ERANGE; ++ for (i = 0; i < b->leaves; i++) ++ if (b->leaf_index[i] == di) ++ break; ++ if (i == b->leaves) { ++ if (i == ARRAY_SIZE(b->leaf)) ++ return -ENOSPC; ++ ret = g5p_boot_alloc_page(g, &b->leaf[i]); ++ if (ret) ++ return ret; ++ b->leaf_index[i] = di; ++ g5p_put64(b->directory.cpu + di * 8, ++ g5p_directory_entry(b->leaf[i].dma)); ++ b->leaves++; ++ } ++ if (g5p_le32(b->leaf[i].cpu + ti * 8)) ++ return -EEXIST; ++ g5p_put64(b->leaf[i].cpu + ti * 8, g5p_leaf_entry(pa, attributes)); ++ g5p_put32(b->catalogue.cpu + ci * 4, ++ g5p_catalogue_entry(b->directory.dma)); ++ b->mapped++; ++ return 0; ++} ++ ++#include "mmu-attrs.inc" ++#include "mmu-tables.inc" ++ ++static int g5p_boot_region(struct g5p_diag *g, struct g5p_dma_page *p, u32 size, ++ u64 va) ++{ ++ u32 offset; ++ u32 attributes = va == G5P_IMAGE_VA ? 0x10 : 0x14; ++ int ret; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) { ++ ret = g5p_boot_alloc_page(g, p); ++ if (ret) ++ return ret; ++ ret = g5p_boot_map_page(g, va + offset, p->dma, attributes); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void g5p_boot_copy(struct g5p_dma_page *p, const u8 *data, u32 size) ++{ ++ u32 offset; ++ ++ for (offset = 0; offset < size; offset += G5P_GPU_PAGE, p++) ++ memcpy(p->cpu, data + offset, G5P_GPU_PAGE); ++} ++ ++static u8 *g5p_boot_shared(struct g5p_boot *b, u32 slot) ++{ ++ u32 off = b->layout.offset[slot]; ++ ++ return b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static int g5p_boot_maps(struct g5p_diag *g) ++{ ++ struct g5p_boot *b; ++ u8 *root = NULL, *shared = NULL; ++ bool m71c_wrote; ++ u32 i, m71c, seed; ++ int ret; ++ ++ if (!g->loaded || !g->id_checked || PAGE_SIZE != G5P_GPU_PAGE) ++ return -EPERM; ++ if (g->core_id != 0x011b0602 || g->core_revision != 0x00090f00) ++ return -ENODEV; ++ if (g->boot) ++ return g->boot->maps_ready ? 0 : -EIO; ++ b = devm_kzalloc(g->dev, sizeof(*b), GFP_KERNEL); ++ if (!b) ++ return -ENOMEM; ++ g->boot = b; ++ b->r50_mode = (0); ++ ; ++ b->r27_variant = (0); ++ ; ++ b->r46_reset = (0); ++ ; ++ b->r72_meta6c8 = (1); ++ b->r78_ctrl0 = (0); ++ if (b->r78_ctrl0 && b->r78_ctrl0 != 0x5f) { ++ b->failed = true; ++ return -EINVAL; ++ }; ++ b->r73_aux_drain = (0); ++ if (b->r73_aux_drain > 1 || (b->r73_aux_drain && !1)) { ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->release_ack = (0); ++ b->table_clean = (1); ++ b->compute_boot_vm = (0); ++ b->compute_slot = 0 ? 0 : (7); ++ b->preinstall_context = (0) || 0; ++ ; ++ b->setup_variant = (1); ++ ; ++ b->fault_va = READ_ONCE(mmu_fault_va); ++ b->fault_va_valid = (0); ++ ; ++ b->dc_variant = (7); ++ b->work_variant = (0); ++ b->dc_observe = (0); ++ ; ++ /* iOS: meta+0x71c = 340, the pstate-1 frequency in MHz. */ ++ b->meta71c = (340); ++ m71c = g5p_m71c_check(b->meta71c, 7); ++ if (m71c != G5P_M71C_OK) { ++ dev_err(g->dev, ++ "META71C refused requested=%u variant=%u why=%s before_release=1 end=meta71c\n", ++ b->meta71c, b->dc_variant, g5p_m71c_whys[m71c]); ++ b->failed = true; ++ return -EINVAL; ++ } ++ b->fw_power_words = (0); ++ b->gpu_perf = 400; ++ b->meta6d4 = false; ++ seed = (0); ++ b->gpu_oc_mhz = 0; ++ ; ++ b->perf_poweron = ++ (1) ? (0 ? G5P_OC_STATE : g5p_perf_seed_q(b->gpu_perf)) : 0; ++ b->perf_seed = seed && !b->perf_poweron ? g5p_perf_seed_q(b->gpu_perf) : ++ 0; ++ b->perf_seed_mode = b->perf_seed ? seed : 0; ++ if (!(g5p_perf_valid(b->gpu_perf) || ++ (0 && b->gpu_perf == G5P_OC_PIN)) || ++ (b->gpu_perf && b->fw_power_words) || seed > 5 || (1) > 1 || ++ (b->gpu_perf > 100 && !(1)) || b->meta6d4 > 1 || ++ (b->meta6d4 && !b->gpu_perf)) { ++ dev_err(g->dev, ++ "GPUPERF refused pin=%u power_words=%u gpc_any_pstate=%u meta6d4=%u seed=%u " ++ "before_release=1 end=gpuperf\n", ++ b->gpu_perf, b->fw_power_words, (1), b->meta6d4, seed); ++ b->failed = true; ++ return -EINVAL; ++ } ++ if (b->fw_power_words > G5P_L123_FULL || (0) > G5P_L123_TICK_MASK) { ++ dev_err(g->dev, ++ "L123 refused power_words=%u tick_mask=%u before_release=1 end=l123\n", ++ b->fw_power_words, (0)); ++ b->failed = true; ++ return -EINVAL; ++ } ++ ret = g5p_r73b_latch(g); ++ if (ret) { ++ b->failed = true; ++ return ret; ++ }; ++ dev_info( ++ g->dev, ++ "DC matrix latched variant=%u observe=%u (changes require reboot)\n", ++ b->dc_variant, b->dc_observe); ++ dev_info( ++ g->dev, ++ "COMPUTE VM latched slot=%u boot_vm=%u preinstall=%u (changes require reboot)\n", ++ b->compute_slot, b->compute_boot_vm, b->preinstall_context); ++ spin_lock_init(&b->event_lock); ++ atomic_set(&b->irqs, 0); ++ ret = g5p_plan_layout(&b->layout); ++ if (ret) ++ goto out; ++ root = kvzalloc(G5P_ROOT_SIZE, GFP_KERNEL); ++ shared = kvzalloc(G5P_SHARED_SIZE, GFP_KERNEL); ++ if (!root || !shared) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_boot_alloc_page(g, &b->catalogue); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->directory); ++ if (ret) ++ goto out; ++ ret = g5p_boot_alloc_page(g, &b->zero); ++ if (ret) ++ goto out; ++ ret = g5p_boot_sink_init(g); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->image, G5P_IMAGE_SIZE, G5P_IMAGE_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->root, G5P_ROOT_SIZE, G5P_ROOT_VA); ++ if (ret) ++ goto out; ++ ret = g5p_boot_region(g, b->shared, G5P_SHARED_SIZE, G5P_SHARED_VA); ++ if (ret) ++ goto out; ++ /* R22 metadata mappings are installed before release, only behind its ++ * default-off gate. Compute/boot defaults allocate no additional pages. ++ */ ++ { ++ ret = g5p_boot_region(g, b->render_meta, 2 * G5P_GPU_PAGE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE); ++ if (ret) ++ goto out; ++ b->pm_sets = min((8), G5P_PMSET_MAX); ++ if (b->pm_sets) { ++ ret = g5p_boot_region(g, b->pm_set, ++ b->pm_sets * G5P_PMSET_PAGES * ++ G5P_GPU_PAGE, ++ G5P_PMSET_VA); ++ if (ret) ++ goto out; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-sets boot sets=%u va=%#llx..%#llx before_release=1 end=v3\n", ++ b->pm_sets, G5P_PMSET_VA, ++ g5p_pmset_va(b->pm_sets, 0)); ++ } ++ } ++ ret = 0; ++ if (ret) ++ goto out; ++ /* c80758: Fender (2 pages), AIC timer, AIC software interrupt. */ ++ for (i = 0; i < 4; i++) { ++ u64 pa = i < 2 ? 0x201e00000ULL + i * G5P_GPU_PAGE : ++ i == 2 ? 0x20e100000ULL : ++ 0x20e104000ULL; ++ ++ ret = g5p_boot_map_page(g, G5P_PIO_VA + i * G5P_GPU_PAGE, pa, ++ 0x18); ++ if (ret) ++ goto out; ++ } ++ g5p_make_boot_config(&g->image, &b->layout); ++ g5p_make_init(&b->layout, root, shared, &g->image, b->catalogue.dma, ++ arch_timer_read_counter()); ++ g5p_dc_init_variant(&b->layout, root, shared, 7); ++ /* meta+0x784 is the abstime OFFSET the firmware adds to its timebase (0x100221e8 = PIO-1 ++ * counter + priv+688, loaded from meta+0x784 at 0x1001ff58); iOS writes ++ * ml_get_abstime_offset(). Linux's timebase is the counter itself: 0. (make_init's absolute ++ * stamp put the firmware's clock ~uptime ahead of every host stamp in policy+0x8f8.) ++ */ ++ g5p_put64(g5p_object(&b->layout, root, shared, 0x4d) + 0x784, 0); ++ if (g5p_boot_ctrl0(b)) ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x5e), ++ g5p_boot_ctrl0(b)); ++ dev_info( ++ g->dev, ++ "R78 ctrl0 requested=%08x installed=%08x before_release=1 end=r78\n", ++ g5p_boot_ctrl0(b), ++ g5p_le32(g5p_object(&b->layout, root, shared, 0x5e))); ++ ; ++ ; ++ { ++ g5p_put32(g5p_object(&b->layout, root, shared, 0x4d) + 0x6c8, ++ 1); ++ }; ++ m71c = g5p_m71c_install(&b->layout, root, shared, b->meta71c, ++ &m71c_wrote); ++ dev_info( ++ g->dev, ++ "META71C requested=%u installed=%08x variant=%u wrote=%u before_release=1 " ++ "end=meta71c\n", ++ b->meta71c, m71c, b->dc_variant, m71c_wrote); ++ if (m71c != b->meta71c) { ++ ret = -EIO; ++ goto out; ++ } ++ if (b->fw_power_words) ++ g5p_l123_arm_a(&b->layout, root, shared, b->fw_power_words); ++ if (b->gpu_perf) { ++ u8 *pol = g5p_object(&b->layout, root, shared, G5P_PERF_POLICY); ++ u8 *mt = g5p_object(&b->layout, root, shared, G5P_PERF_META); ++ u8 *ct = g5p_object(&b->layout, root, shared, G5P_PERF_CTL); ++ ++ g5p_perf_install(&b->layout, root, shared, b->gpu_perf); ++ { ++ g5p_oc_clear(g); ++ } ++ if (b->meta6d4) ++ g5p_put32(mt + 0x6d4, 1); ++ if (b->perf_poweron) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_poweron, ++ false); ++ dev_info( ++ g->dev, ++ "GPUPERF poweron q=%u policy28=%u/%u policy0=%u policy4=%u seed=%u ignored " ++ "before_release=1 end=gpuperf\n", ++ b->perf_poweron, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), seed); ++ } else { ++ dev_info( ++ g->dev, ++ "GPUPERF poweron ignored pin=%u (pins 200..400 only) before_release=1 end=gpuperf\n", ++ b->gpu_perf); ++ } ++ if (seed && !b->perf_seed && !b->perf_poweron) ++ dev_info( ++ g->dev, ++ "GPUPERF seed ignored pin=%u (seeds pins 200..400 only) before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf); ++ if (b->perf_seed) { ++ g5p_perf_seed(&b->layout, root, shared, b->perf_seed, ++ seed >= 2); ++ /* seed 5: meta+0x784 is the abstime OFFSET the firmware adds to its timebase ++ * (0x100221e8 = PIO-1 counter + priv+688, loaded from meta+0x784 at 0x1001ff58); ++ * iOS writes ml_get_abstime_offset(), make_init the absolute counter, so every ++ * policy+0x8f8 stamp lay ~counter in the firmware's past. Linux has no offset. ++ */ ++ if (seed == 5) ++ g5p_put64(mt + 0x784, 0); ++ dev_info( ++ g->dev, ++ "GPUPERF seed mode=%u q=%u policy28=%u/%u policy30=%u policy8=%u policy0=%u " ++ "policy4=%u meta784=%llx before_release=1 end=gpuperf\n", ++ seed, b->perf_seed, g5p_le32(pol + 0x28), ++ g5p_le32(pol + 0x2c), g5p_le32(pol + 0x30), ++ g5p_le32(pol + 8), g5p_le32(pol + 0), ++ g5p_le32(pol + 4), ++ (u64)g5p_le32(mt + 0x784) | ++ (u64)g5p_le32(mt + 0x788) << 32); ++ } ++ dev_info( ++ g->dev, ++ "GPUPERF boot pin=%u table=%u/%u/%u,%u,%u,%u,%u limits=%u/%u/%u ovr=%u/%u " ++ "policy688=%u/%u policy6f0=%u/%u policy4=%u policy8=%u/%u q=%u mhz=%u " ++ "gpc_any_pstate=%u seed=%u/%u policy0=%u policy28=%u/%u before_release=1 " ++ "end=gpuperf\n", ++ b->gpu_perf, g5p_le32(mt + 0x710), g5p_le32(mt + 0x714), ++ g5p_le32(mt + 0x718), g5p_le32(mt + 0x71c), ++ g5p_le32(mt + 0x720), g5p_le32(mt + 0x724), ++ g5p_le32(mt + 0x728), g5p_le32(ct + 0x1028), ++ g5p_le32(ct + 0x102c), g5p_le32(ct + 0x1030), ++ g5p_le32(ct + 0x1038), g5p_le32(ct + 0x1040), ++ g5p_le32(pol + 0x688), g5p_le32(pol + 0x68c), ++ g5p_le32(pol + 0x6f0), g5p_le32(pol + 0x6f4), ++ g5p_le32(pol + 4), g5p_le32(pol + 8), ++ g5p_le32(pol + 0xc), b->gpu_perf / 100, ++ b->gpu_perf / 100 < ARRAY_SIZE(g5p_perf_mhz) ? ++ g5p_perf_mhz[b->gpu_perf / 100] : ++ b->gpu_oc_mhz, ++ (1), seed, b->perf_seed, g5p_le32(pol + 0), ++ g5p_le32(pol + 0x28), g5p_le32(pol + 0x2c)); ++ } ++ g5p_put32(g5p_object(&b->layout, root, shared, G5P_L123_CTL) + 4, (0)); ++ { ++ u8 *meta = g5p_object(&b->layout, root, shared, G5P_L123_META); ++ ++ dev_info( ++ g->dev, ++ "L123 boot power_words=%u meta6d4=%u table=%u/%u/%u,%u,%u,%u,%u policy4=%u " ++ "ctl04=%u limits=%u/%u/%u before_release=1 end=l123\n", ++ b->fw_power_words, g5p_le32(meta + 0x6d4), ++ g5p_le32(meta + 0x710), g5p_le32(meta + 0x714), ++ g5p_le32(meta + 0x718), g5p_le32(meta + 0x71c), ++ g5p_le32(meta + 0x720), g5p_le32(meta + 0x724), ++ g5p_le32(meta + 0x728), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_POLICY) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 4), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1028), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x102c), ++ g5p_le32(g5p_object(&b->layout, root, shared, ++ G5P_L123_CTL) + ++ 0x1030)); ++ /* the accessor's licence bound (mmio-access.md §4.2 step 4): meta+0x710 as written */ ++ WRITE_ONCE(g->mmio_max_pstate, ++ g5p_mmio_max_pstate(g5p_le32(meta + 0x710))); ++ }; ++ ; ++ g5p_boot_copy(b->image, g->image.data, G5P_IMAGE_SIZE); ++ g5p_boot_copy(b->root, root, G5P_ROOT_SIZE); ++ g5p_boot_copy(b->shared, shared, G5P_SHARED_SIZE); ++ dma_wmb(); ++ ret = 0; ++ if (ret) ++ goto out; ++ g5p_tables_clean(g, "boot-before-release", NULL, 0); ++ b->maps_ready = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 maps: %u x 16KiB allocations; %u PTEs, %u leaves, PC=%pad\n", ++ b->pages, b->mapped, b->leaves, &b->catalogue.dma); ++ dev_info( ++ g->dev, ++ "G2 init: root=%llx control=%llx event-ctl=%llx event-data=%llx\n", ++ g5p_object_va(&b->layout, 0x50), ++ g5p_object_va(&b->layout, 0x5f), ++ g5p_object_va(&b->layout, 0x78), ++ g5p_object_va(&b->layout, 0x7c)); ++ dev_info( ++ g->dev, ++ "G2 experimental zero policy fields; fault observation only in map\n"); ++out: ++ kvfree(root); ++ kvfree(shared); ++ if (ret) ++ b->failed = true; ++ g5p_ds_hook_maps(g, ret); ++ return ret; ++} ++ ++#include "boot-r74f.inc" ++#include "boot-r74g.inc" ++ ++static u32 g5p_transport_event_limit(void); ++ ++static int g5p_boot_drain(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ dev_err(g->dev, "G2 invalid event indices %u/%u\n", consumer, ++ producer); ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ for (n = 0; n < 16 && consumer != producer; n++) { ++ u32 words[11], off = b->layout.offset[0x74] + consumer * 44, t, ++ n; ++ u8 *src; ++ unsigned int i; ++ bool want; ++ ++ /* Heartbeat116: an R74 epoch bounds ring entries per window ++ * in the IRQ thread instead (heartbeat116.h EVENTS). ++ */ ++ if (!g5p_hb116_on(b) && ++ b->events >= g5p_transport_event_limit()) { ++ ret = -EOVERFLOW; ++ goto out; ++ } ++ /* Entries may straddle DMA pages. */ ++ for (i = 0; i < ARRAY_SIZE(words); i++) { ++ u32 at = off + i * 4; ++ ++ src = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ words[i] = g5p_le32(src); ++ } ++ /* A healthy meta6c8 setup loop can emit thousands of identical ++ * ALIVEs. Keep its first four packets and the exact aggregate, ++ * without making printk throughput part of the experiment. ++ * R74 also quiets a moved loop: each new word1 prints once. ++ */ ++ want = !b->r73b || words[0] || b->events < 4 || ++ (words[1] != 0x8000a120 && g5p_r74_word1(b, words[1])); ++ /* Stamp events (type 1, two per job) dump once per epoch, then only count: ++ * at a few jobs per frame they were most of the log and evicted the faults ++ * (release8 bench, 306 k lines). Every other type dumps as before. ++ */ ++ t = min_t(u32, words[0], ARRAY_SIZE(b->event_n) - 1); ++ n = ++b->event_n[t]; ++ if (words[0] == 1 && n > 1 && !(0)) { ++ if (is_power_of_2(n)) ++ g5p_job_printk( ++ g->dev, ++ "G2 events type=1 n=%u: stamp events counted, dumped once (boot_event_dump 1: each)\n", ++ n); ++ want = false; ++ } ++ /* Alive (0) and stamp (1) events are routine, one or more per job: job_log. Every other ++ * type (recovery, page request, channel error, fault) prints as before. ++ */ ++ if (want && words[0] > 1) { ++ dev_info(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ print_hex_dump(KERN_INFO, ++ "G2 event raw: ", DUMP_PREFIX_NONE, 16, ++ 4, words, sizeof(words), false); ++ } else if (want) { ++ g5p_job_printk(g->dev, ++ "G2 event index=%u type=%u word1=%08x\n", ++ consumer, words[0], words[1]); ++ g5p_job_hex("G2 event raw: ", words, sizeof(words)); ++ }; ++ if (g->event_notify) ++ g->event_notify(g->event_cookie, words); ++ /* c99788 routes these to recovery/page-request/channel-error paths. */ ++ if (words[0] != 0 && words[0] != 1) { ++ WRITE_ONCE(b->failed, true); ++ switch (words[0]) { ++ case G5P_FW_EVENT_RECOVERY: { ++ /* The kext's fields (render-user-recover.h). */ ++ struct g5p_fw_recovery rec = ++ g5p_fw_recovery_decode(words); ++ ++ b->fw_recovery = rec; ++ b->fw_recoveries++; ++ dev_err(g->dev, ++ "G2 recovery event slot=%u flag=%u code=%08x w18=%08x w20=%08x w24=%08x n=%u: " ++ "the firmware stopped; iOS restarts the GPU\n", ++ rec.slot, rec.flag, rec.code, rec.w18, ++ rec.w20, rec.w24, b->fw_recoveries); ++ break; ++ } ++ case 7: ++ dev_err(g->dev, ++ "G2 page-request event payload=%08x/%08x/%08x; address encoding unresolved\n", ++ words[1], words[2], words[3]); ++ break; ++ case 8: ++ dev_err(g->dev, ++ "G2 channel-error event kind=%u engine=%u token=%08x stamp=%08x; no " ++ "acknowledgment command guessed\n", ++ words[1], words[2], words[3], words[4]); ++ break; ++ } ++ dev_err(g->dev, ++ "G2 unsupported/fault event=%u; epoch failed, entry retained (the render device " ++ "recovers it if it may)\n", ++ words[0]); ++ ret = -EIO; ++ break; ++ } ++ if (words[0] == 0 && READ_ONCE(b->released) && ++ !READ_ONCE(b->stopped)) { ++ b->alive_word = words[1]; ++ WRITE_ONCE(b->alive, true); ++ } ++ b->events++; ++ consumer = (consumer + 1) & 255; ++ /* c99788 uses a full barrier: finish slot reads before returning it. */ ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ /* Return the ALIVE slot before ringing work. No logging or ++ * other MMIO may be inserted between these two operations. ++ */ ++ if (!words[0] && g5p_r74f_on(b) && ++ READ_ONCE(b->r73b->owner) == G5P_R73B_OWNER_IRQ && ++ READ_ONCE(b->released) && !READ_ONCE(b->stopped)) ++ g5p_r74f_alive(g); ++ } ++out: ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++static irqreturn_t g5p_boot_irq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ struct g5p_boot *b = g->boot; ++ u32 n = atomic_inc_return(&b->irqs); ++ int ret = 0; ++ ++ if (b->r73b) ++ return g5p_r73b_irq(g, irq, n); ++ if (n > g5p_transport_event_limit()) { ++ /* Never leave the software source asserted, even when capped. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (!b->irq_capped) { ++ b->irq_capped = true; ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ disable_irq_nosync(irq); ++ dev_err(g->dev, ++ "G2 IRQ174 storm capped; reboot required\n"); ++ } ++ return IRQ_HANDLED; ++ } ++ /* KPF level6G: Apple uses this AIC ACK before all ring drains. */ ++ writel(0x4000, b->swirq + 0x94); ++ if (READ_ONCE(b->released)) ++ ret = g5p_boot_drain(g); ++ if (ret) { ++ WRITE_ONCE(b->event_failed, true); ++ WRITE_ONCE(b->failed, true); ++ } ++ return IRQ_HANDLED; ++} ++ ++static int g5p_boot_irq_setup(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b->irq_ready) ++ return 0; ++ b->swirq = devm_ioremap(g->dev, 0x20e104000ULL, 0x100); ++ if (!b->swirq) ++ return -ENOMEM; ++ b->irq = platform_get_irq(to_platform_device(g->dev), 4); ++ if (b->irq < 0) ++ return b->irq; ++ ret = devm_request_threaded_irq(g->dev, b->irq, NULL, g5p_boot_irq, ++ IRQF_ONESHOT, "g5p-meta", g); ++ if (ret) ++ return ret; ++ b->irq_ready = true; ++ dev_info(g->dev, "G2 IRQ174 registered as Linux IRQ %d\n", b->irq); ++ return 0; ++} ++ ++/* Cache first, then report: sysfs status never issues GPU MMIO. */ ++static void g5p_boot_snapshot(struct g5p_diag *g, const char *stage, ++ enum g5p_readback_state mode) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 i, state; ++ ++ b->snapshot_stage = stage; ++ b->snapshot_valid = false; ++ b->rgx_valid = false; ++ b->rgx_count = 0; ++ g5p_power_snapshot(g, stage); ++ dev_info(g->dev, "G2 snapshot %s: R1 first\n", stage); ++ for (i = 0; i < ARRAY_SIZE(g5p_fender_regs); i++) { ++ b->fender_raw[i] = readl(g->window[1] + g5p_fender_regs[i]); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x\n", g5p_fender_regs[i], ++ b->fender_raw[i]); ++ } ++ b->snapshot_valid = true; ++ state = b->fender_raw[4]; ++ /* GPC=1 did not make META a18 safe while bit32 was held. */ ++ b->rgx_count = g5p_readback_rgx_count(state, mode); ++ if (!b->rgx_count) { ++ dev_info(g->dev, "G2 R0 snapshot skipped: R1+1000=%08x\n", ++ state); ++ return; ++ } ++ for (i = 0; i < b->rgx_count; i++) { ++ dev_info(g->dev, "G2 R R0+%05x\n", g5p_rgx_regs[i]); ++ b->rgx_raw[i] = readl(g->rgx + g5p_rgx_regs[i]); ++ dev_info(g->dev, "G2 raw R0+%05x=%08x\n", g5p_rgx_regs[i], ++ b->rgx_raw[i]); ++ } ++ b->rgx_valid = true; ++} ++ ++static int g5p_fender_write(void *ctx, u32 reg, u32 value) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ ++ /* recovery108: never an R1 access after a FINAL teardown retired the device. */ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 W R1+%05x=%08x\n", reg, value); ++ writel(value, g->window[1] + reg); ++ raw = readl(g->window[1] + reg); ++ dev_info(g->dev, "G2 raw R1+%05x=%08x (write fence)\n", reg, raw); ++ return 0; ++} ++ ++static int g5p_fender_poll(void *ctx, u32 reg, u32 mask, u32 expected) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g5p_r8_r1_allowed()) ++ return -ESHUTDOWN; ++ dev_info(g->dev, "G2 poll R1+%05x mask=%08x expected=%08x (100ms)\n", ++ reg, mask, expected); ++ ret = readl_poll_timeout(g->window[1] + reg, raw, ++ (raw & mask) == expected, 10, 100000); ++ dev_info(g->dev, "G2 poll raw R1+%05x=%08x ret=%d\n", reg, raw, ret); ++ return ret; ++} ++ ++static int g5p_r74e_power_poll(struct g5p_diag *g, u32 expected, u32 timeout_us, ++ u64 *elapsed_ns, u32 *samples) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u64 start = ktime_get_mono_fast_ns(); ++ u32 raw = U32_MAX, n = 0; ++ int ret; ++ ++ for (;;) { ++ if (READ_ONCE(b->alive)) { ++ ret = -EHOSTDOWN; ++ break; ++ } ++ raw = readl(g->window[1] + 0x1000); ++ n++; ++ if ((raw & 0x1f) == expected) { ++ ret = 0; ++ break; ++ } ++ if (ktime_get_mono_fast_ns() - start >= ++ (u64)timeout_us * 1000) { ++ ret = -ETIMEDOUT; ++ break; ++ } ++ udelay(10); ++ } ++ *elapsed_ns = ktime_get_mono_fast_ns() - start; ++ /* wake112: the reads taken; 0 means raw is still the U32_MAX initializer, not a read. */ ++ *samples = n; ++ r->early_power_raw = raw; ++ return ret; ++} ++ ++/* wake112 (r74e112.h): a failed deferred proof is the cold power abort, absorbing. */ ++static void g5p_r74e_fail(struct g5p_r73b *r, u32 why) ++{ ++ r->early_proof_why = why; ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ atomic_set_release(&r->early_proof, G5P_PX_FAILED); ++} ++ ++/* The once-owner takes the deferred proof right before its one R1+0x1000 read. ++ * false: that read must not happen, and the proof has failed. ++ */ ++static bool g5p_r74e_take(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ struct g5p_px_pre p; ++ u32 why; ++ ++ if (atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, G5P_PX_TAKEN) != ++ G5P_PX_DEFERRED) ++ return false; ++ p = (struct g5p_px_pre){ ++ .gen = r->early_proof_gen, ++ .gen_now = g5p_ds_gen(), ++ .alive = READ_ONCE(b->alive), ++ .released = READ_ONCE(b->released), ++ .stopped = READ_ONCE(b->stopped), ++ .failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ .r1_allowed = g5p_r8_r1_allowed(), ++ }; ++ r->early_proof_gen_now = p.gen_now; ++ why = g5p_px_check(&p); ++ if (why) ++ g5p_r74e_fail(r, why); ++ return !why; ++} ++ ++/* The owner's read decides: only the state field 1 licenses the release. */ ++static void g5p_r74e_settle(struct g5p_r73b *r, u32 raw) ++{ ++ if (!g5p_px_on(raw)) { ++ g5p_r74e_fail(r, G5P_PXW_STATE); ++ return; ++ } ++ WRITE_ONCE(r->early_release_ready, true); ++ atomic_set_release(&r->early_proof, G5P_PX_PROVEN); ++} ++ ++/* No ALIVE wait ends with the proof still owed: close it, no register access. ++ * true: it was still owed. ++ */ ++static bool g5p_r74e_close(struct g5p_diag *g) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ ++ if (!r || atomic_cmpxchg(&r->early_proof, G5P_PX_DEFERRED, ++ G5P_PX_TAKEN) != G5P_PX_DEFERRED) ++ return false; ++ g5p_r74e_fail(r, G5P_PXW_UNTAKEN); ++ return true; ++} ++ ++/* boot_gpu_perf_seed 3/4: the sampler's last-sample time policy+0x8f8 = now. Seed 3 stamps at the ++ * start of the release (flown 2026-11-03: 38 ms before the stream's end, the window was missed); ++ * seed 4 stamps as the last host write before the one that lets META run, the GPC1 request ++ * R1+0x1000 = 0x11 (writel orders the RAM store before it). ++ */ ++static void g5p_perf_seed_stamp(struct g5p_boot *b, u32 mode) ++{ ++ u32 o; ++ ++ if (mode == 4 ? b->perf_seed_mode < 4 : b->perf_seed_mode != mode) ++ return; /* the GPC1 stamp serves seeds 4 and 5 */ ++ o = b->layout.offset[G5P_PERF_POLICY] + 0x8f8; ++ b->seed_stamp = arch_timer_read_counter(); ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)b->seed_stamp)); ++ o += 4; ++ WRITE_ONCE(*(u32 *)(b->root[o / G5P_GPU_PAGE].cpu + o % G5P_GPU_PAGE), ++ cpu_to_le32((u32)(b->seed_stamp >> 32))); ++ wmb(); ++} ++ ++static int g5p_r74e_fender_start(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxp latch; ++ bool on_ran = false; ++ u32 samples = ++ 0; /* wake112: the reads of the poll that ran last (the printed raw) */ ++ int ret; ++ ++ r->early_power_start_ns = ktime_get_mono_fast_ns(); ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_r74e_power_poll(g, 0, 1000, &r->early_off_ns, ++ &samples); ++ if (!ret) { ++ g5p_perf_seed_stamp(b, 4); ++ ret = g5p_fender_write(g, 0x1000, ++ 0x10 | (b->perf_poweron ?: 1)); ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ if (!ret) { ++ on_ran = true; ++ /* poweron: the first request above pstate 1, a voltage step; a longer bound */ ++ ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, ++ b->perf_poweron ? 10000 : 1800, ++ &r->early_on_ns, &samples); ++ if (b->perf_poweron) ++ dev_emerg( ++ g->dev, ++ "GPUPERF poweron q=%u r1_1000=%08x on_ns=%llu ret=%d end=gpuperf\n", ++ b->perf_poweron, r->early_power_raw, ++ r->early_on_ns, ret); ++ } ++ /* wake112: only ALIVE preempting the on-poll before its first read defers; the ++ * first-ALIVE owner owes the proof. A sampled on-poll (0x11, ...) aborts as before. ++ */ ++ latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples); ++ if (latch == G5P_PXP_DEFER) { ++ r->early_defer_ret = ret; ++ r->early_proof_gen = g5p_ds_gen(); ++ ret = 0; ++ atomic_set_release(&r->early_proof, G5P_PX_DEFERRED); ++ } ++ r->early_power_ret = ret; ++ if (latch == G5P_PXP_ABORT) { ++ WRITE_ONCE(r->early_power_abort, true); ++ WRITE_ONCE(r->fast_window, false); ++ } else if (latch == G5P_PXP_READY) { ++ WRITE_ONCE(r->early_release_ready, true); ++ } ++ dev_emerg( ++ g->dev, ++ "R74E power off_ns=%llu on_ns=%llu raw=%08x ret=%d release_ready=%u alive=%u " ++ "sampled=%u end=r74e\n", ++ r->early_off_ns, r->early_on_ns, r->early_power_raw, ret, ++ r->early_release_ready, READ_ONCE(b->alive), samples); ++ if (latch == G5P_PXP_DEFER) ++ dev_emerg( ++ g->dev, ++ "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u end=r74e112\n", ++ r->early_proof_gen, r->early_defer_ret, ++ r->early_power_raw, READ_ONCE(b->alive)); ++ return ret; ++} ++ ++static void g5p_fifo_word(struct g5p_diag *g, u32 word) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 raw; ++ ++ if (b->fifo_error) ++ return; ++ if (b->fifo < 0x10000 || b->fifo >= 0x10200) { ++ b->fifo_error = -EOVERFLOW; ++ return; ++ } ++ writel(word, g->window[1] + b->fifo); ++ raw = readl(g->window[1] + b->fifo); ++ if (raw != word) { ++ dev_err(g->dev, ++ "G2 FIFO verify R1+%05x actual=%08x expected=%08x\n", ++ b->fifo, raw, word); ++ b->fifo_error = -EIO; ++ } ++ b->fifo += 4; ++} ++ ++static void g5p_fifo32(struct g5p_diag *g, u32 reg, u32 value, u32 flags) ++{ ++ dev_info(g->dev, "G2 FIFO R0+%05x=%08x flags=%02x\n", reg, value, ++ flags); ++ g5p_fifo_word(g, (reg & 0xff00ffff) | (flags & 0xff) << 16); ++ g5p_fifo_word(g, value); ++} ++ ++static void g5p_fifo64(struct g5p_diag *g, u32 reg, u64 value) ++{ ++ g5p_fifo32(g, reg + 4, value >> 32, 0); ++ g5p_fifo32(g, reg, value, 0xff); ++} ++ ++static int g5p_boot_fifo_dump(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off, raw, end = min(b->fifo, 0x10200U); ++ ++ if (end <= 0x10000) ++ return -ENODATA; ++ dev_info(g->dev, "G2 FIFO dump R1+10000..%05x (<=128 words)\n", ++ end - 4); ++ for (off = 0x10000; off < end; off += 4) { ++ raw = readl(g->window[1] + off); ++ dev_info(g->dev, "G2 FIFO raw R1+%05x=%08x\n", off, raw); ++ } ++ return 0; ++} ++ ++static u64 g5p_snapshot64(struct g5p_boot *b, u32 reg) ++{ ++ u32 i; ++ ++ for (i = 0; i + 1 < b->rgx_count; i++) ++ if (g5p_rgx_regs[i] == reg && g5p_rgx_regs[i + 1] == reg + 4) ++ return (u64)b->rgx_raw[i + 1] << 32 | b->rgx_raw[i]; ++ return 0; ++} ++ ++static void g5p_release_observe(void *ctx, const char *stage) ++{ ++ ; ++ ; ++} ++ ++/* A diagnostic notification ACK only, never engine repair or a retry loop. ++ * Snapshot first, require the active wrapper, ACK only200 once, fence and ++ * re-sample. A cleared event alone is not evidence the underlying fault healed. ++ */ ++static int g5p_release_ack(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 state, event, after; ++ ++ if (!b->release_ack) ++ return 0; ++ if (!b->released || b->stopped || b->release_ack_attempted) ++ return -EPERM; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_fault_snapshot(g, "release-ack-before"); ++ g5p_mmu_progress(g, "release-ack-event", 0x130); ++ event = readl(g->rgx + 0x130); ++ b->release_ack_attempted = true; ++ if (event & 0x200) { ++ g5p_mmu_progress(g, "release-ack-write", 0x138); ++ writel(0x200, g->rgx + 0x138); ++ /* Ordered same-device read fences the single write. */ ++ g5p_mmu_progress(g, "release-ack-fence", 0x130); ++ after = readl(g->rgx + 0x130); ++ g5p_mmu_record( ++ g, ++ "MMU release-ack mask=00000200 before=%08x after=%08x; notification only\n", ++ event, after); ++ } else { ++ g5p_mmu_record(g, ++ "MMU release-ack skipped=no-event before=%08x\n", ++ event); ++ } ++ g5p_fault_snapshot(g, "release-ack-after"); ++ return 0; ++} ++ ++static int g5p_boot_program_stream(struct g5p_diag *g, bool release) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ u64 reset; ++ int ret; ++ ++ if (b->armed && !release) ++ return -EALREADY; ++ ret = g5p_boot_irq_setup(g); ++ if (ret) ++ return ret; ++ g5p_boot_snapshot(g, release ? "before-release" : "before-arm", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ ; ++ ret = 0; ++ if (ret) ++ goto out; ++ /* Slave-port access remains forbidden for the rest of this boot. */ ++ g->reset_transition = true; ++ /* c9fa10: core request false, GPC 0. Establish an actual state edge; ++ * wake left GPC at 1, so writing 1 again did not prove FIFO execution. ++ * No R0 accesses between this point and the successful active-state poll. ++ */ ++ dev_info(g->dev, "G2 Fender park: core=0, GPC=0\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ goto out; ++ /* c9fc10: core request true before prepareBoot c9feb4. */ ++ dev_info(g->dev, "G2 Fender prepare: core=20, require GPC=0\n"); ++ ret = g5p_fender_hold(&io); ++ if (ret) ++ goto out; ++ dev_info(g->dev, ++ "G2 init stream: master boot, release=%u; FIFO reset\n", ++ release); ++ ret = g5p_fender_write(g, 0x10, 0); ++ if (ret) ++ goto out; ++ /* setMode(3), after Fender prepare cleared +10, before firmware runs. ++ * Both arm and release execute prepare; never reapply after ALIVE. ++ */ ++ ; ++ ; ++ ; ++ g5p_fender_write(g, 0, 0x40000000); ++ ; ++ ; ++ b->fifo = 0x10000; ++ b->fifo_error = 0; ++ ; ++ ; ++ g5p_fifo64(g, 0x100, ++ G5P_BOOT_RESET1 | (0 ? G5P_BOOT_R46_RESET_BITS : 0)); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x00e7fe071ffffc1dULL); ++ ; ++ ; ++ g5p_fifo64(g, 0x100, 0x100000000ULL); ++ ; ++ ; ++ g5p_fifo32(g, 0xbf8, 1, 0xff); ++ ; ++ ; ++ g5p_fifo32(g, 0xe140, 0, 0xff); ++ ; ++ ; ++ g5p_fifo64(g, 0xe148, b->catalogue.dma >> 12); ++ ; ++ ; ++ g5p_fifo64(g, 0xe200, 3); ++ ; ++ ; ++ g5p_fifo64(g, 0xe208, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fifo64(g, 0xe210, 0xb42dd24b4bd22db4ULL); ++ ; ++ ; ++ g5p_fender_write(g, 0x2010, 0xbf); ++ ; ++ ; ++ g5p_fender_write(g, 0x2014, 0x233); ++ ; ++ ; ++ g5p_fifo64(g, 0xe160, b->zero.dma); ++ ; ++ ; ++ g5p_fifo64(g, 0xbb8, 0); ++ ; ++ ; ++ if (release) { ++ g5p_fifo64(g, 0x100, 0); ++ ; ++ } ++ g5p_fifo_word(g, 0xffff0000); ++ ; ++ ; ++ ret = b->fifo_error; ++ if (ret) ++ goto out; ++ dma_wmb(); ++ dev_info( ++ g->dev, ++ "G2 FIFO stored and verified: %u words (not execution proof)\n", ++ (b->fifo - 0x10000) / 4); ++ if (release) { ++ WRITE_ONCE(b->released, true); ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_FIFO, G5P_R73B_NONE, 0); ++ ret = 0; ++ if (ret) ++ goto out; ++ } ++ /* c9f954: core request false, then GPC 1. */ ++ dev_info(g->dev, "G2 Fender execute: core=0, GPC=1 (from 0)\n"); ++ if (release && !(b->r73b && 1)) { ++ g5p_perf_seed_stamp( ++ b, ++ 4); /* the non-r74e start: as close as this path gets */ ++ if (b->seed_stamp) ++ b->seed_gpc1 = arch_timer_read_counter(); ++ } ++ ret = release && b->r73b && 1 ? g5p_r74e_fender_start(g) : ++ release ? g5p_fender_start_observed_q(&io, g5p_release_observe, ++ b->perf_poweron ?: 1) : ++ g5p_fender_start(&io); ++ if (release) ++ g5p_r73b_crumb(b, G5P_R73B_RELEASE_GPC, G5P_R73B_NONE, ret); ++ if (ret) ++ goto out; ++ /* r74e: after GPC1 is active, wait for ALIVE without optional MMIO. */ ++ if (release && b->r73b && 1) ++ return 0; ++ usleep_range(10, 20); ++ g5p_boot_snapshot(g, release ? "after-release" : "after-arm", ++ release ? G5P_READBACK_WRAPPER_ONLY : ++ G5P_READBACK_RESET_HELD); ++ ; ++ /* Only the fault-status subset above is sampled after release. */ ++ if (release) ++ return g5p_release_ack(g); ++ if (!b->rgx_valid) ++ return -EIO; ++ reset = g5p_snapshot64(b, 0x100); ++ dev_info(g->dev, "G2 verify reset actual=%016llx expected=%016llx\n", ++ reset, 0x100000000ULL); ++ dev_info( ++ g->dev, ++ "G2 expected catalogue=%llx scratch=%llx SLC=3 (not read: state safety)\n", ++ (u64)b->catalogue.dma >> 12, (u64)b->zero.dma); ++ if (reset != 0x100000000ULL) ++ return -EIO; ++ dev_info( ++ g->dev, ++ "G2 reset-held gate verified; remaining FIFO effects await firmware event\n"); ++ b->armed = true; ++ return 0; ++out: ++ /* A slow r74e epoch is scoreable without another unsafe MMIO read. */ ++ if (release && b->r73b && 1 && READ_ONCE(b->r73b->early_power_abort)) ++ return ret; ++ /* Wrapper-only snapshot if GPC remains off/busy. */ ++ g5p_boot_snapshot(g, "transition-or-FIFO-failed", ++ G5P_READBACK_WRAPPER_ONLY); ++ return ret; ++} ++ ++static int g5p_boot_arm(struct g5p_diag *g) ++{ ++ return g5p_boot_program_stream(g, false); ++} ++ ++static int g5p_boot_load(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 off; ++ ++ /* c9f1bc/ca43b0: segment and LDR configuration pairs are supplied in ++ * mapped RAM. Polling MSLVCTRL1 in the reset-held state locks this SoC. ++ */ ++ for (off = 0; off < G5P_IMAGE_SIZE; off += G5P_GPU_PAGE) ++ if (memcmp(b->image[off / G5P_GPU_PAGE].cpu, ++ g->image.data + off, G5P_GPU_PAGE)) ++ return -EBADMSG; ++ dma_wmb(); ++ b->copied = true; ++ ; ++ dev_info( ++ g->dev, ++ "G2 RAM load verified bytes=%x segment_pairs=13 LDR_pairs=27; no slave-port MMIO\n", ++ G5P_IMAGE_SIZE); ++ return 0; ++} ++ ++static int g5p_boot_release(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u64 seed_ts = 0; ++ int ret; ++ ++ if (b->released) ++ return -EALREADY; ++ /* recovery108: fail closed before the release touches anything. */ ++ ret = g5p_ds_hook_release_gate(g); ++ if (ret) ++ return ret; ++ g5p_r73b_release_entry(g); ++ if (b->r73b && 1) { ++ ret = g5p_r73e_prepare(g); ++ if (ret) ++ return ret; ++ } ++ dev_info(g->dev, ++ "G2 release: META master boot; no GPU job submitted\n"); ++ /* Reproduce the complete original stream, including its final reset release. ++ * Do not assume the experimental arm stage preserves a FIFO read cursor. ++ */ ++ /* boot_gpu_perf_seed 3: the sampler's last-sample time = now, as close to the release as the ++ * host gets (perf-envelope.h); the firmware divides (now - policy+0x8f8) by 24000 * policy+4. ++ */ ++ if (b->perf_seed_mode >= 3) { ++ seed_ts = arch_timer_read_counter(); ++ g5p_perf_seed_stamp(b, 3); ++ } ++ ret = g5p_boot_program_stream(g, true); ++ if (seed_ts) { ++ u64 end = arch_timer_read_counter(), ++ rate = arch_timer_get_rate(); ++ ++ /* always printed on a seed 3/4 boot, before ALIVE is awaited */ ++ dev_emerg( ++ g->dev, ++ "GPUPERF seed mode=%u stamp=%llx release_to_gpc1_us=%llu stamp_to_gpc1_us=%llu " ++ "gpc1_to_stream_end_us=%llu stamp_to_stream_end_us=%llu period_us=16000 " ++ "before_alive=1 end=gpuperf\n", ++ b->perf_seed_mode, b->seed_stamp, ++ b->seed_gpc1 ? ++ div_u64((b->seed_gpc1 - seed_ts) * 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? div_u64((b->seed_gpc1 - b->seed_stamp) * ++ 1000000ULL, ++ rate) : ++ 0, ++ b->seed_gpc1 ? ++ div_u64((end - b->seed_gpc1) * 1000000ULL, ++ rate) : ++ 0, ++ div_u64((end - b->seed_stamp) * 1000000ULL, rate)); ++ } ++ g5p_r73b_release_exit(g, ret); ++ g5p_ds_hook_release(g, ret); ++ if (ret) ++ return ret; ++ if (b->r73b && 1) ++ return g5p_r73c_fast_control(g); ++ return 0; ++} ++ ++static int g5p_boot_events_once(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 3 * HZ; ++ u64 submit_wait_start = 0; ++ u32 pass = 0, submit_wait_passes = 0; ++ int ret; ++ ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_EVENTS, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->alive)) { ++ if (b->r73b && 1) { ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ if (ret == -EAGAIN) ++ submit_wait_start = ++ ktime_get_mono_fast_ns(); ++ while (ret == -EAGAIN && ++ ktime_get_mono_fast_ns() - ++ submit_wait_start < ++ 100ULL * NSEC_PER_MSEC) { ++ usleep_range(50, 100); ++ ret = g5p_r73e_alive_submit( ++ g, G5P_R73B_OWNER_EVENTS); ++ submit_wait_passes++; ++ } ++ if (submit_wait_passes) ++ dev_info( ++ g->dev, ++ "R73C alive-submit synchronize wait_ns=%llu passes=%u ret=%d submitted=%u end=r73c\n", ++ ktime_get_mono_fast_ns() - ++ submit_wait_start, ++ submit_wait_passes, ret, ++ READ_ONCE( ++ b->control_submitted)); ++ if (ret) ++ return ret; ++ } ++ dev_info( ++ g->dev, ++ "G2 ALIVE type=0 word1=%08x events=%u IRQ174=%d\n", ++ b->alive_word, b->events, ++ atomic_read(&b->irqs)); ++ g5p_r41_bridge_firmware(g); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, ++ "G2 no alive event: events=%u IRQ174=%d; preserve baseline\n", ++ b->events, atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++/* recovery108: every ALIVE wait (release fast path and the events command) is an ALIVE event. */ ++static int g5p_boot_events(struct g5p_diag *g) ++{ ++ int ret = g5p_boot_events_once(g); ++ ++ /* wake112: no ALIVE wait ends with an R74E proof still owed (r74e112.h). */ ++ if (g5p_r74e_close(g)) { ++ dev_emerg( ++ g->dev, ++ "R74E112 close gen=%u why=untaken ret=%d end=r74e112\n", ++ g->boot->r73b->early_proof_gen, ret); ++ if (!ret) ++ ret = -EHOSTDOWN; ++ } ++ g5p_ds_hook_alive(g, ret); ++ return ret; ++} ++ ++static int g5p_boot_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (b && b->stopped) ++ return -ESHUTDOWN; ++ if (sysfs_streq(command, "map")) ++ return g5p_boot_maps(g); ++ /* Read-only diagnostics remain available after a latched boot failure. */ ++ if (b && g->powered && g->fender_ready && g->id_checked) { ++ if (sysfs_streq(command, "snapshot")) { ++ g5p_boot_snapshot(g, "explicit", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); ++ return 0; ++ } ++ if (sysfs_streq(command, "fifo")) ++ return g5p_boot_fifo_dump(g); ++ } ++ if (!boot_experiment || !b || !b->maps_ready || b->failed) ++ return -EPERM; ++ if (sysfs_streq(command, "arm")) { ++ ret = g5p_boot_arm(g); ++ } else if (sysfs_streq(command, "load") && b->armed && !b->released && ++ !b->copied) { ++ ret = g5p_boot_load(g); ++ g5p_ds_hook_load(g, ret); ++ } else if (sysfs_streq(command, "release") && b->copied) { ++ ret = g5p_boot_release(g); ++ } else if (sysfs_streq(command, "events") && b->released) { ++ ret = g5p_boot_events(g); ++ g5p_r73b_events_exit(g, ret); ++ } else { ++ return -EPERM; ++ } ++ if (ret && ret != -EALREADY) ++ b->failed = true; ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/compute-user-layout.h b/drivers/gpu/apple-g5p/compute-user-layout.h +new file mode 100644 +index 000000000..78b459832 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/compute-user-layout.h +@@ -0,0 +1,207 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CDM channel and per-job firmware objects. Layout helpers perform no I/O. ++ * The kernel owns the channel, control watermarks, pointer ring, work record, ++ * completion microsequence, timestamp queue, stamps and dependency object. ++ * GPU program instructions are provided by userspace, never embedded here. ++ * ++ * Compute uses the context's TA MMU slot. The scheduler serializes TA and CDM ++ * work across contexts, and a lost CDM job blocks the channel until retirement ++ * or recovery. Stamp identity zero is safe because the TA event latch is armed ++ * only while executing a TA phase; completion also requires coherent stamps. ++ */ ++#ifndef G5P_COMPUTE_USER_LAYOUT_H ++#define G5P_COMPUTE_USER_LAYOUT_H ++ ++#define G5P_CU_ENGINE 2U ++#define G5P_CU_OFFSET (G5P_R22_3D + 0x3000U) /* in the shared region */ ++#define G5P_CU_BYTES 0x1000U ++#define G5P_CU_W 0x400U ++#define G5P_CU_W_BYTES 0x290U /* through the timestamps at W+0x280..0x288 */ ++#define G5P_CU_SKU 0x800U ++#define G5P_CU_SKU_BYTES 0xc0U ++#define G5P_CU_TQ 0x900U ++#define G5P_CU_START 0x940U /* W258: the retirement mirror */ ++#define G5P_CU_END 0x944U /* W25c: finish33's stamp */ ++#define G5P_CU_DEP 0x980U ++#define G5P_CU_DEP_BYTES 0x280U ++#define G5P_CU_IDENTITY 0U ++#define G5P_CU_SEQ_MAX 0x7fffffffU ++ ++/* Firmware configuration W+0x244 and compute record type. */ ++#define G5P_CU_W244 0x00a00004U ++#define G5P_CU_TAG 3U ++ ++/* The validated register images of one job (compute-user-validate.h fills it). */ ++struct g5p_cu_regs { ++ u64 stream; /* W+0x70 -> R0+0x480 */ ++ u64 terminate; /* the stream's terminate word: W+0x214, W+0x23c */ ++ u64 state[4]; /* W+0x1ec..0x204 */ ++ u64 tail; /* W+0x60 -> R0+0x498, W+0x20c */ ++ u32 config[2]; /* W+0x10, W+0x14 -> R0+0x4c0, 0x4c8 */ ++ /* {code | data << 32}: W+0x21c, W+0x22c, W+0x68 -> R0+0x4b8 */ ++ u64 store, resume, term; ++ u32 store1, resume1; /* W+0x224 -> R0+0x4b0, W+0x234 */ ++}; ++ ++/* Job `seq` (1-based) occupies pointer-ring entry (seq - 1) % 16 and publishes seq % 16. */ ++static inline u32 g5p_cu_ring_index(u32 seq) ++{ ++ return (seq - 1) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_cu_endpoint(u32 seq) ++{ ++ return seq % G5P_WORK_SLOTS; ++} ++ ++/* The channel at its epoch start: header, ring capacity, timestamp queue, dependency object ++ * (g5p_compute_dependency()); watermarks, ring and stamps zero. `p` is the channel's ++ * G5P_CU_BYTES, `meta` its META address. ++ */ ++static inline void g5p_cu_channel_init(u8 *p, u32 meta) ++{ ++ u8 *d = p + G5P_CU_DEP; ++ ++ memset(p, 0, G5P_CU_BYTES); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_CU_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_CU_TQ + 4, meta + G5P_CU_TQ); ++ g5p_put64(d + 0x30, 1); ++ g5p_put64(d + 0x48, ~0ULL); ++ g5p_put32(d + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(d + off + 8, 0xf); ++} ++ ++/* The SKU microsequence of job `seq`: start32, timestamp, wait COMPUTE_FINISHED (event 4), ++ * wait (R0+0x4a0 & 1) == 1, timestamp, finish33 (W25c = W260), the backlink, end. ++ * Each sequence reference uses the current job number. ++ */ ++static inline u32 g5p_cu_sku(u8 *sku, u32 meta, u32 global, u32 slot, u32 seq) ++{ ++ u32 w = meta + G5P_CU_W, end = meta + G5P_CU_END; ++ const u32 words[] = { ++ 0x32, ++ w + 0x10, ++ w + 0x60, ++ global + 0x720, ++ meta, ++ slot, ++ 0, ++ 0, ++ 0, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ 0x80000018, ++ w + 0x278, ++ w + 0x280, ++ w + 0x280, ++ seq, ++ 0x101, ++ 0x004a000a, ++ 1, ++ 1, ++ 0x18, ++ w + 0x278, ++ w + 0x280, ++ w + 0x284, ++ seq, ++ 0x33, ++ global + 0x720, ++ meta, ++ slot, ++ w + 0x1e8, ++ 0, ++ 0, ++ seq, ++ end, ++ seq, ++ (u32)-0x6c, ++ 0x40000017, ++ }; ++ ++ memset(sku, 0, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(words); i++) ++ g5p_put32(sku + i * 4, words[i]); ++ return ARRAY_SIZE(words) * 4; ++} ++ ++/* Every per-job object of job `seq` on slot `slot`: W, SKU and its pointer-ring entry. The ++ * channel header, the dependency object and the timestamp queue stay as initialised. ++ */ ++static inline void g5p_cu_job(u8 *p, u32 meta, u32 global, u32 slot, u32 seq, ++ const struct g5p_cu_regs *k) ++{ ++ u8 *w = p + G5P_CU_W; ++ ++ memset(w, 0, G5P_CU_W_BYTES); ++ g5p_put32(w, G5P_CU_TAG); ++ g5p_put32(w + 8, slot); ++ g5p_put32(w + 0xc, meta + G5P_CU_DEP); ++ g5p_put32(w + 0x10, k->config[0]); ++ g5p_put32(w + 0x14, k->config[1]); ++ g5p_put64(w + 0x60, k->tail); ++ g5p_put64(w + 0x68, k->term); ++ g5p_put64(w + 0x70, k->stream); ++ /* The six selected-resource entries at W+0x78..0xa7 stay zero (disabled). */ ++ g5p_put32(w + 0x1e0, meta + G5P_CU_SKU); ++ g5p_put32(w + 0x1e4, G5P_CU_SKU_BYTES); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x1ec + i * 8, k->state[i]); ++ g5p_put64(w + 0x20c, k->tail); ++ g5p_put64(w + 0x214, k->terminate); ++ g5p_put64(w + 0x21c, k->store); ++ g5p_put32(w + 0x224, k->store1); ++ g5p_put64(w + 0x22c, k->resume); ++ g5p_put32(w + 0x234, k->resume1); ++ g5p_put64(w + 0x23c, k->terminate); ++ g5p_put32(w + 0x244, G5P_CU_W244); ++ g5p_put32(w + 0x258, meta + G5P_CU_START); ++ g5p_put32(w + 0x25c, meta + G5P_CU_END); ++ g5p_put32(w + 0x260, seq); ++ g5p_put32(w + 0x264, G5P_CU_IDENTITY); ++ g5p_put32(w + 0x270, seq); ++ g5p_cu_sku(p + G5P_CU_SKU, meta, global, slot, seq); ++ g5p_put32(p + G5P_WORK_RING + g5p_cu_ring_index(seq) * 4, ++ meta + G5P_CU_W); ++} ++ ++/* Retirement of job `seq` (the r19d predicate, RAM only): W still names our stamps and ++ * sequence, finish33's stamp and the retirement mirror both at seq, all five channel ++ * watermarks at its endpoint. ++ */ ++static inline bool g5p_cu_retired(const u8 *p, u32 meta, u32 seq) ++{ ++ const u8 *w = p + G5P_CU_W; ++ u32 endpoint = g5p_cu_endpoint(seq); ++ ++ if (g5p_le32(w + 0x258) != meta + G5P_CU_START || ++ g5p_le32(w + 0x25c) != meta + G5P_CU_END || ++ g5p_le32(w + 0x260) != seq || g5p_le32(p + G5P_CU_START) != seq || ++ g5p_le32(p + G5P_CU_END) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* The stamps are the previous job's or ours, and no watermark runs past our endpoint's ++ * predecessor: anything else is a protocol error (another writer, or the wrong channel). ++ */ ++static inline bool g5p_cu_stamp_sane(const u8 *p, u32 seq) ++{ ++ u32 start = g5p_le32(p + G5P_CU_START), end = g5p_le32(p + G5P_CU_END); ++ ++ return (start == seq - 1 || start == seq) && ++ (end == seq - 1 || end == seq); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +new file mode 100644 +index 000000000..709efed16 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -0,0 +1,567 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared private bring-up implementation for mutually exclusive frontends. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++/* perf60 clock: which GPC states license an R0 access (a bell, a TLB pass, a register sample). */ ++/* release8 default */ ++ ++/* The GPC is powered and not mid-transition: state 1 only, or (gpc_any_pstate) any nonzero ++ * pstate nibble with no request in flight. A parked GPC (nibble 0) never qualifies. ++ */ ++static inline bool g5p_gpc_ok(u32 raw) ++{ ++ ; ++ ; ++ return !(raw & 0x10) && (raw & 0xf); ++} ++ ++/* The pure headers (readback.h, r74e112.h, render-user-wake.h, dirtystart108.h) judge the GPC state ++ * through G5P_GPC_ON(); their default is the old == 1, for the host tests. The kernel takes the rule. ++ */ ++#define G5P_GPC_ON(raw) g5p_gpc_ok(raw) ++ ++#include "pack.h" ++#include "work-layout.h" ++#include "mmio-licence.h" ++#include "gpu-oc.h" ++#include ++#include "readback.h" ++ ++/* nopatch: the TYPED_V3 boot path (r74/r74f/r74g, meta6c8=1) on Apple's original image. The release8 ++ * default since its device pass (2026-11-03, release8-nopatch a10a8556b9): an explicit firmware patch ++ * request (boot_r73f_case 1..4, or the r75/r79 cases that ride on it) takes the patched path instead, ++ * so the patched fallback is boot_r73f_case 1 alone (g5p_fw_unpatched_want()). ++ */ ++/* release8 default */ ++ ++/* The effective nopatch latch: the parameter, unless a patched image is asked for. */ ++static u32 g5p_fw_unpatched_want(u32 param, u32 r73f, u32 r75, u32 r79) ++{ ++ return param == 1 && !r73f && !r75 && !r79; ++} ++ ++#define G5P_MSLVDATAX 0x0a00 ++#define G5P_MSLVDATAT 0x0a08 ++#define G5P_MSLVCTRL0 0x0a10 ++#define G5P_MSLVCTRL1 0x0a18 ++#define G5P_MSLVIRQSTATUS 0x0ac8 ++#define G5P_SP_READY (BIT(24) | BIT(26)) ++#define G5P_FW_NAME "apple/agx-g5p-c0.ldr" ++ ++struct g5p_boot; ++ ++struct g5p_diag { ++ struct g5p_diagnostics *diagnostics; ++ struct g5p_boot *boot; ++ ++ spinlock_t slot_lock; ++ DECLARE_BITMAP(user_slots, 256); ++ struct device *dev; ++ void __iomem *window[2]; ++ void __iomem *rgx; ++ struct regmap *pmgr; ++ /* Serializes commands and all hardware access. */ ++ struct mutex lock; ++ bool powered; ++ bool loaded; ++ ++ struct g5p_ldr_image image; ++ bool fender_ready; ++ bool id_checked; ++ int last_result; ++ u32 words_written; ++ u32 core_id, core_revision; ++ u32 last_sp_addr; ++ bool reset_transition; ++ void (*event_notify)(void *cookie, const u32 words[11]); ++ void *event_cookie; ++ /* The GPU register licence (mmio.inc, docs/gpu/experiments/mmio-access.md §4). mmio_lock is ++ * a leaf: taken inside event_lock where both are held, never the reverse, nothing inside. ++ * mmio_off forbids R1/R0 through the accessor from before any PMGR gfx gate until Fender is ++ * up again (g5p_mmio_gate(), cleared by g5p_fender_wake_once()). ++ */ ++ spinlock_t mmio_lock; ++ bool mmio_off; ++ u32 mmio_max_pstate; ++ struct g5p_mmio_self *mmio_self; ++}; ++ ++/* Before every PMGR gfx gate (boot R33 off, dirty-start, kexec quiesce and in-boot recovery ++ * cycles): no accessor MMIO from here until g5p_fender_wake_once() brings Fender up again. ++ * Apple clears fender+0xc inside setPlatformPower(0), before its gate (G5). ++ */ ++static void g5p_mmio_gate(struct g5p_diag *g) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = true; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++} ++ ++/* gpu-recovery108: retire predicates (teardown.inc) and dirty-start hooks (dirtystart108.inc). */ ++static bool g5p_r8_r1_allowed(void); ++static bool g5p_r8_is_retired(void); ++static void g5p_ds_hook_probe(struct g5p_diag *g); ++static void g5p_ds_hook_power(struct g5p_diag *g); ++static int g5p_ds_hook_reset(struct g5p_diag *g); ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret); ++static int g5p_ds_hook_release_gate(struct g5p_diag *g); ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret); ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret); ++/* wake112: the reset generation of a deferred R74E proof (dirtystart108.inc). */ ++static u32 g5p_ds_gen(void); ++ ++#include "diagnostics.inc" ++ ++/* The GPU ADT clock/power gate124 resolves to ps-regs[4]+index8*8. ++ * Read via its existing syscon, sharing the provider mapping without writes. ++ */ ++static void g5p_power_observer_init(struct g5p_diag *g) ++{ ++ struct device_node *domain, *parent; ++ u32 offset; ++ ++ domain = of_parse_phandle(g->dev->of_node, "power-domains", 0); ++ if (!domain) ++ return; ++ parent = of_get_parent(domain); ++ if (of_device_is_compatible(domain, "apple,s8000-pmgr-pwrstate") && ++ !of_property_read_u32(domain, "reg", &offset) && ++ offset == 0x80340 && parent && ++ of_device_is_compatible(parent, "apple,s8000-pmgr")) { ++ g->pmgr = syscon_node_to_regmap(parent); ++ if (IS_ERR(g->pmgr)) ++ g->pmgr = NULL; ++ } ++ of_node_put(parent); ++ of_node_put(domain); ++} ++ ++/* pmgr-pwrstate.c masks; history flags are sticky, not current clock state. */ ++static void g5p_power_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) { ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx unavailable: provider not resolved\n", ++ stage); ++ return; ++ } ++ dev_info(g->dev, "POWER %s reading PMGR gfx20e080340 gate124\n", stage); ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (ret) { ++ dev_info(g->dev, "POWER %s PMGR gfx read error=%d\n", stage, ++ ret); ++ return; ++ } ++ dev_info( ++ g->dev, ++ "POWER %s PMGR gfx20e080340=%08x target=%x actual=%x auto=%u auto_state=%x " ++ "min=%x reset=%u parent_off=%u disabled=%u; current outer-domain fields only\n", ++ stage, raw, raw & 0xf, (raw >> 4) & 0xf, (raw >> 28) & 1, ++ (raw >> 24) & 0xf, (raw >> 16) & 0xf, raw >> 31, ++ (raw >> 11) & 1, (raw >> 10) & 1); ++ dev_info( ++ g->dev, ++ "POWER %s PMGR history was_clkgated=%u was_pwrgated=%u; sticky history, not " ++ "current state\n", ++ stage, (raw >> 9) & 1, (raw >> 8) & 1); ++} ++ ++static int g5p_power(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->powered) ++ return 0; ++ dev_info(g->dev, "request gfx power domain\n"); ++ ret = pm_runtime_resume_and_get(g->dev); ++ if (ret < 0) ++ return ret; ++ g->powered = true; ++ dev_info(g->dev, "gfx runtime PM active\n"); ++ g5p_ds_hook_power(g); ++ return 0; ++} ++ ++#include "power-r33.inc" ++#include "power-r41.inc" ++ ++static bool boot_gfx_off_done; ++ ++static int g5p_fender_wake_once(struct g5p_diag *g) ++{ ++ void __iomem *fender = g->window[1]; ++ u32 value; ++ int ret; ++ ++ ret = g5p_power(g); ++ if (ret) ++ return ret; ++ /* recovery108 mode 3: the one bounded incoming cycle, before any GPU MMIO. */ ++ ret = g5p_ds_hook_reset(g); ++ if (ret) ++ return ret; ++ /* R33: once per boot, before the first Fender write (no GPU MMIO yet). */ ++ ; ++ ret = g5p_r41_bridge_prepare(g); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "W R1+10000=ffff0000 (Apple ...6c93fe8)\n"); ++ writel(0xffff0000, fender + 0x10000); ++ dev_info(g->dev, "W R1+00000=40000000 (Apple ...6c93ff0)\n"); ++ writel(0x40000000, fender); ++ dev_info(g->dev, "W R1+01000=00000011 (Apple ...6c93ff8)\n"); ++ writel(0x11, fender + 0x1000); ++ dev_info(g->dev, "poll R1+01000 bit4 clear\n"); ++ ret = readl_poll_timeout(fender + 0x1000, value, !(value & BIT(4)), 10, ++ 100000); ++ dev_info(g->dev, "Fender wake status=%08x result=%d\n", value, ret); ++ if (ret) ++ return ret; ++ usleep_range(10, 20); ++ g->fender_ready = true; ++ { ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g->mmio_lock, flags); ++ g->mmio_off = false; ++ spin_unlock_irqrestore(&g->mmio_lock, flags); ++ } ++ return 0; ++} ++ ++static int g5p_fender_wake(struct g5p_diag *g) ++{ ++ int ret; ++ ++ if (g->fender_ready) ++ return 0; ++ ret = g5p_fender_wake_once(g); ++ g5p_ds_hook_wake(g, ret); ++ return ret; ++} ++ ++static int g5p_sp_wait(struct g5p_diag *g) ++{ ++ u32 value; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = readl_poll_timeout(g->rgx + G5P_MSLVCTRL1, value, ++ (value & G5P_SP_READY) == G5P_SP_READY, 10, ++ 100000); ++ if (ret) ++ dev_err(g->dev, "slave port timeout CTRL1=%08x words=%u\n", ++ value, g->words_written); ++ if (ret) ++ dev_err(g->dev, "slave target=%08x\n", g->last_sp_addr); ++ return ret; ++} ++ ++static int g5p_meta_read(struct g5p_diag *g, u32 addr, u32 *out) ++{ ++ int ret; ++ ++ g->last_sp_addr = addr; ++ ret = g5p_sp_wait(g); ++ ++ if (ret) ++ return ret; ++ writel(addr | 1, g->rgx + G5P_MSLVCTRL0); ++ readl(g->rgx + G5P_MSLVCTRL0); ++ ret = g5p_sp_wait(g); ++ if (ret) ++ return ret; ++ *out = readl(g->rgx + G5P_MSLVDATAX); ++ return 0; ++} ++ ++static int g5p_firmware(struct g5p_diag *g, bool prepare) ++{ ++ static const u8 expected[SHA256_DIGEST_SIZE] = { ++ 0x33, 0xae, 0x20, 0xcb, 0xbe, 0x24, 0x98, 0xe4, ++ 0xf1, 0x19, 0xba, 0x4e, 0x9a, 0x55, 0x46, 0x7e, ++ 0x4e, 0x01, 0x92, 0xd4, 0xc8, 0x6d, 0x5b, 0xf1, ++ 0x90, 0x3a, 0xdc, 0xe6, 0x3f, 0xfa, 0x5d, 0xca, ++ }; ++ /* SHA-256 of the packed image of that LDR, unmodified (nopatch's receipt) */ ++ static const u8 g5p_fw_packed_sha[SHA256_DIGEST_SIZE] = { ++ 0x48, 0xea, 0x03, 0x14, 0x08, 0x23, 0x96, 0x23, ++ 0xd8, 0x6f, 0x79, 0xb8, 0xa5, 0xda, 0x55, 0x7e, ++ 0x25, 0x3d, 0xb1, 0x6f, 0xb6, 0x23, 0x47, 0x95, ++ 0xe8, 0xed, 0xba, 0xa5, 0xcd, 0x9b, 0xb9, 0x1a, ++ }; ++ const struct firmware *fw; ++ u8 digest[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ ret = request_firmware_direct(&fw, G5P_FW_NAME, g->dev); ++ if (ret) ++ return ret; ++ sha256(fw->data, fw->size, digest); ++ if (fw->size != 119112 || memcmp(digest, expected, sizeof(expected))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, NULL, NULL); ++ if (ret) ++ goto out; ++ if (prepare && !g->loaded) { ++ g->image.data = kvzalloc(G5P_IMAGE_SIZE, GFP_KERNEL); ++ if (!g->image.data) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_ldr_walk(fw->data, fw->size, g5p_pack_record, ++ &g->image); ++ if (!ret && g->image.config_count != G5P_CONFIG_WRITES) ++ ret = -EINVAL; ++ if (!ret) { ++ sha256(g->image.data, G5P_IMAGE_SIZE, digest); ++ if (memcmp(digest, g5p_fw_packed_sha, sizeof(digest))) ++ ret = -EBADMSG; ++ } ++ if (ret) { ++ kvfree(g->image.data); ++ memset(&g->image, 0, sizeof(g->image)); ++ goto out; ++ } ++ g->loaded = true; ++ dev_info( ++ g->dev, ++ "NOPATCH packed apple_original=1 packed_sha256=%32phN end=nopatch\n", ++ digest); ++ } ++out: ++ release_firmware(fw); ++ return ret; ++} ++ ++static int g5p_sample(struct g5p_diag *g) ++{ ++ u32 value; ++ unsigned int thread, reg; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ if (!g->rgx) ++ return -EINVAL; ++ dev_info(g->dev, "RGX CTRL1=%08x IRQSTATUS=%08x\n", ++ readl(g->rgx + G5P_MSLVCTRL1), ++ readl(g->rgx + G5P_MSLVIRQSTATUS)); ++ for (thread = 0; thread < 2; thread++) { ++ for (reg = 0; reg <= 0x10; reg += 0x10) { ++ ret = g5p_meta_read( ++ g, 0x04800000 + thread * 0x1000 + reg, &value); ++ if (ret) ++ return ret; ++ dev_info(g->dev, "META T%u %s=%08x\n", thread, ++ reg ? "STATUS" : "ENABLE", value); ++ } ++ } ++ return 0; ++} ++ ++static int g5p_read_id(struct g5p_diag *g) ++{ ++ u32 words[2], version; ++ static const u32 offsets[] = { 0x20, 0x18 }; ++ unsigned int i; ++ ++ if (!g->fender_ready || READ_ONCE(g->reset_transition)) ++ return -EPERM; ++ g->id_checked = false; ++ for (i = 0; i < ARRAY_SIZE(offsets); i++) { ++ dev_info(g->dev, "R R0+%05x\n", offsets[i]); ++ words[i] = readl(g->rgx + offsets[i]); ++ dev_info(g->dev, "R0+%05x=%08x\n", offsets[i], words[i]); ++ } ++ g->core_revision = words[0]; ++ g->core_id = words[1]; ++ if ((!words[0] && !words[1]) || words[0] == ~0U || words[1] == ~0U) ++ return -ENODEV; ++ /* Match Apple readChipInfo at ...6ca644c before deeper META accesses. */ ++ version = (words[0] >> 8) & 0xff; ++ if ((words[0] & 0xff0000) != 0x90000 || ++ (version != 7 && version != 10 && version != 13 && version != 15)) ++ return -ENODEV; ++ g->id_checked = true; ++ return 0; ++} ++ ++#include "boot.inc" ++#include "mmio.inc" ++#include "kick.inc" ++#include "lifecycle.inc" ++#include "teardown.inc" ++#include "dirtystart108.inc" ++#include "recover.inc" ++#include "idle-power.inc" ++#include "transport-budget.inc" ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++ ++static ssize_t result_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ return sysfs_emit( ++ buf, ++ "last=%d loaded=%u maps=%u armed=%u copied=%u released=%u alive=%u " ++ "control_complete=%u failed=%u\n", ++ READ_ONCE(g->last_result), READ_ONCE(g->loaded), ++ b && b->maps_ready, b && b->armed, b && b->copied, ++ b && b->released, b && b->alive, b && b->control_complete, ++ b && b->failed); ++} ++static DEVICE_ATTR_RO(result); ++ ++/* Cached raw words from the last explicit/automatic snapshot: no GPU MMIO. */ ++ ++/* One bring-up phase, as the command file and the probe-time bring-up run it. *ran is set ++ * when the phase itself ran (and its result was recorded). ++ */ ++static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) ++{ ++ int ret; ++ ++ *ran = false; ++ if (g5p_r8_is_retired()) ++ return -ESHUTDOWN; ++ guard(mutex) ++ (&g->lock); ++ if (g->boot && (g->boot->stopped || g->boot->user_compute_owned)) ++ return -EBUSY; ++ if (sysfs_streq(buf, "validate") || sysfs_streq(buf, "prepare")) ++ ret = g5p_firmware(g, sysfs_streq(buf, "prepare")); ++ else if (sysfs_streq(buf, "wake")) ++ ret = g5p_fender_wake(g); ++ else if (sysfs_streq(buf, "id")) ++ ret = g5p_read_id(g); ++ else if (sysfs_streq(buf, "sample")) ++ ret = g->id_checked ? g5p_sample(g) : -EPERM; ++ else if (sysfs_streq(buf, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(buf, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(buf, "control-wait")) ++ ret = g5p_control_wait(g); ++ else if (sysfs_streq(buf, "map") || sysfs_streq(buf, "arm") || ++ sysfs_streq(buf, "load") || sysfs_streq(buf, "release") || ++ sysfs_streq(buf, "events")) ++ ret = g5p_boot_command(g, buf); ++ else ++ return -EOPNOTSUPP; ++ g->last_result = ret; ++ *ran = true; ++ return ret; ++} ++ ++static ssize_t command_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ bool ran; ++ int ret; ++ ++ if (!capable(CAP_SYS_RAWIO)) ++ return -EPERM; ++ if (sysfs_streq(buf, "td-kexec")) ++ return g5p_td_command(g, buf) ?: count; ++ ret = g5p_run_phase(g, buf, &ran); ++ if (ran) ++ dev_info(dev, "command %.*s result=%d\n", (int)count, buf, ret); ++ return ret ? ret : count; ++} ++static DEVICE_ATTR_WO(command); ++ ++static struct attribute *g5p_attrs[] = { ++ &dev_attr_command.attr, ++ &dev_attr_result.attr, ++ &dev_attr_recovery_status.attr, ++ &dev_attr_idle_power_status.attr, ++ NULL, ++}; ++static const struct attribute_group g5p_group = { .attrs = g5p_attrs }; ++__ATTRIBUTE_GROUPS(g5p); ++ ++static int g5p_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct g5p_diag *g; ++ unsigned int i; ++ ++ g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL); ++ if (!g) ++ return -ENOMEM; ++ g->dev = dev; ++ if (g5p_diagnostics_init(g)) ++ return -ENOMEM; ++ mutex_init(&g->lock); ++ spin_lock_init(&g->slot_lock); ++ spin_lock_init(&g->mmio_lock); ++ g->mmio_max_pstate = 1; ++ bitmap_set(g->user_slots, 0, 7); ++ for (i = 0; i < 2; i++) { ++ g->window[i] = devm_platform_ioremap_resource(pdev, i); ++ if (IS_ERR(g->window[i])) ++ return PTR_ERR(g->window[i]); ++ } ++ g->rgx = g->window[0]; ++ g5p_power_observer_init(g); ++ platform_set_drvdata(pdev, g); ++ if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40))) ++ return -ENODEV; ++ pm_runtime_enable(dev); ++ g5p_td_attach(g); ++ g5p_ds_hook_probe(g); ++ ; ++ dev_info(dev, "mapped both ADT windows; awaiting explicit command\n"); ++ return 0; ++} ++ ++static void g5p_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ /* With teardown on, keep the PM reference: an outgoing PMGR off is unvalidated. */ ++ bool keep_power; ++ ++ ; ++ keep_power = g5p_td_remove(g); ++ if (g->powered && !keep_power) { ++ g5p_mmio_gate(g); ++ pm_runtime_put_sync(g->dev); ++ } ++ pm_runtime_disable(g->dev); ++ kvfree(g->image.data); ++} ++ +diff --git a/drivers/gpu/apple-g5p/diagnostics.inc b/drivers/gpu/apple-g5p/diagnostics.inc +new file mode 100644 +index 000000000..184ce954b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/diagnostics.inc +@@ -0,0 +1,181 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Cached append-only text. Readers never take the command lock or touch MMIO. ++ * Preserve earliest records on overflow, expose dropped bytes explicitly. ++ */ ++#define G5P_BLOB_SIZE SZ_4M ++#define G5P_MMU_HISTORY_SIZE SZ_32K ++ ++struct g5p_diagnostics { ++ spinlock_t lock; ++ u32 used, history_used; ++ u64 dropped, history_dropped; ++ char stage[40]; ++ u32 next_register; ++ bool reading; ++ char history[G5P_MMU_HISTORY_SIZE]; ++ char blob[G5P_BLOB_SIZE]; ++}; ++ ++static void g5p_diagnostics_free(void *data) ++{ ++ kvfree(data); ++} ++ ++static int g5p_diagnostics_init(struct g5p_diag *g) ++{ ++ g->diagnostics = kvzalloc(sizeof(*g->diagnostics), GFP_KERNEL); ++ if (!g->diagnostics) ++ return -ENOMEM; ++ spin_lock_init(&g->diagnostics->lock); ++ return devm_add_action_or_reset(g->dev, g5p_diagnostics_free, ++ g->diagnostics); ++} ++ ++static void g5p_diagnostic_append(struct g5p_diag *g, bool history, ++ const char *text, size_t len) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ u32 *used = history ? &d->history_used : &d->used; ++ u64 *dropped = history ? &d->history_dropped : &d->dropped; ++ u32 capacity = history ? G5P_MMU_HISTORY_SIZE : G5P_BLOB_SIZE; ++ char *dst = history ? d->history : d->blob; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ /* Keep complete lines; once full, never replace earlier evidence. */ ++ if (*dropped || len > capacity - *used) { ++ *dropped += len; ++ } else { ++ memcpy(dst + *used, text, len); ++ *used += len; ++ } ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_blob_printf(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ char line[512]; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, false, line, len); ++} ++ ++static void g5p_blob_hex(struct g5p_diag *g, const char *name, const u8 *data, ++ u32 len) ++{ ++ for (u32 off = 0; off < len; off += 16) { ++ char text[80]; ++ ++ hex_dump_to_buffer(data + off, min_t(u32, len - off, 16), 16, 4, ++ text, sizeof(text), false); ++ g5p_blob_printf(g, "%s: %04x: %s\n", name, off, text); ++ } ++} ++ ++/* R1+1000 uses synthetic address11000; other offsets denote R0. */ ++static void g5p_mmu_progress(struct g5p_diag *g, const char *stage, u32 next) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->lock, flags); ++ strscpy(d->stage, stage, sizeof(d->stage)); ++ d->next_register = next; ++ d->reading = true; ++ spin_unlock_irqrestore(&d->lock, flags); ++} ++ ++static __printf(2, 3) void g5p_mmu_record(struct g5p_diag *g, const char *fmt, ++ ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ dev_info(g->dev, "%s", line); ++} ++ ++/* The routine per-job and per-event lines (a job's publish, wake and success receipt, the firmware's ++ * alive and stamp events). At desktop job rates they are hundreds of lines a second, more than netconsole ++ * can send: its queue grew to 700 MB of unreclaimable slab (release8, 55 min). Faults, losses, ++ * recoveries, terminal latches and every failed job keep their own levels and always print. ++ * job_log 0 dev_dbg (dynamic debug: off unless enabled in dynamic_debug/control) ++ * 1 KERN_DEBUG: in dmesg for the parsers (RENDER-USER job=, RENDER publish), below the ++ * default console loglevel, so not on a console or netconsole ++ * 2 KERN_INFO as before (short netconsole traces only: floods at desktop job rates) ++ */ ++static unsigned int job_log = 0; ++module_param(job_log, uint, 0600); ++MODULE_PARM_DESC( ++ job_log, ++ "default0 routine per-job/per-event lines are dev_dbg; 1 KERN_DEBUG (dmesg, no " ++ "console); 2 KERN_INFO as before (floods netconsole)"); ++ ++#define g5p_job_printk(dev, fmt, ...) \ ++ do { \ ++ switch (READ_ONCE(job_log)) { \ ++ case 0: \ ++ dev_dbg(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ case 1: \ ++ dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ default: \ ++ dev_info(dev, fmt, ##__VA_ARGS__); \ ++ break; \ ++ } \ ++ } while (0) ++ ++/* g5p_mmu_record for a routine per-job line: the blob keeps it, the console sees it only at job_log 2. */ ++static __printf(2, 3) __maybe_unused ++ void g5p_mmu_record_job(struct g5p_diag *g, const char *fmt, ...) ++{ ++ struct g5p_diagnostics *d = g->diagnostics; ++ char line[256]; ++ unsigned long flags; ++ va_list args; ++ size_t len; ++ ++ va_start(args, fmt); ++ len = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ g5p_diagnostic_append(g, true, line, len); ++ spin_lock_irqsave(&d->lock, flags); ++ d->reading = false; ++ spin_unlock_irqrestore(&d->lock, flags); ++ g5p_job_printk(g->dev, "%s", line); ++} ++ ++/* A routine event's raw words at the job_log level. */ ++static __maybe_unused void g5p_job_hex(const char *prefix, const void *buf, ++ size_t len) ++{ ++ switch (READ_ONCE(job_log)) { ++ case 0: ++ print_hex_dump_debug(prefix, DUMP_PREFIX_NONE, 16, 4, buf, len, ++ false); ++ break; ++ case 1: ++ print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ default: ++ print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_NONE, 16, 4, buf, ++ len, false); ++ break; ++ } ++} +diff --git a/drivers/gpu/apple-g5p/dirtystart108.h b/drivers/gpu/apple-g5p/dirtystart108.h +new file mode 100644 +index 000000000..fcf7432fa +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.h +@@ -0,0 +1,454 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DIRTYSTART108_H ++#define G5P_DIRTYSTART108_H ++ ++#include "bits-compat.h" ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * G5P incoming dirty-start state machine (series gpu-recovery108, patch 0007). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 4. ++ * ++ * The bootstrap is stepped by userspace (map, load, release, events, ++ * control), so this is an event-driven machine, not one run: the glue feeds ++ * one event per driver step with the step's result and RAM-only inputs, and ++ * asks the machine at two gates (before release, at the DRM claim). It ++ * assumes nothing pristine: the origin of the power state is classified from ++ * the kexec flag, the requested incoming PMGR cycle and the PMGR word, and ++ * every step's own result is evidence. ++ * ++ * PROBE origin inputs: booted from kexec, kexec_gpu_cycle requested ++ * POWER the PMGR gfx word, read once after the PM reference (PMGR read) ++ * RESET the bounded incoming power cycle (g5p_r33_gfx_off(), one attempt ++ * per boot, only in mode 3, only before the first wrapper wake) ++ * WAKE the Fender wake result ++ * MAPS fresh shared state: every ring control word in range (after the wake: ++ * map needs the core id, read through the woken wrapper) ++ * LOAD the firmware image memcmp of the RAM copy (b->copied) ++ * RELEASE the reset-stream/release result, the R74E power result ++ * ALIVE the first ALIVE wait result and, when read, the first-ALIVE ++ * GPC word (r73e); 0x10/0x11 is the C3 closed gate ++ * CLAIM the DRM claim asks; answers from the ALIVE proof ++ * ++ * Modes (parameter dirty_start): ++ * 0 observe: receipts only, both gates open (combined107 behaviour) ++ * 1 gate (default): fail closed on evidence the driver itself produced (a ++ * failed or out-of-order step, a power abort, a GPC word other than ++ * 0x01 at the first ALIVE, a ring out of range); a claim needs ALIVE ++ * proven in the current reset generation. Equal to combined107 on every ++ * bootstrap that succeeds. ++ * 2 strict: also refuse release and claim for an origin that is not cold ++ * and not a witnessed cycle (warm-unproven, warm-uncycled, unknown), ++ * unless a reset succeeded in this generation ++ * 3 strict + reset: as 2, and the glue may spend the one RESET attempt ++ * on such an origin before the wrapper wake (destructive: PMGR write) ++ * ++ * Generations: gen starts at 1 at PROBE and steps once per RESET attempt. A ++ * reset voids every proof of the previous generation; ALIVE proven in an ++ * earlier generation never exposes. gen never decreases. ++ * ++ * Pure policy: no register access, no kernel API except scnprintf(). Each ++ * event emits exactly one DS108 line through the emit callback. ++ */ ++ ++enum g5p_ds_ev { ++ G5P_DSE_PROBE, ++ G5P_DSE_POWER, ++ G5P_DSE_RESET, ++ G5P_DSE_WAKE, ++ G5P_DSE_MAPS, ++ G5P_DSE_LOAD, ++ G5P_DSE_RELEASE, ++ G5P_DSE_ALIVE, ++ G5P_DSE_CLAIM, ++ G5P_DSE_RECOVER, /* an in-boot recovery cycled the gate after a proven epoch */ ++ G5P_DSE_N, ++}; ++ ++static const char *const g5p_ds_ev_name[G5P_DSE_N] = { ++ "probe", "power", "reset", "wake", "maps", ++ "load", "release", "alive", "claim", "recover", ++}; ++ ++enum g5p_ds_origin { ++ G5P_DSO_UNKNOWN, /* not probed, or the PMGR word could not be read */ ++ G5P_DSO_COLD, /* not booted from kexec */ ++ G5P_DSO_WARM_CYCLED, /* kexec, incoming cycle requested and witnessed (was_pwrgated) */ ++ G5P_DSO_WARM_UNPROVEN, /* kexec, cycle requested, no witness in the PMGR word */ ++ G5P_DSO_WARM_UNCYCLED, /* kexec, no incoming cycle requested */ ++ G5P_DSO_N, ++}; ++ ++static const char *const g5p_ds_origin_name[G5P_DSO_N] = { ++ "unknown", "cold", "warm-cycled", "warm-unproven", "warm-uncycled", ++}; ++ ++enum g5p_ds_why { ++ G5P_DSW_NONE, ++ G5P_DSW_PROTOCOL, /* a step out of order (its predecessor not ok in this gen) */ ++ G5P_DSW_WAKE, /* the wrapper wake failed */ ++ G5P_DSW_MAPS, /* map failed or a ring control word out of range */ ++ G5P_DSW_LOAD, /* the RAM image is not the firmware (memcmp) */ ++ G5P_DSW_RELEASE, /* release / reset stream failed */ ++ G5P_DSW_POWER_ABORT, /* the R74E power poll or the first-ALIVE gate aborted */ ++ G5P_DSW_GPC, /* first-ALIVE GPC word not 0x01 (0x10/0x11: C3) */ ++ G5P_DSW_ALIVE, /* no ALIVE, or the ALIVE wait failed */ ++ G5P_DSW_RESET, /* the incoming power cycle failed: power state unknown */ ++ G5P_DSW_N, ++}; ++ ++static const char *const g5p_ds_why_name[G5P_DSW_N] = { ++ "none", "protocol", "wake", "maps", "load", ++ "release", "power-abort", "gpc", "alive", "reset", ++}; ++ ++/* Gate answers (not latched): why a gate is closed right now. */ ++enum g5p_ds_gate { ++ G5P_DSG_OPEN, ++ G5P_DSG_FAILED, /* the fail-closed latch */ ++ G5P_DSG_NOT_PROVEN, /* no ALIVE proof in the current generation */ ++ G5P_DSG_ORIGIN, /* strict: origin neither cold nor a witnessed cycle */ ++ G5P_DSG_STATE, /* the claim snapshot contradicts the proof */ ++ G5P_DSG_N, ++}; ++ ++static const char *const g5p_ds_gate_name[G5P_DSG_N] = { ++ "open", "failed", "not-proven", "origin", "state", ++}; ++ ++#define G5P_DS_MODE_MAX 3 ++#define G5P_DS_GPC_ON 0x01u ++#define G5P_DS_PMGR_PWRGATED \ ++ BIT(8) /* sticky was_pwrgated history bit of 0x80340 */ ++ ++#define G5P_DSF_RINGS_BUSY \ ++ BIT(0) /* a ring not empty at map (data, not a failure) */ ++#define G5P_DSF_ALIVE_LATE \ ++ BIT(1) /* ALIVE first seen at the claim (no ALIVE event) */ ++#define G5P_DSF_RESET_RAN BIT(2) /* a RESET attempt ran this boot */ ++#define G5P_DSF_RESET_OK \ ++ BIT(3) /* ... and succeeded (gen of the success = gen) */ ++/* a second attempt or one after the wake was refused */ ++#define G5P_DSF_RESET_REFUSED BIT(4) ++#define G5P_DSF_DUP BIT(5) /* an event repeated (recorded, not a failure) */ ++#define G5P_DSF_PMGR_UNREAD BIT(6) /* the PMGR word could not be read */ ++ ++/* One flat input record; each event reads only its own fields (RAM or step results). */ ++struct g5p_ds_in { ++ bool kexec; /* PROBE: chosen linux,booted-from-kexec */ ++ bool cycle_requested; /* PROBE: apple_pmgr.kexec_gpu_cycle=1 on the cmdline */ ++ bool pmgr_ok; /* POWER: the PMGR read succeeded */ ++ u32 pmgr_raw; /* POWER: the gfx word */ ++ int ret; /* RESET/WAKE/MAPS/LOAD/RELEASE/ALIVE: the step's return */ ++ u32 rings_bad; /* MAPS: control words > 255 */ ++ u32 rings_busy; /* MAPS: rings not empty */ ++ int power_ret; /* RELEASE: R74E power result */ ++ u32 power_raw; /* RELEASE: its last R1+0x1000 sample */ ++ bool power_abort; /* RELEASE/ALIVE/CLAIM: r73b early_power_abort */ ++ bool gpc_valid; /* ALIVE: the r73e first-ALIVE word was read */ ++ u32 gpc; /* ALIVE: that word */ ++ bool alive, released, failed, ++ control_complete; /* ALIVE/CLAIM snapshot */ ++}; ++ ++struct g5p_ds { ++ u32 gen; /* reset generation, monotonic, 0 before PROBE */ ++ u32 seen, ok; /* event bitmasks, current generation */ ++ u32 flags; ++ enum g5p_ds_origin origin; ++ bool kexec, cycle; /* PROBE inputs, kept for POWER */ ++ u32 pmgr_raw; ++ u32 resets; /* RESET attempts this boot (at most one runs) */ ++ u32 reset_ok_gen; /* gen in which a RESET succeeded, 0 never */ ++ u32 alive_gen; /* gen in which ALIVE was proven, 0 never */ ++ bool failed; /* fail-closed latch (absorbing) */ ++ enum g5p_ds_why why; ++ u32 events, lines, claims, refused; ++ enum g5p_ds_ev last; ++ enum g5p_ds_gate last_gate; ++}; ++ ++static inline bool g5p_ds_ok(const struct g5p_ds *d, enum g5p_ds_ev e) ++{ ++ return d->ok & (1u << e); ++} ++ ++static inline void g5p_ds_fail(struct g5p_ds *d, enum g5p_ds_why why) ++{ ++ if (!d->failed) ++ d->why = why; ++ d->failed = true; ++} ++ ++static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool cycle, ++ bool pmgr_ok, u32 raw) ++{ ++ if (!pmgr_ok) ++ return G5P_DSO_UNKNOWN; ++ if (!kexec) ++ return G5P_DSO_COLD; ++ if (!cycle) ++ return G5P_DSO_WARM_UNCYCLED; ++ return (raw & G5P_DS_PMGR_PWRGATED) ? G5P_DSO_WARM_CYCLED : ++ G5P_DSO_WARM_UNPROVEN; ++} ++ ++/* The origin is trusted (cold, witnessed cycle) or a reset succeeded in this generation. */ ++static inline bool g5p_ds_origin_trusted(const struct g5p_ds *d) ++{ ++ return d->origin == G5P_DSO_COLD || d->origin == G5P_DSO_WARM_CYCLED || ++ (d->reset_ok_gen && d->reset_ok_gen == d->gen); ++} ++ ++/* Should the glue spend the one RESET attempt now (mode 3, before the wake)? */ ++static inline bool g5p_ds_reset_wanted(const struct g5p_ds *d, u32 mode) ++{ ++ return mode >= 3 && !d->failed && !d->resets && ++ !(d->seen & (1u << G5P_DSE_WAKE)) && ++ (d->seen & (1u << G5P_DSE_POWER)) && !g5p_ds_origin_trusted(d); ++} ++ ++/* The release gate (asked by the glue right before the reset stream runs). */ ++static inline enum g5p_ds_gate g5p_ds_release_gate(const struct g5p_ds *d, ++ u32 mode) ++{ ++ if (!mode) ++ return G5P_DSG_OPEN; ++ if (d->failed) ++ return G5P_DSG_FAILED; ++ if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ return G5P_DSG_ORIGIN; ++ return G5P_DSG_OPEN; ++} ++ ++/* The ALIVE proof the claim needs: proven in this generation, nothing failed since. */ ++static inline bool g5p_ds_proven(const struct g5p_ds *d) ++{ ++ return !d->failed && d->alive_gen && d->alive_gen == d->gen && ++ g5p_ds_ok(d, G5P_DSE_RELEASE) && g5p_ds_ok(d, G5P_DSE_LOAD); ++} ++ ++static inline int g5p_ds_line(struct g5p_ds *d, char *buf, size_t n, ++ enum g5p_ds_ev e, int ret, u32 a, u32 b) ++{ ++ d->lines++; ++ return scnprintf(buf, n, ++ "DS108 gen=%u ev=%s ret=%d origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x pmgr=%08x a=%x b=%x gate=%s", ++ d->gen, g5p_ds_ev_name[e], ret, g5p_ds_origin_name[d->origin], ++ d->failed, g5p_ds_why_name[d->why], d->seen, d->ok, d->flags, ++ d->pmgr_raw, a, b, g5p_ds_gate_name[d->last_gate]); ++} ++ ++/* ++ * Feed one event. Returns, for CLAIM, 0 (expose) or -EAGAIN/-EIO (refuse); ++ * for every other event 0. mode is the current dirty_start parameter; at mode 0 ++ * the claim answers 0 whatever the state (the legacy driver decides alone). ++ * line/n receive the event's receipt (always written). ++ */ ++static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in, u32 mode, char *line, ++ size_t n) ++{ ++ u32 bit = 1u << e, a = 0, b = 0; ++ int ret = 0; ++ ++ if (mode > G5P_DS_MODE_MAX) ++ mode = G5P_DS_MODE_MAX; ++ d->events++; ++ d->last = e; ++ if ((d->seen & bit) && e != G5P_DSE_CLAIM) ++ d->flags |= G5P_DSF_DUP; ++ switch (e) { ++ case G5P_DSE_PROBE: ++ /* The only event that may start a generation: probe runs once per boot. */ ++ if (!d->gen) ++ d->gen = 1; ++ d->origin = G5P_DSO_UNKNOWN; ++ a = in->kexec; ++ b = in->cycle_requested; ++ d->kexec = in->kexec; ++ d->cycle = in->cycle_requested; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_POWER: ++ if (!(d->seen & (1u << G5P_DSE_PROBE))) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->origin = g5p_ds_origin_of(d->kexec, d->cycle, in->pmgr_ok, ++ in->pmgr_raw); ++ if (!in->pmgr_ok) ++ d->flags |= G5P_DSF_PMGR_UNREAD; ++ d->pmgr_raw = in->pmgr_ok ? in->pmgr_raw : 0; ++ a = in->pmgr_ok; ++ d->seen |= bit; ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RESET: ++ a = in->ret; ++ if (d->resets || (d->seen & (1u << G5P_DSE_WAKE))) { ++ /* The glue must not have run it; record a refused attempt and fail ++ * closed. ++ */ ++ d->flags |= G5P_DSF_RESET_REFUSED; ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->resets++; ++ d->flags |= G5P_DSF_RESET_RAN; ++ /* A reset voids the previous generation's proofs (power epoch boundary). */ ++ d->gen++; ++ /* Unreachable today (a reset runs before the wake, so before every later ++ * step); kept so that a reordered glue cannot carry a proof across it. ++ */ ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->flags |= G5P_DSF_RESET_OK; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_WAKE: ++ d->seen |= bit; ++ if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_WAKE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_MAPS: ++ d->seen |= bit; ++ a = in->rings_bad; ++ b = in->rings_busy; ++ if (in->rings_busy) ++ d->flags |= G5P_DSF_RINGS_BUSY; ++ /* map needs the core id, read through the woken wrapper */ ++ if (!g5p_ds_ok(d, G5P_DSE_WAKE)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret || in->rings_bad) ++ g5p_ds_fail(d, G5P_DSW_MAPS); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_LOAD: ++ d->seen |= bit; ++ if (!g5p_ds_ok(d, G5P_DSE_MAPS)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_LOAD); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_RELEASE: ++ d->seen |= bit; ++ a = (u32)in->power_ret; ++ b = in->power_raw; ++ if (!g5p_ds_ok(d, G5P_DSE_LOAD)) ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ else if (in->power_abort || in->power_ret) ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ else if (in->ret) ++ g5p_ds_fail(d, G5P_DSW_RELEASE); ++ else ++ d->ok |= bit; ++ break; ++ case G5P_DSE_ALIVE: ++ d->seen |= bit; ++ a = in->gpc_valid; ++ b = in->gpc; ++ if (!g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ } else if (in->power_abort) { ++ g5p_ds_fail(d, G5P_DSW_POWER_ABORT); ++ } else if (in->gpc_valid && !G5P_GPC_ON(in->gpc)) { ++ g5p_ds_fail(d, G5P_DSW_GPC); ++ } else if (in->ret || !in->alive || in->failed) { ++ g5p_ds_fail(d, G5P_DSW_ALIVE); ++ } else { ++ d->ok |= bit; ++ d->alive_gen = d->gen; ++ } ++ break; ++ case G5P_DSE_RECOVER: ++ /* The render frontend's in-boot recovery (recover.inc): the gate was cycled ++ * after this generation's release, with the firmware frozen. Like a RESET it ++ * starts a generation that voids every proof of the last one, and it is a ++ * witnessed cycle for the origin gate; the bootstrap's steps then run again ++ * and must each succeed in the new generation. Not after a failure, and not ++ * before a release: there is no epoch to recover then. ++ */ ++ a = in->ret; ++ if (d->failed || !g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ g5p_ds_fail(d, G5P_DSW_PROTOCOL); ++ break; ++ } ++ d->gen++; ++ d->seen &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->ok &= (1u << G5P_DSE_PROBE) | (1u << G5P_DSE_POWER); ++ d->seen |= bit; ++ /* The old ALIVE proof is void; the claim's late path may prove the new one. */ ++ d->alive_gen = 0; ++ if (in->ret) { ++ g5p_ds_fail(d, G5P_DSW_RESET); ++ break; ++ } ++ d->ok |= bit; ++ d->reset_ok_gen = d->gen; ++ break; ++ case G5P_DSE_CLAIM: ++ d->claims++; ++ d->seen |= bit; ++ a = in->alive | in->released << 1 | in->failed << 2 | ++ in->control_complete << 3 | in->power_abort << 4; ++ /* ALIVE reached through a path without the ALIVE event: accept the RAM ++ * latch as the proof only after a successful release of this generation. ++ */ ++ if (!d->failed && !d->alive_gen && in->alive && !in->failed && ++ !in->power_abort && g5p_ds_ok(d, G5P_DSE_RELEASE)) { ++ d->alive_gen = d->gen; ++ d->ok |= 1u << G5P_DSE_ALIVE; ++ d->flags |= G5P_DSF_ALIVE_LATE; ++ } ++ if (!mode) ++ d->last_gate = G5P_DSG_OPEN; ++ else if (d->failed) ++ d->last_gate = G5P_DSG_FAILED; ++ else if (!g5p_ds_proven(d)) ++ d->last_gate = G5P_DSG_NOT_PROVEN; ++ else if (!in->alive || !in->released || in->failed || ++ in->power_abort || !in->control_complete) ++ d->last_gate = G5P_DSG_STATE; ++ else if (mode >= 2 && !g5p_ds_origin_trusted(d)) ++ d->last_gate = G5P_DSG_ORIGIN; ++ else ++ d->last_gate = G5P_DSG_OPEN; ++ if (d->last_gate == G5P_DSG_OPEN) { ++ d->ok |= bit; ++ } else { ++ d->refused++; ++ ret = d->last_gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++ } ++ break; ++ default: ++ break; ++ } ++ g5p_ds_line(d, line, n, e, e == G5P_DSE_CLAIM ? ret : in->ret, a, b); ++ return ret; ++} ++ ++ ++ ++static inline void g5p_ds_init(struct g5p_ds *d) ++{ ++ *d = (struct g5p_ds){ .last = G5P_DSE_N }; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +new file mode 100644 +index 000000000..be023e7e7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -0,0 +1,280 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the incoming dirty-start machine (dirtystart108.h), series ++ * gpu-recovery108 (0007). Design: docs/gpu/experiments/recovery108-driver.md. ++ * ++ * Included from core.inc after teardown.inc, so it may use the teardown ring ++ * read (RAM), g5p_r33_gfx_off() and the retire layer's state. Patch 0008 calls ++ * the hooks from the driver steps; until then nothing here runs. ++ * ++ * Register access added by this file: one PMGR read of the gfx word after the ++ * PM reference (POWER, the same read g5p_power_snapshot() makes), and in mode 3 ++ * only one bounded PMGR cycle through g5p_r33_gfx_off() before the first ++ * wrapper wake. Every other input is RAM or a step's own return value. ++ */ ++#include ++#include "dirtystart108.h" ++ ++static struct g5p_ds g5p_ds_state = { .last = G5P_DSE_N }; ++static DEFINE_SPINLOCK(g5p_ds_lock); ++ ++/* Latched from /chosen at probe, before POWER or the first wrapper wake. */ ++static u32 g5p_ds_boot_mode = 1; ++ ++static u32 g5p_ds_mode(void) ++{ ++ return READ_ONCE(g5p_ds_boot_mode); ++} ++ ++/* One event, one DS108 line (dev_emerg: netconsole at any loglevel). */ ++static int g5p_ds_feed(struct g5p_diag *g, enum g5p_ds_ev e, ++ const struct g5p_ds_in *in) ++{ ++ unsigned long flags; ++ char line[224]; ++ int ret; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ ret = g5p_ds_event(&g5p_ds_state, e, in, g5p_ds_mode(), line, ++ sizeof(line)); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "%s\n", line); ++ return ret; ++} ++ ++/* wake112: the current reset generation, recorded with a deferred R74E proof (RAM, leaf lock). */ ++static u32 g5p_ds_gen(void) ++{ ++ unsigned long flags; ++ u32 gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return gen; ++} ++ ++/* The last apple_pmgr.kexec_gpu_cycle= token of the chosen bootargs (the one the kernel obeys). */ ++static bool g5p_ds_cycle_requested(struct device_node *chosen) ++{ ++ static const char key[] = "apple_pmgr.kexec_gpu_cycle="; ++ const char *args, *p, *last = NULL; ++ char tok[8]; ++ bool v = false; ++ size_t n; ++ ++ if (!chosen || of_property_read_string(chosen, "bootargs", &args)) ++ return false; ++ for (p = strstr(args, key); p; p = strstr(p + 1, key)) ++ if (p == args || p[-1] == ' ') ++ last = p; ++ if (!last) ++ return false; ++ last += sizeof(key) - 1; ++ n = strcspn(last, " "); ++ if (!n || n >= sizeof(tok)) ++ return false; ++ memcpy(tok, last, n); ++ tok[n] = 0; ++ return !kstrtobool(tok, &v) && v; ++} ++ ++static void g5p_ds_hook_probe(struct g5p_diag *g) ++{ ++ struct device_node *chosen = of_find_node_by_path("/chosen"); ++ struct g5p_ds_in in = {}; ++ ++ in.kexec = chosen && ++ of_property_read_bool(chosen, "linux,booted-from-kexec"); ++ in.cycle_requested = g5p_ds_cycle_requested(chosen); ++ of_node_put(chosen); ++ /* Retained warm state needs strict origin checks and the bounded reset. */ ++ WRITE_ONCE(g5p_ds_boot_mode, in.kexec ? 3 : 1); ++ dev_info(g->dev, "DS108 boot mode=%u kexec=%u\n", ++ g5p_ds_mode(), in.kexec); ++ g5p_ds_feed(g, G5P_DSE_PROBE, &in); ++} ++ ++/* After the PM reference is taken: one PMGR read (the gfx word), no write. */ ++static void g5p_ds_hook_power(struct g5p_diag *g) ++{ ++ struct g5p_ds_in in = {}; ++ u32 raw = 0; ++ ++ in.pmgr_ok = g->pmgr && !regmap_read(g->pmgr, 0x80340, &raw); ++ in.pmgr_raw = raw; ++ g5p_ds_feed(g, G5P_DSE_POWER, &in); ++} ++ ++/* Before the first wrapper wake. Mode 3 only, one attempt per boot, through the ++ * r33 helper (its own refusal rules: min/reset/auto or not on; every exit leaves ++ * the gate requested on). Returns the cycle's error; the wake then fails closed. ++ */ ++static int g5p_ds_hook_reset(struct g5p_diag *g) ++{ ++ u32 ms = clamp_t(u32, (10), 1, G5P_R33_GFX_OFF_MAX_MS); ++ struct g5p_ds_in in = {}; ++ unsigned long flags; ++ bool want; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ want = g5p_ds_reset_wanted(&g5p_ds_state, g5p_ds_mode()); ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ if (!want) ++ return 0; ++ g5p_mmio_gate(g); ++ in.ret = g5p_r33_gfx_off(g->dev, g->pmgr, ms); ++ g5p_ds_feed(g, G5P_DSE_RESET, &in); ++ boot_gfx_off_done = true; /* one PMGR cycle per boot, whoever runs it */ ++ return in.ret; ++} ++ ++/* The legacy boot_gfx_off_ms cycle (R33) is the same mechanism: record it as the reset. */ ++ ++/* An in-boot recovery's gate cycle (recover.inc): a new generation, or a failed machine. */ ++static int g5p_ds_hook_recover(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ unsigned long flags; ++ bool failed; ++ ++ g5p_ds_feed(g, G5P_DSE_RECOVER, &in); ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ failed = g5p_ds_state.failed; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return failed ? -EIO : 0; ++} ++ ++static void g5p_ds_hook_wake(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_WAKE, &in); ++} ++ ++/* Fresh shared state: every ring control word in range (RAM, the teardown ring read). */ ++static void g5p_ds_hook_maps(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ u32 i, cons, prod; ++ ++ for (i = 0; !ret && i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ in.rings_bad++; ++ continue; ++ } ++ if (cons > 255 || prod > 255) ++ in.rings_bad++; ++ else if (cons != prod) ++ in.rings_busy++; ++ } ++ g5p_ds_feed(g, G5P_DSE_MAPS, &in); ++} ++ ++static void g5p_ds_hook_load(struct g5p_diag *g, int ret) ++{ ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ g5p_ds_feed(g, G5P_DSE_LOAD, &in); ++} ++ ++/* Asked before the release touches anything. -EIO: the fail-closed latch; -EAGAIN: strict origin. */ ++static int g5p_ds_hook_release_gate(struct g5p_diag *g) ++{ ++ enum g5p_ds_gate gate; ++ unsigned long flags; ++ u32 mode = g5p_ds_mode(), gen; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ gate = g5p_ds_release_gate(&g5p_ds_state, mode); ++ g5p_ds_state.last_gate = gate; ++ if (gate != G5P_DSG_OPEN) ++ g5p_ds_state.refused++; ++ gen = g5p_ds_state.gen; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ dev_emerg(g->dev, "DS108 gen=%u gate=release mode=%u answer=%s\n", gen, ++ mode, g5p_ds_gate_name[gate]); ++ if (gate == G5P_DSG_OPEN) ++ return 0; ++ return gate == G5P_DSG_FAILED ? -EIO : -EAGAIN; ++} ++ ++static void g5p_ds_hook_release(struct g5p_diag *g, int ret) ++{ ++ struct g5p_r73b *r = g->boot ? g->boot->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (r) { ++ in.power_ret = r->early_power_ret; ++ in.power_raw = r->early_power_raw; ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ } ++ g5p_ds_feed(g, G5P_DSE_RELEASE, &in); ++} ++ ++static void g5p_ds_hook_alive(struct g5p_diag *g, int ret) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b ? b->r73b : NULL; ++ struct g5p_ds_in in = { .ret = ret }; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed); ++ } ++ if (r) { ++ in.power_abort = READ_ONCE(r->early_power_abort); ++ /* r73e reads R1+0x1000 at the first ALIVE; 0 means not read. */ ++ in.gpc = READ_ONCE(r->early_state); ++ in.gpc_valid = 1 && in.gpc; ++ } ++ g5p_ds_feed(g, G5P_DSE_ALIVE, &in); ++} ++ ++/* The DRM claim: 0 expose, -EAGAIN not proven / strict origin, -EIO the fail-closed latch. */ ++static int __maybe_unused g5p_ds_hook_claim(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_ds_in in = {}; ++ ++ if (b) { ++ in.alive = READ_ONCE(b->alive); ++ in.released = READ_ONCE(b->released); ++ in.failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ in.control_complete = READ_ONCE(b->control_complete); ++ in.power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ return g5p_ds_feed(g, G5P_DSE_CLAIM, &in); ++} ++ ++/* RAM only: the retire layer and the dirty-start machine, for capture without a log. */ ++static ssize_t recovery_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ const struct g5p_r8 *s = &g5p_r8_state; ++ unsigned long flags; ++ struct g5p_ds d; ++ ++ spin_lock_irqsave(&g5p_ds_lock, flags); ++ d = g5p_ds_state; ++ spin_unlock_irqrestore(&g5p_ds_lock, flags); ++ return sysfs_emit( ++ buf, ++ "r8 gen=%u retired=%d mode=%s class=%s verdict=%s flags=%02x admit_closed=%u " ++ "inside=%u detach_queued=%u lines=%u\n" ++ "ds gen=%u mode=%u origin=%s failed=%u why=%s seen=%03x ok=%03x flags=%02x " ++ "pmgr=%08x alive_gen=%u resets=%u reset_ok_gen=%u events=%u claims=%u refused=%u " ++ "gate=%s\n", ++ READ_ONCE(s->gen), READ_ONCE(s->retired), ++ g5p_td_mode_name[s->mode], g5p_r8_class_name[s->cls], ++ g5p_td_verdict_name[s->verdict], s->flags, ++ g5p_r8_admit_closed(&s->admit), g5p_r8_admit_inside(&s->admit), ++ READ_ONCE(s->detach_queued), s->lines, d.gen, g5p_ds_mode(), ++ g5p_ds_origin_name[d.origin], d.failed, g5p_ds_why_name[d.why], ++ d.seen, d.ok, d.flags, d.pmgr_raw, d.alive_gen, d.resets, ++ d.reset_ok_gen, d.events, d.claims, d.refused, ++ g5p_ds_gate_name[d.last_gate]); ++} ++static DEVICE_ATTR_RO(recovery_status); +diff --git a/drivers/gpu/apple-g5p/drm-validate.h b/drivers/gpu/apple-g5p/drm-validate.h +new file mode 100644 +index 000000000..671e13ab8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/drm-validate.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_DRM_VALIDATE_H ++#define G5P_DRM_VALIDATE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_USER_VA_START 0x10000ULL ++#define G5P_USER_VA_END 0x10000000000ULL ++#define G5P_USER_RESERVED_START G5P_IMAGE_VA ++#define G5P_USER_RESERVED_END (G5P_PIO_VA + G5P_PIO_SIZE) ++#define G5P_MAX_BO_SIZE (64ULL << 20) ++/* VA bookkeeping: the sum of mapping sizes in one VM. A GEM bound twice ++ * (GBM alloc, then the EGL import of its own dma-buf) counts twice here. ++ */ ++#define G5P_MAX_VM_BYTES BIT_ULL(30) ++/* Memory behind one VM's mappings, counted once per GEM object. */ ++#define G5P_MAX_PINNED_BYTES (768ULL << 20) ++ ++static inline bool g5p_user_pa_valid(u64 pa) ++{ ++ return pa < 0x10000000000ULL && !(pa & (G5P_GPU_PAGE - 1)); ++} ++ ++static inline bool g5p_user_range_valid(u64 va, u64 size) ++{ ++ if (!size || ((va | size) & (G5P_GPU_PAGE - 1)) || ++ va < G5P_USER_VA_START || va >= G5P_USER_VA_END || ++ size > G5P_USER_VA_END - va) ++ return false; ++ return va >= G5P_USER_RESERVED_END || ++ va + size <= G5P_USER_RESERVED_START; ++} ++ ++static inline bool g5p_bo_range_valid(u64 offset, u64 size, u64 bo_size) ++{ ++ return size && !((offset | size) & (G5P_GPU_PAGE - 1)) && ++ offset < bo_size && size <= bo_size - offset; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/fender.h b/drivers/gpu/apple-g5p/fender.h +new file mode 100644 +index 000000000..cbdf203c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/fender.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_FENDER_H ++#define G5P_FENDER_H ++ ++/* Every poll callback must enforce a deadline and retain its final raw word. ++ * Apple 20H350: stop c9fa10, prepare c9fc10, boot c9f954; ++ * Fender helpers ca4ccc (GPC state) and ca6544 (core-power request). ++ * Hardware interpretation of the 0 -> 1 transition remains experimental. ++ */ ++struct g5p_fender_io { ++ void *ctx; ++ int (*write)(void *ctx, u32 reg, u32 value); ++ int (*poll)(void *ctx, u32 reg, u32 mask, u32 expected); ++}; ++ ++static inline int g5p_fender_idle(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x1000, 0x10, 0); ++ if (ret) ++ return ret; ++ ret = io->write(io->ctx, 0x1000, 0x10); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++static inline int g5p_fender_hold(const struct g5p_fender_io *io) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0x20); ++ if (ret) ++ return ret; ++ ret = io->poll(io->ctx, 0x20000, 0x20, 0x20); ++ if (ret) ++ return ret; ++ return io->poll(io->ctx, 0x1000, 0x1f, 0); ++} ++ ++/* q: the pstate the GPC is requested at (R1+0x1000 = 0x10 | q, poll for q); 1 as before, ++ * or boot_gpu_perf_poweron's pin (boot.inc). ++ */ ++static inline int ++g5p_fender_start_observed_q(const struct g5p_fender_io *io, ++ void (*observe)(void *, const char *), u32 q) ++{ ++ int ret; ++ ++ ret = io->write(io->ctx, 0x20000, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-core-clear"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, 0); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc0-verified"); ++ ret = io->write(io->ctx, 0x1000, 0x10 | q); ++ if (ret) ++ return ret; ++ if (observe) ++ observe(io->ctx, "release-gpc1-requested"); ++ ret = io->poll(io->ctx, 0x1000, 0x1f, q); ++ if (!ret && observe) ++ observe(io->ctx, "release-gpc1-active"); ++ return ret; ++} ++ ++static inline int g5p_fender_start_observed(const struct g5p_fender_io *io, ++ void (*observe)(void *, ++ const char *)) ++{ ++ return g5p_fender_start_observed_q(io, observe, 1); ++} ++ ++static inline int g5p_fender_start(const struct g5p_fender_io *io) ++{ ++ return g5p_fender_start_observed(io, NULL); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/gpu-oc.h b/drivers/gpu/apple-g5p/gpu-oc.h +new file mode 100644 +index 000000000..733aba9d4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/gpu-oc.h +@@ -0,0 +1,149 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * GPU overclock record (boot_gpu_oc_mhz): docs/power/ds-gpu-dvfs.md §3-4, §9 R5; ++ * docs/gpu/experiments/clock-mechanisms.md §2. Host test: scripts/gpu/perf60-host/test_oc.c. ++ * ++ * The hardware applies PSINFO record n (PMGR+0x70000 + 0x10*n, written by LLB 180395fd8 for ++ * slots 0-4 once per SoC reset, volatile) when Fender is asked for pstate n. Word 0 = core-rail ++ * DAC code [31:24] | PLL valid [21] | multiplier [20:12] | predivider [8:4] | postdivider-1 [3:0]; ++ * word 1 [7:0] = buck8 code; word 2 = timing selectors. ++ * ++ * PLL: VCO = 24 MHz * mult / prediv, f = VCO / postdiv. Apple's GPU-domain (0x17) descriptors ++ * (LLB 0x1803bc718, all three chip sections) run VCOs from 800 to 2160 MHz (2160 = id 0x20 of ++ * section 0x2f0001ff, 90/1/3); the live states use 948..1650. The overclock keeps postdiv 2 (every ++ * live state at or above 474 MHz uses 2 or 3) and the VCO inside Apple's range: f <= 1080 MHz. ++ * Apple's own descriptors at 804 (67/1/2), 850 (425/6/2), 900 (75/1/2); elsewhere record 4's ++ * dividers (prediv 4, postdiv 2: f = 3 MHz * mult) on a 3 MHz grid. ++ * ++ * Voltage: record 5 keeps record 4's timing word; its core (rail 1, PMIC 0x11, 450 mV + 3.125 mV * ++ * code, GPU only) and buck8 (rail 8, PMIC 0x18, 600 mV + 3.125 mV * code) codes default to record ++ * 4's (138 = 881.25 mV, 124 = 987.5 mV), never lower, raised only on request in whole 3.125 mV ++ * codes, ++ * and never above the ceilings: the boot parameters (default state 4 + 50 mV, the CPU overclock's ++ * rail-offset cap) bounded by the compiled maxima below (state 4 + 150 mV; raising those needs a ++ * rebuild). The encoding itself reaches 1296 mV on both rails; that is not an operating point. ++ */ ++#ifndef G5P_GPU_OC_H ++#define G5P_GPU_OC_H ++ ++#include "bits-compat.h" ++ ++#define G5P_PSINFO_BASE 0x70000U /* PMGR syscon offset */ ++#define G5P_PSINFO_REC(n) (G5P_PSINFO_BASE + 0x10U * (n)) ++#define G5P_OC_STATE 5U ++#define G5P_OC_PIN 500U /* boot_gpu_perf for state 5 */ ++#define G5P_OC_STOCK4 \ ++ 0x8a2f1041U /* record 4 word 0 as LLB writes it on this chip */ ++#define G5P_OC_STOCK4_BUCK8 124U /* record 4 word 1 [7:0] */ ++#define G5P_OC_MIN_MHZ 726U /* the first grid step above 723 */ ++#define G5P_OC_VCO_MAX 2160U /* Apple's highest GPU-domain VCO */ ++#define G5P_OC_MAX_MHZ (G5P_OC_VCO_MAX / 2) /* postdiv 2: 1080 */ ++#define G5P_OC_VCO_STOCK_MAX \ ++ 1650U /* the highest VCO of a live state (550 MHz) */ ++#define G5P_OC_VALID BIT(21) ++#define G5P_OC_PLL_MASK 0x001ff1ffU /* mult, prediv, postdiv-1 */ ++#define G5P_OC_UV_STEP 3125U ++#define G5P_OC_CORE_BASE_UV 450000U ++#define G5P_OC_BUCK8_BASE_UV 600000U ++/* 881250 */ ++#define G5P_OC_CORE_STOCK_UV \ ++ (G5P_OC_CORE_BASE_UV + G5P_OC_UV_STEP * (G5P_OC_STOCK4 >> 24)) ++/* 987500 */ ++#define G5P_OC_BUCK8_STOCK_UV \ ++ (G5P_OC_BUCK8_BASE_UV + G5P_OC_UV_STEP * G5P_OC_STOCK4_BUCK8) ++/* the boot parameters' default ceiling: P4 + 50 mV */ ++#define G5P_OC_DEFAULT_HEADROOM_UV 50000U ++#define G5P_OC_ABS_HEADROOM_UV \ ++ 150000U /* compiled: P4 + 150 mV, no parameter goes above */ ++ ++struct g5p_oc_pll { ++ u32 mhz, mult, prediv, postdiv; ++ bool llb; /* Apple's own descriptor */ ++}; ++ ++static const struct g5p_oc_pll g5p_oc_llb[] = { ++ { 804, 67, 1, 2, true }, /* LLB id 0x22, all three GPU sections */ ++ { 850, 425, 6, 2, true }, /* LLB id 0x23, the live section */ ++ { 900, 75, 1, 2, true }, /* LLB id 0x24, the live section */ ++}; ++ ++/* ++ * The PLL for a requested MHz: Apple's descriptor at 804/850/900 (the request or its 3 MHz ++ * rounding), ++ * else the 3 MHz grid (rounded down) with record 4's dividers; false outside 726..1080. ++ */ ++static inline bool g5p_oc_pll(u32 want, struct g5p_oc_pll *p) ++{ ++ u32 mhz = want / 3 * 3; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_oc_llb); i++) ++ if (want == g5p_oc_llb[i].mhz || mhz == g5p_oc_llb[i].mhz) { ++ *p = g5p_oc_llb[i]; ++ return true; ++ } ++ if (mhz < G5P_OC_MIN_MHZ || mhz > G5P_OC_MAX_MHZ) ++ return false; ++ *p = (struct g5p_oc_pll){ mhz, mhz / 3, 4, 2, false }; ++ return true; ++} ++ ++/* A rail voltage as a code: exact multiples of 3.125 mV above the base only. */ ++static inline bool g5p_oc_code(u32 uv, u32 base, u32 *code) ++{ ++ if (uv < base || (uv - base) % G5P_OC_UV_STEP || ++ (uv - base) / G5P_OC_UV_STEP > 0xff) ++ return false; ++ *code = (uv - base) / G5P_OC_UV_STEP; ++ return true; ++} ++ ++/* The voltage codes for a request (0 = state 4's), bounded below by state 4 and above by the ++ * ceilings (each already clamped to the compiled maximum by g5p_oc_ceiling()). ++ */ ++ ++/* Record 5's words 0 and 1 from the live record 4 (stock only), a PLL and the voltage codes. */ ++static inline bool g5p_oc_words(const u32 r4[4], u32 want, u32 core, u32 buck8, ++ u32 w[4], struct g5p_oc_pll *p) ++{ ++ if (r4[0] != G5P_OC_STOCK4 || (r4[1] & 0xff) != G5P_OC_STOCK4_BUCK8 || ++ !g5p_oc_pll(want, p) || core < (G5P_OC_STOCK4 >> 24) || ++ core > 0xff || buck8 < G5P_OC_STOCK4_BUCK8 || buck8 > 0xff) ++ return false; ++ w[0] = core << 24 | G5P_OC_VALID | p->mult << 12 | p->prediv << 4 | ++ (p->postdiv - 1); ++ w[1] = (r4[1] & ~0xffU) | buck8; ++ w[2] = r4[2]; ++ w[3] = r4[3]; ++ return true; ++} ++ ++/* Hz of a record word 0, as iBoot 87001c3ac decodes it (0 if the predivider is 0). */ ++static inline u64 g5p_oc_hz(u32 w0) ++{ ++ u32 mult = (w0 >> 12) & 0x1ff, prediv = (w0 >> 4) & 0x1f, ++ postdiv = (w0 & 0xf) + 1; ++ ++ return prediv ? 24000000ULL * mult / prediv / postdiv : 0; ++} ++ ++/* A nonzero record 5 that this driver writes: record 4's timing and word-3 words, valid, a PLL ++ * g5p_oc_pll() produces, voltage codes between state 4's and the compiled maxima. Anything else in ++ * slot 5 is not ours: never overwritten or cleared. ++ */ ++static inline bool g5p_oc_is_ours(const u32 r4[4], const u32 r5[4]) ++{ ++ u32 core = r5[0] >> 24, buck8 = r5[1] & 0xff, w[4], cmax, bmax; ++ struct g5p_oc_pll p; ++ u64 hz = g5p_oc_hz(r5[0]); ++ ++ if (!g5p_oc_code(G5P_OC_CORE_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_CORE_BASE_UV, &cmax) || ++ !g5p_oc_code(G5P_OC_BUCK8_STOCK_UV + G5P_OC_ABS_HEADROOM_UV, ++ G5P_OC_BUCK8_BASE_UV, &bmax) || ++ core > cmax || buck8 > bmax || !hz || hz % 1000000ULL || ++ !g5p_oc_words(r4, (u32)(hz / 1000000ULL), core, buck8, w, &p)) ++ return false; ++ return w[0] == r5[0] && w[1] == r5[1] && w[2] == r5[2] && w[3] == r5[3]; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/heartbeat116.h b/drivers/gpu/apple-g5p/heartbeat116.h +new file mode 100644 +index 000000000..428146764 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/heartbeat116.h +@@ -0,0 +1,175 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_HEARTBEAT116_H ++#define G5P_HEARTBEAT116_H ++ ++/* ++ * Heartbeat116: the R74 IRQ174 storm bound as a window rate, not a lifetime count. ++ * ++ * A healthy R74 epoch takes IRQ174 at about 575/s and consumes about 290 event-ring ++ * entries/s, idle or not: one type-0 ALIVE per firmware scheduler iteration, and two IRQ ++ * entries per ring entry (irqs = 2 * events - 18 over the Warm113 C4-prep R74 milestones; ++ * r81 measured 577/289 per s). Wake112 bounded both counts over the epoch's lifetime: ++ * IRQ entries against boot_r74_irq_cap (1048576) and ring entries against the compute event ++ * limit (524544). Either bound latched failed/event_failed, so an idle, healthy GPU faulted ++ * after about 30 minutes. ++ * ++ * Here both bounds are rates. A window opens at the first IRQ entry and closes at the first ++ * entry at least window_ns later, then re-bases on the totals of that entry. Totals are u32 ++ * and every in-window count is a u32 difference, so a wrap of a total is harmless. ++ * ++ * RATE more than irq_ceiling IRQ entries in the open window (checked on every entry) ++ * EVENTS more than event_ceiling ring entries in the open window (checked on every entry) ++ * STALL stall_windows consecutive closed windows in which ++ * irqs > ratio * events + slack. Healthy traffic is 2:1. A latch that re-fires ++ * without ring progress, or a heartbeat after a fault stopped the drain, is not. ++ * ++ * A trip is absorbing. Nothing here clears a failure; the glue adds the same terminal stop ++ * as the lifetime cap had (cap_seen, failed, event_failed, one disable_irq_nosync). ++ * ++ * Pure policy: no register access, no lock, no kernel API. The caller serializes (the R73B ++ * IRQ thread is the only caller). ++ */ ++ ++#define G5P_HB_WINDOW_NS 1000000000ULL ++#define G5P_HB_IRQ_CEILING 16384U ++#define G5P_HB_EVENT_CEILING 8192U ++#define G5P_HB_RATIO 8U ++#define G5P_HB_SLACK 512U ++#define G5P_HB_STALL_WINDOWS 3U ++ ++enum g5p_hb_why { ++ G5P_HBW_NONE, ++ G5P_HBW_RATE, ++ G5P_HBW_EVENTS, ++ G5P_HBW_STALL, ++ G5P_HBW_N, ++}; ++ ++static const char *const g5p_hb_why_name[G5P_HBW_N] = { ++ "none", ++ "rate", ++ "events", ++ "stall", ++}; ++ ++enum g5p_hb_res { ++ G5P_HBR_OPEN, /* counted in the open window */ ++ G5P_HBR_CLOSED, /* this entry closed a window (h->last) */ ++ G5P_HBR_TRIP, /* tripped now or earlier */ ++}; ++ ++struct g5p_hb_cfg { ++ u64 window_ns; ++ u32 irq_ceiling, event_ceiling, ratio, slack, stall_windows; ++}; ++ ++/* The totals the IRQ thread samples after servicing one entry. */ ++struct g5p_hb_in { ++ u64 now_ns; ++ u32 irqs; /* IRQ174 entries (b->irqs) */ ++ u32 events; /* event-ring entries consumed (b->events) */ ++ u32 acks; /* AIC ACKs written by the R73B handler */ ++ u32 empty; /* entries whose service consumed no ring entry */ ++ u32 aux; /* auxiliary-ring entries consumed */ ++}; ++ ++/* One window: its start and length and its in-window counts. */ ++struct g5p_hb_win { ++ u64 start_ns, ns; ++ u32 irqs, events, acks, empty, aux; ++}; ++ ++struct g5p_hb { ++ u32 open; /* zero until the first entry (kzalloc) */ ++ u32 why; /* enum g5p_hb_why, written once */ ++ u32 windows; /* windows closed */ ++ u32 stalled; /* consecutive unproductive closed windows */ ++ u32 peak_irqs, peak_events; ++ u64 start_ns; ++ struct g5p_hb_in base; /* totals at the window's first entry */ ++ struct g5p_hb_win last; /* the last closed window */ ++ struct g5p_hb_win trip; /* the open window at the trip */ ++ u64 trip_ns; ++}; ++ ++static inline void g5p_hb_cfg_default(struct g5p_hb_cfg *c) ++{ ++ c->window_ns = G5P_HB_WINDOW_NS; ++ c->irq_ceiling = G5P_HB_IRQ_CEILING; ++ c->event_ceiling = G5P_HB_EVENT_CEILING; ++ c->ratio = G5P_HB_RATIO; ++ c->slack = G5P_HB_SLACK; ++ c->stall_windows = G5P_HB_STALL_WINDOWS; ++} ++ ++static inline void g5p_hb_count(const struct g5p_hb *h, ++ const struct g5p_hb_in *in, ++ struct g5p_hb_win *w) ++{ ++ w->start_ns = h->start_ns; ++ w->ns = in->now_ns >= h->start_ns ? in->now_ns - h->start_ns : 0; ++ w->irqs = in->irqs - h->base.irqs; ++ w->events = in->events - h->base.events; ++ w->acks = in->acks - h->base.acks; ++ w->empty = in->empty - h->base.empty; ++ w->aux = in->aux - h->base.aux; ++} ++ ++static inline bool g5p_hb_unproductive(const struct g5p_hb_cfg *c, ++ const struct g5p_hb_win *w) ++{ ++ return (u64)w->irqs > (u64)c->ratio * w->events + c->slack; ++} ++ ++static inline enum g5p_hb_res ++g5p_hb_trip(struct g5p_hb *h, u32 why, const struct g5p_hb_win *w, u64 now_ns) ++{ ++ h->why = why; ++ h->trip = *w; ++ h->trip_ns = now_ns; ++ return G5P_HBR_TRIP; ++} ++ ++/* One serviced IRQ entry. */ ++static inline enum g5p_hb_res g5p_hb_step(struct g5p_hb *h, ++ const struct g5p_hb_cfg *c, ++ const struct g5p_hb_in *in) ++{ ++ struct g5p_hb_win w; ++ ++ if (h->why) ++ return G5P_HBR_TRIP; ++ if (!h->open) { ++ h->open = 1; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ return G5P_HBR_OPEN; ++ } ++ g5p_hb_count(h, in, &w); ++ if (w.irqs > c->irq_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_RATE, &w, in->now_ns); ++ if (w.events > c->event_ceiling) ++ return g5p_hb_trip(h, G5P_HBW_EVENTS, &w, in->now_ns); ++ if (w.ns < c->window_ns) ++ return G5P_HBR_OPEN; ++ h->last = w; ++ h->windows++; ++ if (w.irqs > h->peak_irqs) ++ h->peak_irqs = w.irqs; ++ if (w.events > h->peak_events) ++ h->peak_events = w.events; ++ h->stalled = g5p_hb_unproductive(c, &w) ? h->stalled + 1 : 0; ++ h->start_ns = in->now_ns; ++ h->base = *in; ++ if (h->stalled >= c->stall_windows) ++ return g5p_hb_trip(h, G5P_HBW_STALL, &w, in->now_ns); ++ return G5P_HBR_CLOSED; ++} ++ ++/* Receipts: the first four closed windows, then every 64th. */ ++static inline bool g5p_hb_report(const struct g5p_hb *h) ++{ ++ return h->windows <= 4 || !(h->windows % 64); ++} ++ ++#endif /* G5P_HEARTBEAT116_H */ +diff --git a/drivers/gpu/apple-g5p/idle-power.inc b/drivers/gpu/apple-g5p/idle-power.inc +new file mode 100644 +index 000000000..d2ce71879 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/idle-power.inc +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Software/shared-RAM observation only. Never wake or probe GPU registers. */ ++static ssize_t idle_power_status_show(struct device *dev, ++ struct device_attribute *attr, char *buf) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_boot *b; ++ u32 p, state, target, mirror, period; ++ int ret; ++ ++ if (!mutex_trylock(&g->lock)) ++ return sysfs_emit(buf, "available=0 why=busy mmio=0\n"); ++ b = g->boot; ++ if (!b || !b->maps_ready) { ++ ret = sysfs_emit(buf, "available=0 why=no-maps mmio=0\n"); ++ goto out; ++ } ++ p = b->layout.offset[0x4a]; ++ dma_rmb(); ++ period = g5p_dc_root_word(b, p + 4); ++ state = g5p_dc_root_word(b, p + 8); ++ target = g5p_dc_root_word(b, p + 0x28); ++ mirror = g5p_dc_root_word(b, p + 0x34); ++ /* +34 is both a sampled Fender nibble and the setter's requested index. ++ * It is not a completed transition or stable-off certificate. Stale RAM ++ * remains readable across a stopped/failed epoch; label it explicitly. ++ */ ++ ret = sysfs_emit(buf, ++ "available=1 gen=%u alive=%u failed=%u stopped=%u powered=%u " ++ "policy_state=%u target=%u policy_pstate=%u sample_ms=%u " ++ "meta71c=%u source=shared-ram atomic=0 hw_ack=0 mmio=0\n", ++ g5p_ds_gen(), b->alive, b->failed, b->stopped, ++ g->powered, state, target, mirror, period, b->meta71c); ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++static DEVICE_ATTR_RO(idle_power_status); +diff --git a/drivers/gpu/apple-g5p/kick.inc b/drivers/gpu/apple-g5p/kick.inc +new file mode 100644 +index 000000000..6ef7395c6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/kick.inc +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Shared pre-kick step recovered from ca0358/ca05d4 and ca6644. */ ++static int g5p_prepare_kick(struct g5p_diag *g, const char *name) ++{ ++ u32 raw, state; ++ ++ if (gfx_bridge_replay && !strcmp(name, "RENDER-TA")) ++ g5p_r41_bridge_ta(g); ++ /* ca05d4 always invokes Fender +80 before inspecting GPC state. ++ * ca6644: W R1+14=7, then read fence. Hardware bit names unknown. ++ * This is distinct from acknowledging the host AIC interrupt. ++ */ ++ dev_info(g->dev, "%s Fender W R1+00014=00000007 before MTS kick\n", ++ name); ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ dev_info(g->dev, "%s Fender raw R1+00014=%08x (write fence)\n", name, ++ raw); ++ if (raw & ~7U) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ if (READ_ONCE(g->boot->failed)) ++ return -EIO; ++ return 0; ++} +diff --git a/drivers/gpu/apple-g5p/latency123.h b/drivers/gpu/apple-g5p/latency123.h +new file mode 100644 +index 000000000..164dc8e68 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/latency123.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: firmware words a powered iOS GPU has and our bootstrap leaves zero ++ * (docs/gpu/firmware-re/wake-latency-re.md sections 3.4 and 4.2). ++ * ++ * Arm A. The firmware copies meta+0x6c0..0x758 into its private block once, at init ++ * (MINIM 0x100009ac -> 0x10003ca4), so these words only act from the boot image: ++ * meta+0x6d4 = 1 per-tick cache maintenance 0x1001b638 (688/1720/e230/e138) ++ * meta+0x710/0x714 pstate table max index / count ++ * meta+0x718.. pstate MHz; the tick period is table[Fender+0x1000 & 0xf] ++ * policy+0x4 = 16 governor sample period, ms (the governor reads it live) ++ * 1 = capped: table { 0, 340 }, max index 1. The firmware's own pstate setter ++ * (0x1001c88c, Fender+0x1000 = index | 0x10) can then only ask for pstate 1, the state ++ * the host already requests, so the clock does not move. ++ * 2 = full: table { 0, 340, 474, 550, 723 }, max index 4, plus the limits README rank 2 ++ * pairs with it (control+0x1028/0x102c/0x1030 = 3348/400/400, policy+0x7cc/0x6fc = ++ * 400). The firmware may then raise the GPU clock itself; nothing on Linux programs ++ * a GPU voltage table, so this arm runs after the capped one. ++ * ++ * Arm B. ctl+0x04 is the per-engine tick-service mask (iOS 7). Every tick re-copies ++ * ctl+0x00..0x20 (0x10001328), so the word is live; the boot image gets the value the ++ * parameter holds at map and the render path re-applies it before each job. ++ * ++ * RAM only. Temporary A/B code: the winning words become unconditional init writes. ++ */ ++#ifndef APPLE_G5P_LATENCY123_H ++#define APPLE_G5P_LATENCY123_H ++ ++#include "bits-compat.h" ++#define G5P_L123_POLICY 0x4aU ++#define G5P_L123_META 0x4dU ++#define G5P_L123_CTL 0x5eU ++#define G5P_L123_TICK_MASK 7U ++#define G5P_L123_CAPPED 1U ++#define G5P_L123_FULL 2U ++ ++static const u32 g5p_l123_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline void g5p_l123_arm_a(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 arm) ++{ ++ u8 *meta = g5p_object(p, root, shared, G5P_L123_META); ++ u8 *policy = g5p_object(p, root, shared, G5P_L123_POLICY); ++ u8 *ctl = g5p_object(p, root, shared, G5P_L123_CTL); ++ u32 count = arm == G5P_L123_FULL ? ARRAY_SIZE(g5p_l123_mhz) : 2; ++ u32 i; ++ ++ g5p_put32(meta + 0x6d4, 1); ++ g5p_put32(meta + 0x710, count - 1); ++ g5p_put32(meta + 0x714, count); ++ for (i = 0; i < count; i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_l123_mhz[i]); ++ g5p_put32(policy + 4, 16); ++ if (arm != G5P_L123_FULL) ++ return; ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, 400); ++ g5p_put32(ctl + 0x1030, 400); ++ g5p_put32(policy + 0x7cc, 400); ++ g5p_put32(policy + 0x6fc, 400); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/ldr.h b/drivers/gpu/apple-g5p/ldr.h +new file mode 100644 +index 000000000..165aacf85 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/ldr.h +@@ -0,0 +1,154 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Deliberately small Apple G5P META LDR reader. No allocation or MMIO here. */ ++#ifndef APPLE_G5P_LDR_H ++#define APPLE_G5P_LDR_H ++ ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_LDR_MAX_RECORDS 128 ++#define G5P_LDR_MAX_BYTES (256 * 1024) ++ ++enum g5p_ldr_op { G5P_LOAD, G5P_ZERO, G5P_CONFIG, G5P_START }; ++ ++typedef int (*g5p_ldr_emit)(void *ctx, enum g5p_ldr_op op, u32 address, ++ u32 size, const u8 *data); ++ ++static inline u32 g5p_le32(const u8 *p) ++{ ++ return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; ++} ++ ++static inline u16 g5p_le16(const u8 *p) ++{ ++ return (u16)p[0] | (u16)p[1] << 8; ++} ++ ++static inline bool g5p_ldr_bounds(size_t len, u32 off, u32 size) ++{ ++ return off <= len && size <= len - off; ++} ++ ++static inline int g5p_ldr_l2(const u8 *fw, size_t len, u32 off, u16 tag, ++ const u8 **data, u32 *size) ++{ ++ u16 bytes; ++ ++ if (!g5p_ldr_bounds(len, off, 4)) ++ return -EINVAL; ++ bytes = g5p_le16(fw + off + 2); ++ if (bytes < 6 || !g5p_ldr_bounds(len, off, bytes) || ++ g5p_le16(fw + off) != tag) ++ return -EINVAL; ++ *data = fw + off + 4; ++ *size = bytes - 6; ++ return 0; ++} ++ ++/* Validate in a first pass (emit=NULL); only then use a hardware callback. */ ++static inline int g5p_ldr_walk(const u8 *fw, size_t len, g5p_ldr_emit emit, ++ void *ctx) ++{ ++ u32 pos, seen[G5P_LDR_MAX_RECORDS], total = 0; ++ unsigned int nr = 0, starts = 0; ++ ++ if (len != 119112 || g5p_le32(fw) != 0x01aa5500 || ++ !g5p_ldr_bounds(len, g5p_le32(fw + 4), 1)) ++ return -EINVAL; ++ pos = g5p_le32(fw + 8); ++ while (pos != 0xffffffff) { ++ const u8 *p, *data; ++ u32 command, bytes, addr = 0, size = 0, l2, next; ++ enum g5p_ldr_op op; ++ unsigned int i; ++ int ret; ++ ++ if (nr == G5P_LDR_MAX_RECORDS || !g5p_ldr_bounds(len, pos, 12)) ++ return -EINVAL; ++ for (i = 0; i < nr; i++) ++ if (seen[i] == pos) ++ return -EINVAL; ++ seen[nr++] = pos; ++ p = fw + pos; ++ command = g5p_le16(p); ++ bytes = g5p_le16(p + 2); ++ next = g5p_le32(p + 4); ++ if (bytes < 12 || (bytes & 3) || ++ !g5p_ldr_bounds(len, pos, bytes)) ++ return -EINVAL; ++ if (command & 0x10) { ++ if (command != 0x10 || bytes != 16) ++ return -EINVAL; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), 0x8010, ++ &data, &size); ++ if (ret) ++ return ret; ++ goto next_record; ++ } ++ switch (command) { ++ case 0: ++ if (bytes != 20) ++ return -EINVAL; ++ op = G5P_LOAD; ++ addr = g5p_le32(p + 8); ++ l2 = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, l2, 0x8000, &data, &size); ++ if (ret) ++ return ret; ++ break; ++ case 4: ++ if (bytes != 24) ++ return -EINVAL; ++ op = G5P_ZERO; ++ addr = g5p_le32(p + 8); ++ size = g5p_le32(p + 12); ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 16), 0x8004, ++ &data, &l2); ++ if (ret || l2) ++ return -EINVAL; ++ break; ++ case 5: ++ case 3: ++ if (bytes != 16) ++ return -EINVAL; ++ op = command == 5 ? G5P_CONFIG : G5P_START; ++ ret = g5p_ldr_l2(fw, len, g5p_le32(p + 8), ++ 0x8000 | command, &data, &size); ++ if (ret || !size || size % (command == 5 ? 12 : 16)) ++ return -EINVAL; ++ if (command == 5) { ++ for (i = 0; i < size; i += 12) ++ if (g5p_le32(data + i) != 2 || ++ (g5p_le32(data + i + 4) & 3)) ++ return -EINVAL; ++ } else { ++ if (++starts != 1 || size != 32 || ++ next != 0xffffffff || g5p_le32(data) != 1 || ++ g5p_le32(data + 16) != 0) ++ return -EINVAL; ++ } ++ break; ++ default: ++ return -EINVAL; ++ } ++ if (op == G5P_LOAD || op == G5P_ZERO) { ++ if ((addr & 3) || (size & 3) || !size || ++ size > G5P_LDR_MAX_BYTES || ++ addr > (u32)~0U - size || ++ total > G5P_LDR_MAX_BYTES - size) ++ return -EINVAL; ++ total += size; ++ } ++ if (emit) { ++ ret = emit(ctx, op, addr, size, data); ++ if (ret) ++ return ret; ++ } ++next_record: ++ pos = next; ++ } ++ return nr == 94 && starts == 1 ? 0 : -EINVAL; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +new file mode 100644 +index 000000000..9a896569c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -0,0 +1,1032 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Post-alive transport and park experiments; never implicit recovery. */ ++static const bool control_experiment = 1; ++static const bool irq_experiment = 0; ++static const bool teardown_experiment = 1; ++ ++/* A complete sparse image: first sample omits only zero words, later samples ++ * emit every changed word (including transitions to zero). All allocated ++ * root/shared memory is covered, including unknown status/trace buffers. ++ * These are CPU coherent RAM observations, never proof of META cache contents. ++ * IRQs can update memory concurrently; this is deliberately not an atomic dump. ++ */ ++static u32 g5p_dc_root_word(struct g5p_boot *b, u32 offset) ++{ ++ return READ_ONCE(*(u32 *)(b->root[offset / G5P_GPU_PAGE].cpu + ++ offset % G5P_GPU_PAGE)); ++} ++ ++/* nopatch: the policy record (slot 0x4a) words 0x1000d044 decides readiness from: +8 state, ++ * +0xc substate, +0x34 pstate mirror; +0 and +4 (sample period) for context. iOS keeps +8 at 0. ++ */ ++static const u32 g5p_np_words[5] = { 0, 4, 8, 0xc, 0x34 }; ++ ++static void g5p_np_read(struct g5p_boot *b, u32 *w) ++{ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_np_words); i++) ++ w[i] = g5p_dc_root_word(b, b->layout.offset[0x4a] + ++ g5p_np_words[i]); ++} ++ ++static void g5p_np_print(struct g5p_diag *g, const char *stage, u32 seq, u32 e, ++ const u32 *w) ++{ ++ dev_info( ++ g->dev, ++ "NOPATCH policy stage=%s seq=%u engine=%u +0=%08x +4=%08x +8=%08x +c=%08x " ++ "+34=%08x events=%u end=nopatch\n", ++ stage, seq, e, w[0], w[1], w[2], w[3], w[4], ++ READ_ONCE(g->boot->events)); ++} ++ ++/* RAM only; nothing unless the unpatched image is latched. */ ++static void g5p_np_policy(struct g5p_diag *g, const char *stage, u32 seq, u32 e) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 w[ARRAY_SIZE(g5p_np_words)]; ++ ++ if (!b || !b->r73b || !b->r73b->nopatch) ++ return; ++ g5p_np_read(b, w); ++ g5p_np_print(g, stage, seq, e, w); ++} ++ ++/* Read fixed allocated policy objects, never follow firmware-owned pointers. ++ * 1000d044/1000d504 test these states;1001c900 stores requested pstate. ++ * These are sampled boundary predicates, NOT entry/return breadcrumbs or PC. ++ */ ++ ++/* Firmware MMU recovery uses event mask200 and reads e150/e158. ++ * Read raw qword halves without ACK/clear; concurrent updates can tear them. ++ * An absent sampled event does not exclude an earlier firmware-handled fault. ++ */ ++static void g5p_fault_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ u32 event, lo, hi, lo2, hi2; ++ ++ g5p_mmu_progress(g, stage, 0x130); ++ event = readl(g->rgx + 0x130); ++ g5p_mmu_progress(g, stage, 0xe150); ++ lo = readl(g->rgx + 0xe150); ++ g5p_mmu_progress(g, stage, 0xe154); ++ hi = readl(g->rgx + 0xe154); ++ g5p_mmu_progress(g, stage, 0xe158); ++ lo2 = readl(g->rgx + 0xe158); ++ g5p_mmu_progress(g, stage, 0xe15c); ++ hi2 = readl(g->rgx + 0xe15c); ++ g5p_mmu_record( ++ g, ++ "MMU %s event130=%08x pending=%u e150=%08x:%08x e158=%08x:%08x raw-split read-only\n", ++ stage, event, !!(event & 0x200), hi, lo, hi2, lo2); ++} ++ ++/* Same active-wrapper gate as CDM observations. Never write/ACK a register. ++ * 1001c9b8 samples R0+164/160/164; bound host sampling even on rollover. ++ * A split sample with different high words is explicitly unstable. ++ */ ++ ++static void g5p_dc_snapshot(struct g5p_diag *g, const char *stage, bool first) ++{ ++ { ++ return; ++ } ++} ++ ++static int g5p_control_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 *work = (u32 *)g5p_boot_shared(b, 0x68); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ u32 ctl_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6e)); ++ u32 data_meta = g5p_meta_address(g5p_object_va(&b->layout, 0x6f)); ++ u8 *init = b->root[0].cpu; ++ u8 *handoff = b->image[0].cpu + 0x3ff8; ++ u32 *events = (u32 *)g5p_boot_shared(b, 0x70); ++ ++ /* c9f868/c9f884: validate the loaded boot handoff, not just its builder. */ ++ if (g5p_le32(handoff) != g5p_meta_address(G5P_ROOT_VA) || ++ g5p_le32(handoff + 4) != ++ g5p_meta_address(g5p_object_va(&b->layout, 0x5f))) ++ return -EPROTO; ++ /* Drain under event_lock before checking that ALIVE has been returned. */ ++ if (g5p_r73b_service(g, G5P_R73B_OWNER_PREPARE, true)) ++ return -EIO; ++ if (READ_ONCE(events[0]) > 255 || READ_ONCE(events[4]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(events[0]) != READ_ONCE(events[4])) ++ return -EBUSY; ++ dev_info(g->dev, ++ "DC boot handoff verified; event consumer=%u producer=%u\n", ++ READ_ONCE(events[0]), READ_ONCE(events[4])); ++ ++ ; ++ ; ++ ; ++ ; ++ /* c96be8: init+18/+1c are META addresses, never CPU/DMA pointers. */ ++ if (g5p_le32(init + 0x18) != ctl_meta || ++ g5p_le32(init + 0x1c) != data_meta) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC mapping ctl GPU=%llx META=%08x data GPU=%llx META=%08x; init verified\n", ++ g5p_object_va(&b->layout, 0x6e), ctl_meta, ++ g5p_object_va(&b->layout, 0x6f), data_meta); ++ ++ if (b->work_ready) ++ return -EBUSY; ++ if (b->control_ready) ++ return b->r73b && 1 ? 0 : -EALREADY; ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (consumer != producer || READ_ONCE(work[0]) != READ_ONCE(work[4]) || ++ (b->work_submitted && !b->work_complete)) ++ return -EBUSY; ++ /* c993f8, size=0: no payload copy, ctrl+1020=0, command={8,0,0,0,0}. ++ * No claim that this is an opcode-independent firmware NOP. ++ */ ++ g5p_put32(g5p_boot_shared(b, 0x5e) + 0x1020, 0); ++ b->control_target = (producer + 1) & 255; ++ b->control_ready = true; ++ dev_info( ++ g->dev, ++ "DC prepare cmd=8 payload_words=0 consumer=%u producer=%u target=%u; RAM only\n", ++ consumer, producer, b->control_target); ++ return 0; ++} ++ ++static int g5p_r73e_prepare(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ ret = g5p_control_prepare(g); ++ if (!ret) ++ WRITE_ONCE(b->r73b->fast_window, true); ++ dev_info( ++ g->dev, ++ "R73E cmd8 prepared-before-release ret=%d target=%u end=r73e\n", ++ ret, b->control_target); ++ return ret; ++} ++ ++/* Apple's GPC precondition (AGXMetaFender_setGPCPerformanceState, ...6ca4ccc): a request in ++ * device test shows R1+0x1000 bit 4; wait for it to clear before judging the state. R1 reads only, ++ * bounded by boot_alive_settle_us; with 0 the first read stands, as before. ++ */ ++static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw) ++{ ++ u64 start; ++ ++ if (!r->settle_us || !(raw & 0x10)) ++ return raw; ++ if (!r->settle_first) ++ r->settle_first = raw; ++ start = ktime_get_mono_fast_ns(); ++ while ((raw & 0x10) && ktime_get_mono_fast_ns() - start < ++ (u64)r->settle_us * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(g->window[1] + 0x1000); ++ r->settle_samples++; ++ } ++ r->settle_ns += ktime_get_mono_fast_ns() - start; ++ return raw; ++} ++ ++static int g5p_r73e_irq_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 off, raw; ++ enum g5p_pxg gate; ++ ++ if (!b->control_ready || b->control_submitted) ++ return -EPERM; ++ /* wake112: release_ready, or a deferred R74E proof that the read below settles. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate != G5P_PXG_READY && gate != G5P_PXG_PROVE) ++ return -EHOSTDOWN; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ if (gate == G5P_PXG_PROVE && !g5p_r74e_take(g)) ++ return -EHOSTDOWN; ++ r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (gate == G5P_PXG_PROVE) ++ g5p_r74e_settle(r, r->early_state); ++ if (!g5p_gpc_ok(r->early_state)) ++ return -EHOSTDOWN; ++ /* nopatch: five RAM loads, the state the readiness gate will see before this bell. */ ++ if (r->nopatch) ++ g5p_np_read(b, r->np_pre); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_IRQ, 0); ++ /* Functional pre-kick sequence only; no optional diagnostics here. */ ++ writel(7, g->window[1] + 0x14); ++ raw = readl(g->window[1] + 0x14); ++ if (raw & ~7U) ++ return -EPROTO; ++ r->early_after_state = ++ g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ if (!g5p_gpc_ok(r->early_after_state)) ++ return -EHOSTDOWN; ++ dev_emerg( ++ g->dev, ++ "R73E doorbell begin first-ALIVE W0+00b00 value=00000010 end=r73e\n"); ++ writel(0x10, g->rgx + 0xb00); ++ dev_emerg(g->dev, "R73E doorbell end first-ALIVE W0+00b00 end=r73e\n"); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_IRQ, 0); ++ return 0; ++} ++ ++static int g5p_r73e_alive_submit(struct g5p_diag *g, u32 owner) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ enum g5p_pxg gate; ++ u32 px; ++ int ret; ++ ++ /* wake112: a deferred R74E proof admits the once-owner as release_ready does. */ ++ gate = g5p_px_gate(READ_ONCE(r->early_power_abort), ++ READ_ONCE(r->early_release_ready), ++ atomic_read_acquire(&r->early_proof)); ++ if (gate == G5P_PXG_REFUSE) ++ return -EHOSTDOWN; ++ if (gate == G5P_PXG_WAIT) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_busy, 0, 1)) ++ return -EAGAIN; ++ if (atomic_cmpxchg(&r->early_once, 0, 1)) { ++ ret = READ_ONCE(r->early_ret); ++ goto out; ++ } ++ r->early_irq_ns = ktime_get_mono_fast_ns(); ++ r->early_ret = g5p_r73e_irq_submit(g); ++ ret = r->early_ret; ++ WRITE_ONCE(r->fast_window, false); ++ if (ret == -EHOSTDOWN) ++ WRITE_ONCE(r->early_power_abort, true); ++ dev_emerg( ++ g->dev, ++ "R74E ALIVE submit case=%u owner=%u state=%08x after=%08x ret=%d " ++ "release_ready=%u power_abort=%u submitted=%u end=r74e\n", ++ r->r73e_case, owner, r->early_state, r->early_after_state, ret, ++ r->early_release_ready, r->early_power_abort, ++ READ_ONCE(b->control_submitted)); ++ if (r->settle_first) ++ dev_emerg( ++ g->dev, ++ "R74E settle first=%08x state=%08x after=%08x samples=%u ns=%llu limit_us=%u " ++ "ret=%d end=r74e\n", ++ r->settle_first, r->early_state, r->early_after_state, ++ r->settle_samples, r->settle_ns, r->settle_us, ret); ++ if (r->nopatch && READ_ONCE(b->control_submitted)) ++ g5p_np_print(g, "pre-bell", 0, 0, r->np_pre); ++ px = atomic_read(&r->early_proof); ++ /* raw: the proof read, or U32_MAX when it did not happen (a refusal or no take). */ ++ if (px != G5P_PX_NONE) ++ dev_emerg( ++ g->dev, ++ "R74E112 proof owner=%u state=%s why=%s gen=%u/%u raw=%08x release_ready=%u " ++ "power_abort=%u end=r74e112\n", ++ owner, g5p_px_name[min_t(u32, px, G5P_PX_N - 1)], ++ g5p_px_why_name[min_t(u32, r->early_proof_why, ++ G5P_PXW_N - 1)], ++ r->early_proof_gen, r->early_proof_gen_now, ++ px == G5P_PX_PROVEN || ++ r->early_proof_why == G5P_PXW_STATE ? ++ r->early_state : ++ U32_MAX, ++ r->early_release_ready, r->early_power_abort); ++out: ++ atomic_set(&r->early_busy, 0); ++ return ret; ++} ++ ++static int g5p_control_submit(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 producer = (b->control_target - 1) & 255; ++ u32 state, off; ++ int ret; ++ ++ if (b->control_submitted) ++ return b->r73b && 1 ? 0 : -EPERM; ++ if (!b->control_ready) ++ return -EPERM; ++ /* R73 synchronous pre-publish pass closes the race where trace records ++ * do not raise a final IRQ. A negative result is scoreable only when the ++ * factor reaches cmd8 with every outbound auxiliary consumer caught up. ++ */ ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_PUBLISH, true); ++ if (ret) ++ return ret; ++ if (READ_ONCE(ctl[0]) != producer || READ_ONCE(ctl[4]) != producer) ++ return -EPROTO; ++ state = readl(g->window[1] + 0x1000); ++ dev_info(g->dev, "DC raw R1+01000=%08x (require active before kick)\n", ++ state); ++ if (!g5p_gpc_ok(state)) ++ return -EHOSTDOWN; ++ g5p_dc_snapshot(g, "before-publish", true); ++ off = b->layout.offset[0x6d] + producer * 20; ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *dst = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ g5p_put32(dst, i ? 0 : 8); ++ } ++ WRITE_ONCE(b->control_submitted, true); ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], b->control_target); ++ /* ca05d4: full dmb ish after producer, before Fender and MTS. */ ++ mb(); ++ g5p_r73b_crumb(b, G5P_R73B_DC_PUBLISHED, G5P_R73B_OWNER_PUBLISH, 0); ++ for (u32 i = 0; i < 5; i++) { ++ u32 at = off + i * 4; ++ u8 *src = b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ ++ if (g5p_le32(src) != (i ? 0 : 8)) ++ return -EBADMSG; ++ } ++ if (READ_ONCE(ctl[4]) != b->control_target) ++ return -EPROTO; ++ dev_info( ++ g->dev, ++ "DC slot verified words=00000008/0/0/0/0 producer=%u; full barrier\n", ++ b->control_target); ++ ret = g5p_prepare_kick(g, "DC"); ++ if (ret) ++ return ret; ++ ; ++ if (b->r73_aux_drain) { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_KICK, true); ++ if (ret) ++ return ret; ++ } ++ g5p_dc_snapshot(g, "before-kick", false); ++ dev_info( ++ g->dev, ++ "DC publish cmd=8 slot=%u target=%u; W R0+00b00=%08x variant=%u\n", ++ producer, b->control_target, b->dc_variant == 4 ? 0x12 : 0x10, ++ b->dc_variant); ++ /* 0x12 is verified for CL, NOT for DC: variant 4 tests dispatch routing. */ ++ writel(7 == 4 ? 0x12 : 0x10, g->rgx + 0xb00); ++ g5p_r73b_crumb(b, G5P_R73B_DC_KICKED, G5P_R73B_OWNER_KICK, 0); ++ g5p_dc_snapshot(g, "after-kick", false); ++ return 0; ++} ++ ++/* perf60 pin >100 (seed 1-5, poweron): the policy record as the firmware left it, from RAM, plus the ++ * real Fender state through an R1-only ticket of the register licence (any pstate, bit 29 allowed). ++ * The sampler 0x1000aa80 stores its own "now" at policy+0x918 on every call (0x1000ab44) and the ++ * current state at +0x34 (0x1000ab18), so fw_now against the host's arch timer names the ++ * firmware's clock, and +0x34 / +0 / +0x8f8 name the branch it took. ++ */ ++static void g5p_perf_readback(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 pol = b->layout.offset[0x4a], meta = b->layout.offset[0x4d]; ++ u64 host = arch_timer_read_counter(), rate = arch_timer_get_rate(); ++ u64 p8f8, p918, p920, m784; ++ struct g5p_mmio_ticket t; ++ ++ if (!b->gpu_perf || (!b->perf_seed && !b->perf_poweron)) ++ return; ++ dma_rmb(); ++ p8f8 = g5p_dc_root_word(b, pol + 0x8f8) | ++ (u64)g5p_dc_root_word(b, pol + 0x8fc) << 32; ++ p918 = g5p_dc_root_word(b, pol + 0x918) | ++ (u64)g5p_dc_root_word(b, pol + 0x91c) << 32; ++ p920 = g5p_dc_root_word(b, pol + 0x920) | ++ (u64)g5p_dc_root_word(b, pol + 0x924) << 32; ++ m784 = g5p_dc_root_word(b, meta + 0x784) | ++ (u64)g5p_dc_root_word(b, meta + 0x788) << 32; ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ dev_emerg( ++ g->dev, ++ "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " ++ "pol28=%u pol2c=%u pol30=%u pol34=%u p8f8=%llx fw_now918=%llx p920=%llx " ++ "meta784=%llx host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " ++ "host_minus_fw_counter_us=%lld r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", ++ why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, ++ g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30), ++ g5p_dc_root_word(b, pol + 0x34), p8f8, p918, p920, m784, host, ++ b->seed_stamp, ++ b->seed_stamp ? ++ div_s64((s64)(p918 - m784 - b->seed_stamp) * 1000000LL, ++ rate) : ++ 0, ++ div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate), ++ g5p_mmio_did(&t), t.gpc, t.r1_0); ++} ++ ++static int g5p_r73c_fast_control(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *r = b->r73b; ++ int ret; ++ ++ r->fast_release_ns = ktime_get_mono_fast_ns(); ++ WRITE_ONCE(r->fast_window, true); ++ ret = g5p_boot_events(g); ++ r->fast_alive_ns = ktime_get_mono_fast_ns(); ++ if (!ret) ++ ret = g5p_control_prepare(g); ++ if (!ret) { ++ r->fast_state = readl(g->window[1] + 0x1000); ++ ret = g5p_control_submit(g); ++ } ++ r->fast_submit_ns = ktime_get_mono_fast_ns(); ++ r->fast_ret = ret; ++ WRITE_ONCE(r->fast_window, false); ++ g5p_r73b_events_exit(g, ret); ++ dev_info( ++ g->dev, ++ "R73C fast release_to_alive_ns=%llu release_to_submit_ns=%llu state=%08x ret=%d " ++ "control_submitted=%u end=r73c\n", ++ r->fast_alive_ns - r->fast_release_ns, ++ r->fast_submit_ns - r->fast_release_ns, r->fast_state, ret, ++ READ_ONCE(b->control_submitted)); ++ g5p_perf_readback(g, ret ? "alive-timeout" : "alive"); ++ return ret; ++} ++ ++/* perf60 pin 200 (docs/gpu/experiments/perf60-clock.md "Pin 200: the guarded drain read" and ++ * "Clock-change drain RE" §4): at a boot control timeout with boot_gpu_perf set (before any job, ++ * the firmware alive), every read goes through the GPU register licence (mmio.inc) and nothing ++ * wakes the GPU (no R1+0x14 write, mmio-access.md G6): ++ * r1: R1 +0, +8, +0x14, +0x1000, +0x1004, +0x20000 (one R1-only ticket, any pstate) ++ * drain: 11f00 688 1720 1728 e230 e238 e138, then the clock words R0+0x0/0x4/0x8/0xc ++ * timer: R0+0x160/0x164, twice, 1 ms apart ++ * tx: T0/T1 TXENABLE/TXSTATUS (0x04800000/10, 0x04801000/10) through the META slave port ++ * pc: (bit 1 only) T0 PC: RQ 0x0480fff8 = 0x10005, poll bit 31, read 0x0480fff0; x8, 100 us ++ * apart. This is the driver's first slave-port write (CTRL0 = addr, DATAT = value). ++ * One ticket per group or slave-port op (<= G5P_MMIO_MAX_OPS), a refusal retried <= 2 ms. ++ */ ++ ++#define G5P_GPP_BUDGET_US 2000U ++ ++/* One META slave-port read in one ticket: ready, CTRL0 = addr | 1, read back, ready (<= 4 ++ * polls), DATAX: 8 ops. -EBUSY when the port is not ready within the ticket. ++ */ ++static int g5p_mmio_sp_read(struct g5p_diag *g, u32 addr, u32 *out, u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1 = 0; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr | 1)) { ++ g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL0); ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ *out = g5p_mmio_r0_read(g, &t, G5P_MSLVDATAX); ++ ret = 0; ++ break; ++ } ++ } ++ } ++ if (!g5p_mmio_release(g, &t) && !ret) ++ ret = -ESTALE; ++ return ret; ++} ++ ++/* One META slave-port write in one ticket: ready, CTRL0 = addr, DATAT = value, ready (<= 4). */ ++static int g5p_mmio_sp_write(struct g5p_diag *g, u32 addr, u32 value, ++ u32 *tries) ++{ ++ struct g5p_mmio_ticket t; ++ u32 c1; ++ int ret; ++ ++ if (!g5p_slave_access_allowed(READ_ONCE(g->reset_transition))) ++ return -EPERM; ++ ret = g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries); ++ if (ret) ++ return ret; ++ ret = -EBUSY; ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVCTRL0, addr) && ++ g5p_mmio_r0_write(g, &t, G5P_MSLVDATAT, value)) { ++ for (u32 i = 0; i < 4; i++) { ++ c1 = g5p_mmio_r0_read(g, &t, G5P_MSLVCTRL1); ++ if ((c1 & G5P_SP_READY) == G5P_SP_READY) { ++ ret = 0; ++ break; ++ } ++ } ++ } ++ g5p_mmio_release(g, &t); ++ return ret; ++} ++ ++/* A group of plain R0 reads (<= G5P_MMIO_MAX_OPS) under one ticket; returns the ticket's did. */ ++static const char *g5p_gpp_r0(struct g5p_diag *g, const u32 *off, u32 *v, u32 n, ++ u32 *tries, u32 *gpc) ++{ ++ struct g5p_mmio_ticket t; ++ ++ if (g5p_mmio_hold_wait(g, &t, 0, G5P_GPP_BUDGET_US, tries)) { ++ *gpc = t.gpc; ++ return g5p_mmio_did(&t); ++ } ++ for (u32 i = 0; i < n; i++) ++ v[i] = g5p_mmio_r0_read(g, &t, off[i]); ++ g5p_mmio_release(g, &t); ++ *gpc = t.gpc; ++ g5p_mmio_sample_line(g, G5P_MMS_PROBE, &t, v, n, true); ++ return g5p_mmio_did(&t); ++} ++ ++/* What a slave-port op's return means, for the dc-probe lines. */ ++static const char *g5p_sp_why(int ret) ++{ ++ switch (ret) { ++ case 0: ++ return "ok"; ++ case -ENODATA: ++ return "not-run"; ++ case -EPERM: ++ return "forbidden-after-release"; /* g->reset_transition: boot.inc's post-release ban */ ++ case -ENODEV: ++ return "licence-off"; ++ case -EAGAIN: ++ return "licence-refused"; ++ case -EBUSY: ++ return "port-not-ready"; ++ case -ESTALE: ++ return "stale"; ++ case -ETIMEDOUT: ++ return "rq-no-ready"; ++ default: ++ return "error"; ++ } ++} ++ ++static void g5p_gpu_perf_probe(struct g5p_diag *g) ++{ ++ static const u32 drain[] = { 0x11f00, 0x688, 0x1720, 0x1728, ++ 0xe230, 0xe238, 0xe138 }; ++ static const u32 clk[] = { 0x0, 0x4, 0x8, 0xc }; ++ static const u32 timer[] = { 0x160, 0x164 }; ++ static const u32 r1off[] = { 0x0, 0x8, 0x14, 0x1000, 0x1004, 0x20000 }; ++ static const u32 tx[] = { 0x04800000, 0x04800010, 0x04801000, ++ 0x04801010 }; ++ u32 mode = (0); ++ struct g5p_boot *b = g->boot; ++ u32 vd[ARRAY_SIZE(drain)], vc[ARRAY_SIZE(clk)], vt[2][2], ++ r1[ARRAY_SIZE(r1off)]; ++ u32 vx[ARRAY_SIZE(tx)], pc[8], pcrq[8], gate = U32_MAX, tries = 0, ++ gpc = U32_MAX, gpc_x; ++ int txr[ARRAY_SIZE(tx)], pcr[8]; ++ const char *d_r1 = "skip-guard", *d_drain = "skip-guard", ++ *d_clk = "skip-guard"; ++ const char *d_t0 = "skip-guard", *d_t1 = "skip-guard"; ++ struct g5p_mmio_ticket t; ++ ++ if (!mode || !b->gpu_perf) ++ return; ++ memset(vd, 0xff, sizeof(vd)); ++ memset(vc, 0xff, sizeof(vc)); ++ memset(vt, 0xff, sizeof(vt)); ++ memset(r1, 0xff, sizeof(r1)); ++ memset(vx, 0xff, sizeof(vx)); ++ memset(pc, 0xff, sizeof(pc)); ++ memset(pcrq, 0xff, sizeof(pcrq)); ++ for (u32 i = 0; i < ARRAY_SIZE(txr); i++) ++ txr[i] = -ENODATA; ++ for (u32 i = 0; i < ARRAY_SIZE(pcr); i++) ++ pcr[i] = -ENODATA; ++ if (g->pmgr && regmap_read(g->pmgr, 0x80340, &gate)) ++ gate = U32_MAX; ++ if (READ_ONCE(b->failed)) ++ goto print; ++ /* r1: before anything else, R1-only (valid at any pstate and with bit 29 set) */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) { ++ for (u32 i = 0; i < ARRAY_SIZE(r1off); i++) ++ r1[i] = g5p_mmio_r1_read(g, &t, r1off[i]); ++ g5p_mmio_release(g, &t); ++ } ++ d_r1 = g5p_mmio_did(&t); ++ /* gpc: the drain group's licence read; every group's ticket also has its own sample line */ ++ d_drain = g5p_gpp_r0(g, drain, vd, ARRAY_SIZE(drain), &tries, &gpc); ++ d_clk = g5p_gpp_r0(g, clk, vc, ARRAY_SIZE(clk), &tries, &gpc_x); ++ d_t0 = g5p_gpp_r0(g, timer, vt[0], 2, &tries, &gpc_x); ++ usleep_range(1000, 1100); ++ d_t1 = g5p_gpp_r0(g, timer, vt[1], 2, &tries, &gpc_x); ++ for (u32 i = 0; i < ARRAY_SIZE(tx); i++) ++ txr[i] = g5p_mmio_sp_read(g, tx[i], &vx[i], &tries); ++ if (mode & 2) { ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) { ++ pcr[i] = g5p_mmio_sp_write(g, 0x0480fff8, 0x10005, ++ &tries); ++ for (u32 k = 0; !pcr[i] && k < 4; k++) { ++ pcr[i] = g5p_mmio_sp_read(g, 0x0480fff8, ++ &pcrq[i], &tries); ++ if (pcr[i] || (pcrq[i] & BIT(31))) ++ break; ++ udelay(10); ++ } ++ if (!pcr[i]) ++ pcr[i] = pcrq[i] & BIT(31) ? ++ g5p_mmio_sp_read(g, 0x0480fff0, ++ &pc[i], ++ &tries) : ++ -ETIMEDOUT; ++ udelay(100); ++ } ++ } ++print: ++ dev_err(g->dev, ++ "GPUPERF dc-probe did=%s gate=%08x r1=%s r1_0=%08x r1_8=%08x r1_14=%08x " ++ "r1_1000=%08x r1_1004=%08x r1_20000=%08x gpc=%08x tries=%u end=gpuperf\n", ++ d_drain, gate, d_r1, r1[0], r1[1], r1[2], r1[3], r1[4], r1[5], ++ gpc, tries); ++ dev_err(g->dev, ++ "GPUPERF dc-probe drain=%s 11f00=%08x 688=%08x 1720=%08x 1728=%08x e230=%08x " ++ "e238=%08x e138=%08x clk=%s clk=%08x:%08x clkstat=%08x:%08x end=gpuperf\n", ++ d_drain, vd[0], vd[1], vd[2], vd[3], vd[4], vd[5], vd[6], d_clk, ++ vc[1], vc[0], vc[3], vc[2]); ++ dev_err(g->dev, ++ "GPUPERF dc-probe timer=%s/%s 160=%08x/%08x 164=%08x/%08x gap_us=1000 " ++ "txen0=%08x/%s txst0=%08x/%s txen1=%08x/%s txst1=%08x/%s end=gpuperf\n", ++ d_t0, d_t1, vt[0][0], vt[1][0], vt[0][1], vt[1][1], vx[0], ++ g5p_sp_why(txr[0]), vx[1], g5p_sp_why(txr[1]), vx[2], ++ g5p_sp_why(txr[2]), vx[3], g5p_sp_why(txr[3])); ++ if (mode & 2) ++ for (u32 i = 0; i < ARRAY_SIZE(pc); i++) ++ dev_err(g->dev, ++ "GPUPERF dc-probe pc i=%u ret=%s rq=%08x t0pc=%08x end=gpuperf\n", ++ i, g5p_sp_why(pcr[i]), pcrq[i], pc[i]); ++} ++ ++static int g5p_control_wait(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long start = jiffies, deadline = start + 3 * HZ; ++ bool sampled = false, rekicked = false; ++ u32 consumer, producer, pass = 0; ++ int ret; ++ ++ if (!b->control_submitted || b->control_complete) ++ return -EPERM; ++ do { ++ ret = g5p_r73b_service(g, G5P_R73B_OWNER_WAIT, ++ g5p_r73b_sampled(++pass)); ++ if (ret) ++ return ret; ++ consumer = READ_ONCE(ctl[0]); ++ producer = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ if (consumer > 255 || producer != b->control_target || ++ (consumer != b->control_target && ++ consumer != ((b->control_target - 1) & 255))) ++ return -EPROTO; ++ if (consumer == b->control_target) { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (g5p_r74_case(b) == 1) { ++ WRITE_ONCE(b->control_complete, true); ++ g5p_r74_mark(g, G5P_R74_CMD8_CONSUMED, ++ consumer); ++ dev_emerg( ++ g->dev, ++ "R74B cmd8-consumed consumer=%u producer=%u IRQ174=%d ram_only=1 end=r74b\n", ++ consumer, producer, ++ atomic_read(&b->irqs)); ++ g5p_np_policy(g, "cmd8-consumed", consumer, 0); ++ return 0; ++ } ++ g5p_dc_snapshot(g, "consumed", false); ++ b->control_complete = true; ++ ; ++ dev_info( ++ g->dev, ++ "DC consumed cmd=8 consumer=%u producer=%u IRQ174=%d; transport only\n", ++ consumer, producer, atomic_read(&b->irqs)); ++ return 0; ++ } ++ if (!sampled && !time_before(jiffies, start + HZ / 10)) { ++ sampled = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "100ms", false); ++ g5p_np_policy(g, "100ms", consumer, 0); ++ } ++ if (7 == 3 && !rekicked && !time_before(jiffies, start + HZ)) { ++ /* Re-notify the SAME slot once; never publish a second command. */ ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ ret = g5p_prepare_kick(g, "DC re-kick"); ++ if (ret) ++ return ret; ++ mb(); ++ writel(0x10, g->rgx + 0xb00); ++ rekicked = true; ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "re-kick", false); ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (!g5p_r74b_ram_only(g)) ++ g5p_dc_snapshot(g, "timeout", false); ++ g5p_np_policy(g, "timeout", consumer, 0); ++ dev_err(g->dev, ++ "DC timeout consumer=%u producer=%u target=%u; retain RAM, no retry\n", ++ consumer, producer, b->control_target); ++ /* perf60 (boot_gpu_perf): the policy-state words at the stall, RAM only. pol34 = the pstate the ++ * firmware last requested (its setter stores it before the Fender write): q means it reached the ++ * setter, 1 that it stopped before (the clock-change drain 0x1001cec8 -> 0x1001b638). ++ */ ++ if (b->gpu_perf) { ++ u32 pol = b->layout.offset[0x4a]; ++ ++ dma_rmb(); ++ dev_err(g->dev, ++ "GPUPERF dc-timeout pin=%u pol8=%u/%u q=%u x=%u pol34=%u req=%u/%u ceil=%u/%u " ++ "seed=%u pol0=%u pol4=%u pol28=%u pol2c=%u pol30=%u end=gpuperf\n", ++ b->gpu_perf, g5p_dc_root_word(b, pol + 8), ++ g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x38), ++ g5p_dc_root_word(b, pol + 0x34), ++ g5p_dc_root_word(b, pol + 0x6f0), ++ g5p_dc_root_word(b, pol + 0x6f4), ++ g5p_dc_root_word(b, pol + 0x688), ++ g5p_dc_root_word(b, pol + 0x68c), b->perf_seed, ++ g5p_dc_root_word(b, pol + 0), ++ g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30)); ++ g5p_gpu_perf_probe(g); ++ g5p_perf_readback(g, "dc-timeout"); ++ } ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ b->control_timed_out = true; ++ return -ETIMEDOUT; ++} ++ ++static int g5p_irq_test(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + HZ / 10; ++ int before = atomic_read(&b->irqs), ret; ++ ++ if (b->irq_tested || !b->irq_ready || g5p_r74_case(b)) ++ return -EPERM; ++ /* ADT set/ack addresses, not a GPU slave-port transaction. Firmware may ++ * also interrupt concurrently: this tests delivery, not event provenance. ++ */ ++ dev_info(g->dev, "IRQ test baseline=%d W AIC+014=00004000\n", before); ++ writel(0x4000, b->swirq + 0x14); ++ do { ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ if (atomic_read(&b->irqs) > before) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ b->irq_tested = true; ++ ; ++ dev_info( ++ g->dev, ++ "IRQ delivery observed before=%d after=%d; no shader completion implied\n", ++ before, atomic_read(&b->irqs)); ++ return 0; ++ } ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ dev_err(g->dev, "IRQ delivery timeout before=%d after=%d\n", before, ++ atomic_read(&b->irqs)); ++ return -ETIMEDOUT; ++} ++ ++static int g5p_boot_park(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ static const u32 slots[] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ bool failed_control = READ_ONCE(b->failed) && b->control_timed_out; ++ bool failed_work = READ_ONCE(b->failed) && b->work_timed_out; ++ bool failed_context = READ_ONCE(b->failed) && b->context_timed_out; ++ bool failed_timeout = failed_control || failed_work || failed_context; ++ int ret; ++ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ if (READ_ONCE(b->event_failed) || ++ (b->control_submitted && !b->control_complete && !failed_control) || ++ (b->work_submitted && !b->work_complete && !failed_work) || ++ (b->context_submitted && !b->context_complete && !failed_context)) ++ return -EBUSY; ++ /* A failed drain must never be bypassed. On the sole permitted failure ++ * path require the event ring empty below without consuming anything. ++ */ ++ if (!failed_timeout) { ++ ret = g5p_boot_drain(g); ++ if (ret) ++ return ret; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(slots); i++) { ++ u32 *ctl = (u32 *)g5p_boot_shared(b, slots[i]); ++ u32 consumer = READ_ONCE(ctl[0]), producer = READ_ONCE(ctl[4]); ++ ++ if (consumer > 255 || producer > 255) ++ return -EPROTO; ++ if (((failed_control || failed_context) && slots[i] == 0x6c) || ++ (failed_work && slots[i] == 0x68)) { ++ u32 target = failed_context && slots[i] == 0x6c ? ++ b->context_target : ++ slots[i] == 0x6c ? b->control_target : ++ b->work_target; ++ ++ if (producer != target || ++ (consumer != producer && ++ consumer != ((producer - 1) & 255))) ++ return -EPROTO; ++ } else if (consumer != producer) { ++ return -EBUSY; ++ } ++ } ++ if (READ_ONCE(b->event_failed)) ++ return -EIO; ++ if (failed_timeout) ++ dev_info( ++ g->dev, ++ "PARK failed %s timeout; abandon command, preserve failure and RAM\n", ++ failed_context ? "CONTEXT" : ++ failed_work ? "CL" : ++ "DC"); ++ /* Freeze commands before the first transition; no retry even if it fails. ++ * c9fa10 enabled-Fender branch: core=0, GPC=0, no a18 access. ++ */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ dev_info( ++ g->dev, ++ "PARK begin core=0 GPC=0; wrapper only, DMA memory retained\n"); ++ ret = g5p_fender_idle(&io); ++ if (ret) ++ return ret; ++ /* Keep the IRQ/ack handler and every allocation alive until reboot. ++ * GPC=0 alone is not proof that outstanding bus traffic has drained. ++ */ ++ b->parked = true; ++ dev_info( ++ g->dev, ++ "PARK GPC=0 verified; quarantine until reboot, no PMGR off or DMA free\n"); ++ return 0; ++} ++ ++static int g5p_lifecycle_command(struct g5p_diag *g, const char *command) ++{ ++ struct g5p_boot *b = g->boot; ++ int ret; ++ ++ if (!boot_experiment || !b || !b->alive || b->stopped) ++ return -EPERM; ++ if (b->failed && ++ !(sysfs_streq(command, "park") && teardown_experiment && ++ (b->control_timed_out || b->work_timed_out || ++ b->context_timed_out))) ++ return -EPERM; ++ if (!strncmp(command, "control-", 8) && control_experiment) { ++ if (sysfs_streq(command, "control-prepare")) ++ ret = g5p_control_prepare(g); ++ else if (sysfs_streq(command, "control-submit")) ++ ret = g5p_control_submit(g); ++ else if (sysfs_streq(command, "control-wait")) ++ ret = g5p_control_wait(g); ++ else ++ return -EINVAL; ++ } else if (sysfs_streq(command, "irq-test") && ++ (g5p_r74_case(b) || (1))) { ++ /* R74 never runs the AIC self-test r73f case 01 stopped in. */ ++ if (b->r73b) ++ atomic_inc(&b->r73b->r74_irq_refused); ++ g5p_r74_mark(g, G5P_R74_IRQ_TEST_REFUSED, -EPERM); ++ dev_err(g->dev, ++ "R74 refuse irq-test case=%u irq_tested=%u end=r74\n", ++ g5p_r74_case(b), b->irq_tested); ++ return -EPERM; ++ } else if (sysfs_streq(command, "irq-test") && irq_experiment) { ++ ret = g5p_irq_test(g); ++ } else if (sysfs_streq(command, "park") && teardown_experiment) { ++ ret = g5p_boot_park(g); ++ } else { ++ return -EPERM; ++ } ++ if (sysfs_streq(command, "control-wait")) ++ g5p_r73b_crumb(b, G5P_R73B_WAIT_EXIT, G5P_R73B_OWNER_WAIT, ret); ++ if (ret && ret != -EALREADY && ret != -EPERM) ++ WRITE_ONCE(b->failed, true); ++ return ret; ++} ++ ++/* Outgoing-side kexec/reboot quiesce; see docs/gpu/experiments/gpu-reset-recovery-production.md ++ * section 6.10. NOT a reset: Fender core=0/GPC=0 only, no PMGR off, no DMA free and ++ * no R0 access. It acts only when the command lock is free (no job or sysfs command is ++ * inside a hardware wait), the firmware was released and the pstate policy is ++ * host-owned (init meta+0x6c8 == 0); every other state is left alone. Unlike ++ * g5p_boot_park() it does not require drained rings: round 10 flew park after an ++ * unconsumed DC command, and at shutdown there is nothing left to drain for. ++ * GPC=0 is not a proof that bus traffic drained (see g5p_boot_park). ++ */ ++ ++static void g5p_td_shutdown(struct g5p_diag *g); ++ ++static void g5p_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ struct g5p_boot *b; ++ int ret; ++ ++ if (!g) ++ return; ++ /* Idempotent second chance after the reboot notifier; a no-op unless teardown != 0. */ ++ g5p_td_shutdown(g); ++ { ++ return; ++ }; ++ /* A worker or sysfs writer stuck in MMIO holds g->lock: never queue behind it. */ ++ if (!mutex_trylock(&g->lock)) { ++ dev_warn(g->dev, "SHUTDOWN park skipped: command lock busy\n"); ++ return; ++ } ++ b = g->boot; ++ /* R1 is touched only with ps_gfx held on (the retained runtime-PM reference), ++ * the wrapper woken and the firmware released. ++ */ ++ if (!b || !g->powered || !g->fender_ready || !b->maps_ready || ++ !b->released || b->stopped || b->parked) { ++ dev_info(g->dev, "SHUTDOWN park not needed\n"); ++ goto out; ++ } ++ /* With firmware-owned policy (init meta+0x6c8 != 0) the firmware also writes ++ * R1+0x1000, so a host write would race it (dsx-r74b). RAM-only read of the ++ * host-written init word, the same one g5p_dc_snapshot() reports. ++ */ ++ if (g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8)) { ++ dev_info( ++ g->dev, ++ "SHUTDOWN park refused: firmware-owned policy (meta+0x6c8 != 0)\n"); ++ goto out; ++ } ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ /* Freeze commands before the first transition; no retry even if it fails. */ ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++ ret = g5p_fender_idle(&io); ++ if (!ret) ++ WRITE_ONCE(b->parked, true); ++ dev_info( ++ g->dev, ++ "SHUTDOWN park core=0 GPC=0 result=%d; DMA memory retained, no PMGR off\n", ++ ret); ++out: ++ mutex_unlock(&g->lock); ++} +diff --git a/drivers/gpu/apple-g5p/meta71c114.h b/drivers/gpu/apple-g5p/meta71c114.h +new file mode 100644 +index 000000000..cc84dbd82 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/meta71c114.h +@@ -0,0 +1,65 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency114: one RAM dword, meta+0x71c, in the bootstrap image. ++ * ++ * meta (object 0x4d) +0x718 is the firmware's pstate frequency table in MHz. ++ * Its init copies the table to private+0x26c at boot (MINIM 0x1000122c..127c), ++ * and the scheduler loop reloads its timer from entry 1 on both branches ++ * (0x8000a1a4 via private+0x2c4 at pstate 1, 0x8000a338 via private+0x270). ++ * Linux leaves the table zero; Apple has { 0, 340, 474, 550, 723 }. The A/B ++ * installs entry 1 only: 0 is the control (nothing is stored, variant 7 leaves ++ * the word zero), 340 the factor. Everything else is refused at map, before ++ * any store. The store goes into the staging buffer, before g5p_boot_copy() ++ * publishes the image and long before the firmware is released. There is no ++ * register access here. ++ */ ++#ifndef APPLE_G5P_META71C114_H ++#define APPLE_G5P_META71C114_H ++ ++#define G5P_M71C_SLOT 0x4dU ++#define G5P_M71C_OFFSET 0x71cU ++#define G5P_M71C_CONTROL 0U ++#define G5P_M71C_FACTOR 340U ++#define G5P_M71C_VARIANT 7U ++ ++enum { ++ G5P_M71C_OK, ++ G5P_M71C_BAD_VALUE, ++ G5P_M71C_BAD_VARIANT, ++ G5P_M71C_WHYS, ++}; ++ ++static const char *const g5p_m71c_whys[G5P_M71C_WHYS] = { ++ "none", ++ "value", ++ "variant", ++}; ++ ++/* Latched at map: only the two pre-registered arms, the factor on dc7 only. */ ++static inline u32 g5p_m71c_check(u32 requested, u32 variant) ++{ ++ if (requested != G5P_M71C_CONTROL && requested != G5P_M71C_FACTOR) ++ return G5P_M71C_BAD_VALUE; ++ if (requested == G5P_M71C_FACTOR && variant != G5P_M71C_VARIANT) ++ return G5P_M71C_BAD_VARIANT; ++ return G5P_M71C_OK; ++} ++ ++/* ++ * One store into the staging image (the factor only) and its read-back. The ++ * caller fails the map unless the read-back equals the request. ++ */ ++static inline u32 g5p_m71c_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 requested, bool *wrote) ++{ ++ u8 *word = g5p_object(p, root, shared, G5P_M71C_SLOT) + G5P_M71C_OFFSET; ++ ++ *wrote = false; ++ if (requested == G5P_M71C_FACTOR) { ++ g5p_put32(word, requested); ++ *wrote = true; ++ } ++ return g5p_le32(word); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio-licence.h b/drivers/gpu/apple-g5p/mmio-licence.h +new file mode 100644 +index 000000000..d9d23ca2a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio-licence.h +@@ -0,0 +1,148 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU register licence (docs/gpu/experiments/mmio-access.md §4), pure so the host test ++ * (scripts/gpu/perf60-host/test_mmio.c) drives the same code. iOS's ++ * AGXMetaFirmware_isGPURegisterAccessible (fffffff006ca10ec) licenses an R0 access when PMGR gfx ++ * is on and the wrapper is up (fender+0xc), R1+0 bit 29 is clear (the firmware's power-down / ++ * clock-change routine 0x1001cec8 raises it before the GPC goes to 0) and the GPC pstate nibble is ++ * 1..meta+0x710. We are stricter: R1+0x1000 bit 4 (a transition in flight) must be clear and ++ * R1+0 == 0 (a dead wrapper, Apple's ca05d4 vt+0x70 branch) refuses too. ++ * ++ * Order, and nothing else touches the bus: the RAM terms (no MMIO when any fails), then R1+0, then ++ * R1+0x1000, then at most G5P_MMIO_MAX_OPS R0 accesses, then R1+0x1000 and R1+0 again at release ++ * to account a sample whose licence changed under it (stale: data loss, not exclusion). ++ * A refusal never waits and never wakes (no R1+0x14 write): the caller decides whether to retry. ++ */ ++#ifndef G5P_MMIO_LICENCE_H ++#define G5P_MMIO_LICENCE_H ++ ++#include "bits-compat.h" ++ ++#define G5P_MMIO_MAX_OPS 8U /* R0 accesses per ticket */ ++#define G5P_MMIO_R1_CTRL \ ++ 0x0U /* R1+0: bit 29 = firmware power-down in progress */ ++#define G5P_MMIO_R1_GPC \ ++ 0x1000U /* R1+0x1000: bit 4 transition, nibble = pstate */ ++#define G5P_MMIO_BIT29 BIT(29) ++ ++enum g5p_mmio_why { ++ G5P_MMIO_OK, ++ G5P_MMIO_OFF, /* RAM: power epoch not readable (no MMIO done) -> -ENODEV */ ++ G5P_MMIO_BUSY, /* the lock was held (try mode, IRQ census) -> -EAGAIN, no MMIO */ ++ G5P_MMIO_DEAD, /* R1+0 == 0 */ ++ G5P_MMIO_FWOFF, /* R1+0 bit 29 */ ++ G5P_MMIO_TRANS, /* R1+0x1000 bit 4 */ ++ G5P_MMIO_PARKED, /* nibble 0 */ ++ G5P_MMIO_OVERMAX, /* nibble > max pstate */ ++ G5P_MMIO_PARKEPOCH, /* R0 in a park epoch (g5p_mmio_r0_regime_ok) */ ++ G5P_MMIO_NR, ++}; ++ ++static const char *const g5p_mmio_why_names[G5P_MMIO_NR] = { ++ "read", "skip-off", "skip-busy", ++ "skip-dead", "skip-bit29", "skip-transition", ++ "skip-parked", "skip-overmax", "skip-park-epoch", ++}; ++ ++/* The RAM preconditions (§4.2 step 2). b->failed is deliberately not one: the fault read runs ++ * after it latches, as Apple's does. ++ */ ++struct g5p_mmio_ram { ++ bool powered, fender_ready, mmio_off, have_boot, released, stopped; ++}; ++ ++static inline bool g5p_mmio_ram_ok(const struct g5p_mmio_ram *r) ++{ ++ return r->powered && r->fender_ready && !r->mmio_off && r->have_boot && ++ r->released && !r->stopped; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_r1_0(u32 r1_0) ++{ ++ if (!r1_0) ++ return G5P_MMIO_DEAD; ++ if (r1_0 & G5P_MMIO_BIT29) ++ return G5P_MMIO_FWOFF; ++ return G5P_MMIO_OK; ++} ++ ++static inline enum g5p_mmio_why g5p_mmio_check_gpc(u32 gpc, u32 max) ++{ ++ if (gpc & 0x10) ++ return G5P_MMIO_TRANS; ++ if (!(gpc & 0xf)) ++ return G5P_MMIO_PARKED; ++ if ((gpc & 0xf) > max) ++ return G5P_MMIO_OVERMAX; ++ return G5P_MMIO_OK; ++} ++ ++/* ++ * R0 only where the firmware cannot park under it. The governor's park does not raise R1+0 ++ * bit 29 (mmio-access.md §2.3: only the power-down routine 0x1001cec8 does, a few calls before it ++ * drops the GPC), so against a park the licence's R1 checks are a race: a check that passes can be ++ * followed by the GPC going to 0 before the R0 load. The never-park envelope (boot_gpu_perf >= ++ * 100, latched at map: the policy writer's x >= 100, q >= 1, the governor never selects pstate 0) ++ * has no such window; at pin 400/500 the 3D-completion and clock samples saw bit 29 0 times, while ++ * the park tuple's rung 5 (2026-11-03) saw it on 749 of 983 3D completions and then froze the SoC. ++ * iOS reads R0 only at a fault event, never at idle. `override` = mmio_r0_park (DFU-class rungs ++ * only). ++ */ ++static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool override) ++{ ++ return override || gpu_perf >= 100; ++} ++ ++/* ++ * meta+0x710 as written into the boot image; 0 (no table) means the firmware stays at pstate 1. ++ */ ++static inline u32 g5p_mmio_max_pstate(u32 meta710) ++{ ++ return meta710 < 1 ? 1 : meta710 > 15 ? 15 : meta710; ++} ++ ++typedef u32 (*g5p_mmio_rd_t)(void *ctx, u32 off); ++ ++/* §4.2 steps 2-4 under the caller's lock: no bus access unless the RAM terms pass, R1+0 before ++ * R1+0x1000, nothing after a refusal. *r1_0 / *gpc stay U32_MAX when not read. ++ */ ++static inline enum g5p_mmio_why g5p_mmio_licence(const struct g5p_mmio_ram *ram, ++ u32 max, g5p_mmio_rd_t rd, ++ void *ctx, u32 *r1_0, u32 *gpc) ++{ ++ enum g5p_mmio_why why; ++ ++ *r1_0 = 0xffffffffU; ++ *gpc = 0xffffffffU; ++ if (!g5p_mmio_ram_ok(ram)) ++ return G5P_MMIO_OFF; ++ *r1_0 = rd(ctx, G5P_MMIO_R1_CTRL); ++ why = g5p_mmio_check_r1_0(*r1_0); ++ if (why) ++ return why; ++ *gpc = rd(ctx, G5P_MMIO_R1_GPC); ++ return g5p_mmio_check_gpc(*gpc, max); ++} ++ ++/* An R0 op is allowed while ops < G5P_MMIO_MAX_OPS; the counter counts refusals too. */ ++static inline bool g5p_mmio_op_take(u32 *ops) ++{ ++ return (*ops)++ < G5P_MMIO_MAX_OPS; ++} ++ ++/* §4.3 release: the licence fields (nibble, bit 4, bit 29) unchanged and still licensing. */ ++static inline bool g5p_mmio_stale(u32 r1_0, u32 gpc, u32 r1_0_end, u32 gpc_end, ++ u32 max) ++{ ++ if ((gpc & 0x1f) != (gpc_end & 0x1f) || ++ (r1_0 & G5P_MMIO_BIT29) != (r1_0_end & G5P_MMIO_BIT29)) ++ return true; ++ return g5p_mmio_check_r1_0(r1_0_end) || ++ g5p_mmio_check_gpc(gpc_end, max); ++} ++ ++/* mmio_selftest rung 3/5 expectations: CORE_REVISION and CORE_ID as read at bring-up. */ ++#define G5P_MMIO_EXPECT_20 0x00090f00U ++#define G5P_MMIO_EXPECT_18 0x011b0602U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/mmio.inc b/drivers/gpu/apple-g5p/mmio.inc +new file mode 100644 +index 000000000..b8c25bd8d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmio.inc +@@ -0,0 +1,295 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The one GPU register accessor (docs/gpu/experiments/mmio-access.md §4), Apple's licence from ++ * AGXMetaFirmware_isGPURegisterAccessible (mmio-licence.h) under g->mmio_lock: ++ * ++ * if (!g5p_mmio_hold(g, &t, 0)) { ++ * v = g5p_mmio_r0_read(g, &t, off); // <= G5P_MMIO_MAX_OPS, nothing else in between ++ * g5p_mmio_release(g, &t); // false: the licence changed under the sample ++ * } ++ * ++ * hold takes mmio_lock with IRQs off and returns with it held on success. It refuses instead of ++ * waiting and never wakes the GPU (no R1+0x14 write): -ENODEV when the power epoch is not ++ * readable (RAM only, no MMIO done), -EAGAIN when parked, bit 29, in transition, above the ++ * pstate table, or (G5P_MMIO_TRY) when the lock is taken; -EPERM for an R0 ticket in an epoch ++ * the firmware may park (boot_gpu_perf < 100, unless mmio_r0_park). Callable from hard IRQ, ++ * threaded IRQ and process context, inside event_lock and g->lock. Between hold and release: no ++ * printk, no other MMIO, no sleep, no allocation. ++ * ++ * mmio_selftest (0600, default 0, live) drives the proof rungs of §6: ++ * bit 0 rung 1/1b: 1 kHz hrtimer census, R1+0 and R1+0x1000 only (no R0) ++ * bit 1 rung 3/4: every 100 ms hold -> R0+0x20, R0+0x18 -> release (DFU-risk with parks: rung 4) ++ * bit 2 rung 5: the same at every 3D completion decision (drm side; DFU-risk, park tuple) ++ * bit 3 idle check: after 5, 30 and 120 s with no job, the PMGR gfx word, then (gfx on only) ++ * R1+0x1000 and R1+0; never R0 ++ * Lines: GPU-MMIO sample (per sample, at most one per source per second unless the outcome ++ * changes or the sample is stale/bad), GPU-MMIO idle, GPU-MMIO stats (every 10 s while on and ++ * once when it goes back to 0). ++ */ ++#define G5P_MMIO_TRY \ ++ BIT(0) /* trylock: refuse busy instead of spinning (the IRQ census) */ ++/* An R1-only ticket, counted in the census class: held whenever the RAM terms pass (R1 is ++ * readable while PMGR gfx is on, mmio-access.md §1.3), R1+0 and R1+0x1000 both read even when the ++ * licence refuses; R0 only if the licence passed (t->r0_ok). ++ */ ++#define G5P_MMIO_CENSUS BIT(1) ++ ++struct g5p_mmio_ticket { ++ u32 r1_0, gpc, r1_0_end, gpc_end, flags, ops; ++ u64 t_ns, t_end_ns; ++ unsigned long irq; ++ enum g5p_mmio_why why; ++ bool held, stale, r0_ok; ++}; ++ ++enum { ++ G5P_MMS_CENSUS, ++ G5P_MMS_IDLE, ++ G5P_MMS_3D, ++ G5P_MMS_FAULT, ++ G5P_MMS_PROBE, ++ G5P_MMS_NR ++}; ++static const char *const g5p_mms_names[G5P_MMS_NR] = { ++ "census", "idle-100ms", "3d-done", "fault-read", "dc-probe", ++}; ++ ++#define G5P_MMIO_RING 32 ++struct g5p_mmio_rec { ++ u64 t_ns; ++ u32 r1_0, gpc, us; ++ u8 why, stale; ++}; ++ ++struct g5p_mmio_self { ++ struct g5p_diag *g; ++ struct hrtimer timer; ++ struct delayed_work work; ++ bool census_on, was_on; ++ u64 stats_ns; ++ /* counters: [0] R1-only census tickets, [1] R0 tickets */ ++ atomic_t n[2][G5P_MMIO_NR]; ++ atomic_t stale[2], r0_ops, bad; ++ /* census transitions, written by the hrtimer, drained by the work */ ++ spinlock_t ring_lock; ++ struct g5p_mmio_rec ring[G5P_MMIO_RING]; ++ u32 ring_head, ring_tail, ring_drop; ++ struct g5p_mmio_rec census_last; ++ u8 census_prev_why, census_prev_stale; ++ /* per-source print limiter */ ++ u64 printed_ns[G5P_MMS_NR]; ++ u8 printed_why[G5P_MMS_NR]; ++ /* idle check */ ++ u64 job_ns; ++ u32 idle_done; ++ /* bit 4, the GPU clock: per Fender pstate nibble, the highest rate seen and the sample counts */ ++ u32 clk_max_khz[16], clk_cr_khz[16], clk_n[16], clk_busy[16], clk_reset; ++ u64 clk_printed_ns; ++}; ++ ++/* mmio_selftest bit 4: the gap between the two tickets of one clock sample */ ++ ++/* R0 tickets in a park epoch (boot_gpu_perf < 100): refused unless 1 (g5p_mmio_r0_regime_ok). */ ++ ++static u32 g5p_mmio_rd_r1(void *ctx, u32 off) ++{ ++ return readl((void __iomem *)ctx + off); ++} ++ ++static void g5p_mmio_count(struct g5p_diag *g, const struct g5p_mmio_ticket *t) ++{ ++ struct g5p_mmio_self *s = READ_ONCE(g->mmio_self); ++ u32 cls = t->flags & G5P_MMIO_CENSUS ? 0 : 1; ++ ++ if (!s) ++ return; ++ atomic_inc(&s->n[cls][t->why]); ++ if (t->stale) ++ atomic_inc(&s->stale[cls]); ++ if (t->ops && cls) ++ atomic_add(min_t(u32, t->ops, G5P_MMIO_MAX_OPS), &s->r0_ops); ++} ++ ++static int g5p_mmio_hold(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags) ++{ ++ struct g5p_mmio_ram ram = { 0 }; ++ struct g5p_boot *b; ++ unsigned long irq; ++ ++ memset(t, 0, sizeof(*t)); ++ t->flags = flags; ++ t->r1_0 = t->gpc = t->r1_0_end = t->gpc_end = U32_MAX; ++ if (flags & G5P_MMIO_TRY) { ++ if (!spin_trylock_irqsave(&g->mmio_lock, irq)) { ++ t->why = G5P_MMIO_BUSY; ++ goto refused; ++ } ++ } else { ++ spin_lock_irqsave(&g->mmio_lock, irq); ++ } ++ t->t_ns = ktime_get_mono_fast_ns(); ++ ram.mmio_off = g->mmio_off; ++ ram.powered = READ_ONCE(g->powered); ++ ram.fender_ready = READ_ONCE(g->fender_ready); ++ /* mmio_off first: a boot struct behind a gate is never dereferenced */ ++ b = ram.mmio_off ? NULL : READ_ONCE(g->boot); ++ if (b) { ++ ram.have_boot = true; ++ ram.released = READ_ONCE(b->released); ++ ram.stopped = READ_ONCE(b->stopped); ++ } ++ /* before any bus access: an R0 ticket in a park epoch is refused outright */ ++ if (!(flags & G5P_MMIO_CENSUS) && b && ++ !g5p_mmio_r0_regime_ok(b->gpu_perf, (0))) { ++ t->why = G5P_MMIO_PARKEPOCH; ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ t->why = g5p_mmio_licence(&ram, READ_ONCE(g->mmio_max_pstate), ++ g5p_mmio_rd_r1, g->window[1], &t->r1_0, ++ &t->gpc); ++ if (t->why && (!(flags & G5P_MMIO_CENSUS) || t->why == G5P_MMIO_OFF)) { ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ spin_unlock_irqrestore(&g->mmio_lock, irq); ++ goto refused; ++ } ++ if (t->gpc == U32_MAX) ++ t->gpc = readl( ++ g->window[1] + ++ G5P_MMIO_R1_GPC); /* census: after an R1+0 refusal */ ++ t->r0_ok = !t->why; ++ t->irq = irq; ++ t->held = true; ++ return 0; ++refused: ++ g5p_mmio_count(g, t); ++ /* a park epoch does not change by waiting: -EPERM, never retried (g5p_mmio_hold_wait) */ ++ return t->why == G5P_MMIO_OFF ? -ENODEV : ++ t->why == G5P_MMIO_PARKEPOCH ? -EPERM : ++ -EAGAIN; ++} ++ ++/* Only between a successful hold and its release; past G5P_MMIO_MAX_OPS no bus access. */ ++static u32 g5p_mmio_r0_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->rgx + off); ++} ++ ++/* R0 writes count as ops too (the slave-port request words of the pin-200 probe). */ ++static bool g5p_mmio_r0_write(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off, u32 v) ++{ ++ if (!t->held || !t->r0_ok || !g5p_mmio_op_take(&t->ops)) ++ return false; ++ writel(v, g->rgx + off); ++ return true; ++} ++ ++/* Any R1 word under a held ticket (census or licensed); counts as an op. */ ++static u32 g5p_mmio_r1_read(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ u32 off) ++{ ++ if (!t->held || !g5p_mmio_op_take(&t->ops)) ++ return U32_MAX; ++ return readl(g->window[1] + off); ++} ++ ++/* Process context: refusals (-EAGAIN) retried outside the lock every 20-40 us, <= budget_us. */ ++static int g5p_mmio_hold_wait(struct g5p_diag *g, struct g5p_mmio_ticket *t, ++ unsigned int flags, u32 budget_us, u32 *tries) ++{ ++ u64 t0 = ktime_get_mono_fast_ns(); ++ int ret; ++ ++ might_sleep(); ++ for (;;) { ++ (*tries)++; ++ ret = g5p_mmio_hold(g, t, flags); ++ if (ret != -EAGAIN || ktime_get_mono_fast_ns() - t0 >= ++ (u64)budget_us * NSEC_PER_USEC) ++ return ret; ++ usleep_range(20, 40); ++ } ++} ++ ++static bool g5p_mmio_release(struct g5p_diag *g, struct g5p_mmio_ticket *t) ++{ ++ if (!t->held) ++ return false; ++ t->gpc_end = readl(g->window[1] + G5P_MMIO_R1_GPC); ++ t->r1_0_end = readl(g->window[1] + G5P_MMIO_R1_CTRL); ++ t->stale = t->r0_ok && ++ g5p_mmio_stale(t->r1_0, t->gpc, t->r1_0_end, t->gpc_end, ++ READ_ONCE(g->mmio_max_pstate)); ++ t->t_end_ns = ktime_get_mono_fast_ns(); ++ t->held = false; ++ spin_unlock_irqrestore(&g->mmio_lock, t->irq); ++ g5p_mmio_count(g, t); ++ return !t->stale; ++} ++ ++static const char *g5p_mmio_did(const struct g5p_mmio_ticket *t) ++{ ++ return t->stale ? "stale" : g5p_mmio_why_names[t->why]; ++} ++ ++static u32 g5p_mmio_us(const struct g5p_mmio_ticket *t) ++{ ++ return t->t_end_ns > t->t_ns ? ++ (u32)div_u64(t->t_end_ns - t->t_ns, NSEC_PER_USEC) : ++ 0; ++} ++ ++/* The per-sample line, after release. force: print regardless of the per-source limiter. */ ++static void g5p_mmio_sample_line(struct g5p_diag *g, u32 src, ++ const struct g5p_mmio_ticket *t, u32 *v, ++ u32 nv, bool force) ++{ ++ struct g5p_mmio_self *s = g->mmio_self; ++ u64 now = ktime_get_mono_fast_ns(); ++ char vs[9 * G5P_MMIO_MAX_OPS + 1] = "-"; ++ u32 len = 0; ++ ++ if (s && !force && !t->stale && s->printed_why[src] == t->why && ++ now - s->printed_ns[src] < NSEC_PER_SEC) ++ return; ++ if (s) { ++ s->printed_ns[src] = now; ++ s->printed_why[src] = t->why; ++ } ++ for (u32 i = 0; i < nv && i < G5P_MMIO_MAX_OPS; i++) ++ len += scnprintf(vs + len, sizeof(vs) - len, "%s%08x", ++ i ? "," : "", v[i]); ++ dev_info( ++ g->dev, ++ "GPU-MMIO sample why=%s did=%s r1_0=%08x gpc=%08x r1_0_end=%08x gpc_end=%08x " ++ "max=%u v=%s us=%u end=mmio\n", ++ g5p_mms_names[src], g5p_mmio_did(t), t->r1_0, t->gpc, ++ t->r1_0_end, t->gpc_end, READ_ONCE(g->mmio_max_pstate), vs, ++ g5p_mmio_us(t)); ++} ++ ++/* Rung 3/5: one licensed ticket, CORE_REVISION and CORE_ID. */ ++ ++/* drm side: rung 5 at the 3D completion decision, and the idle clock for bit 3. */ ++ ++/* Bit 4: the GPU core clock. CR_TIMER (R0+0x160) ticks once per 256 core clocks whether or not a ++ * phase is open (r3clk: 2823 kHz at pstate 4 = 722.7 MHz, 3703 kHz at 948 MHz), so CR_TIMER over ++ * host time x 256 is the clock at any moment. The Rogue PERF_TA_CYCLE / PERF_3D_CYCLE / ++ * PERF_TA_OR_3D_CYCLE counters (R0+0x6020/0x6028/0x6038, 32 bit) advance at the core clock while a ++ * TA or 3D phase is open (340.0 MHz through open TA phases, docs/gpu/experiments/46-device-tests.md): ++ * over host time, the highest rate per pstate (a window busy all through) is the clock again. One ++ * sample = two licensed tickets, mmio_clock_gap_us apart, each reading the three counters and ++ * CR_TIMER's low word back to back. A cycle rate above 2 GHz is a firmware counter clear ++ * (R0+0x6000 = 1, 0x1001ab00) inside the window and is dropped. ++ */ ++ ++/* Bit 3: does the GPC power down at full idle in state 0? The PMGR word is always readable; ++ * R1 only while the gate reads on (actual == target == 0xf), through a census ticket so the RAM ++ * terms and the lock still apply. A refused licence still returns the R1 words it read. ++ */ +diff --git a/drivers/gpu/apple-g5p/mmu-attrs.inc b/drivers/gpu/apple-g5p/mmu-attrs.inc +new file mode 100644 +index 000000000..694552016 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-attrs.inc +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include ++#include ++ ++/* Read the actual stable kernel mapping of a retained DMA allocation. ++ * DMA memory may be a vmalloc PTE or a cached linear-map block. ++ */ ++static u64 g5p_table_cpu_descriptor(unsigned long va) ++{ ++ pgd_t *pgd = pgd_offset_k(va); ++ p4d_t *p4d; ++ pud_t *pud; ++ pmd_t *pmd; ++ ++ if (pgd_none(READ_ONCE(*pgd)) || pgd_bad(READ_ONCE(*pgd))) ++ return 0; ++ p4d = p4d_offset(pgd, va); ++ if (p4d_none(READ_ONCE(*p4d)) || p4d_bad(READ_ONCE(*p4d))) ++ return 0; ++ pud = pud_offset(p4d, va); ++ if (pud_leaf(READ_ONCE(*pud))) ++ return pud_val(READ_ONCE(*pud)); ++ if (pud_none(READ_ONCE(*pud)) || pud_bad(READ_ONCE(*pud))) ++ return 0; ++ pmd = pmd_offset(pud, va); ++ if (pmd_leaf(READ_ONCE(*pmd))) ++ return pmd_val(READ_ONCE(*pmd)); ++ if (pmd_none(READ_ONCE(*pmd)) || pmd_bad(READ_ONCE(*pmd))) ++ return 0; ++ return pte_val(READ_ONCE(*pte_offset_kernel(pmd, va))); ++} ++ ++static void g5p_table_attrs(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *p) ++{ ++ u64 desc, mair = read_sysreg(mair_el1); ++ u32 index, attr; ++ ++ if (!p->cpu) ++ return; ++ desc = g5p_table_cpu_descriptor((unsigned long)p->cpu); ++ index = (desc & PTE_ATTRINDX_MASK) >> 2; ++ attr = (mair >> (index * 8)) & 0xff; ++ g5p_mmu_record( ++ g, ++ "MMU allocation %s DMA=%pad coherent=%u cpu_desc=%016llx AttrIndx=%u " ++ "MAIR=%016llx attr=%02x table_memtype=%s\n", ++ stage, &p->dma, dev_is_dma_coherent(g->dev), desc, index, mair, ++ attr, ++ !desc ? "unknown" : ++ attr == 0xff ? "Normal-WB" : ++ attr == 0x44 ? "Normal-NC" : ++ "other"); ++} +diff --git a/drivers/gpu/apple-g5p/mmu-tables.inc b/drivers/gpu/apple-g5p/mmu-tables.inc +new file mode 100644 +index 000000000..94ae6ed8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/mmu-tables.inc +@@ -0,0 +1,57 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Fixed, pre-release aliases only. No firmware patch or live table surgery. */ ++#define G5P_TABLE_ALIAS_VA (G5P_ROOT_VA + 0x100000ULL) ++#define G5P_TABLE_COOKIE 0x3ffcU ++ ++static void g5p_table_clean_page(struct g5p_diag *g, struct g5p_dma_page *p) ++{ ++ unsigned long start; ++ ++ if (!p->cpu) ++ return; ++ start = (unsigned long)p->cpu; ++ /* Explicit architecture PoC operation, even if DT calls this coherent. ++ * dma_sync_single_for_device is not an API for coherent allocations. ++ */ ++ dcache_clean_poc(start, start + G5P_GPU_PAGE); ++} ++ ++static void g5p_tables_clean(struct g5p_diag *g, const char *stage, ++ struct g5p_dma_page *user, u32 count) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ g5p_mmu_record( ++ g, ++ "MMU tables %s clean_poc=%u dma_coherent=%u dma_pgprot=%016llx; coherent DMA " ++ "API, extra clean is experimental\n", ++ stage, 1, dev_is_dma_coherent(g->dev), ++ (u64)pgprot_val(dma_pgprot(g->dev, PAGE_KERNEL, 0))); ++ for (u32 i = 0; i < count; i++) ++ g5p_table_attrs(g, stage, &user[i]); ++ g5p_table_attrs(g, stage, &b->catalogue); ++ g5p_table_attrs(g, stage, &b->directory); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_attrs(g, stage, &b->leaf[i]); ++ ; ++ ; ++ for (u32 i = 0; i < count; i++) ++ g5p_table_clean_page(g, &user[i]); ++ for (u32 i = 0; i < b->leaves; i++) ++ g5p_table_clean_page(g, &b->leaf[i]); ++ g5p_table_clean_page(g, &b->sink_leaf); ++ g5p_table_clean_page(g, &b->sink_directory); ++ g5p_table_clean_page(g, &b->directory); ++ g5p_table_clean_page(g, &b->catalogue); ++ dsb(sy); ++ dma_wmb(); ++ g5p_mmu_record( ++ g, ++ "MMU tables %s PoC complete user_pages=%u boot_leaves=%u; dsb sy before publication\n", ++ stage, count, b->leaves); ++} ++ ++/* Deliberately separate from g5p_meta_read(): the generic slave guard stays ++ * latched after reset. This opt-in reader can address only the preinstalled ++ * table aliases and one immutable root control word, never private META RAM. ++ */ +diff --git a/drivers/gpu/apple-g5p/pack.h b/drivers/gpu/apple-g5p/pack.h +new file mode 100644 +index 000000000..942b44ba3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/pack.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef APPLE_G5P_PACK_H ++#define APPLE_G5P_PACK_H ++ ++#include "bits-compat.h" ++#include "ldr.h" ++#ifndef G5P_LDR_USER_TEST ++#include ++#include ++#endif ++ ++#define G5P_IMAGE_SIZE 0x30000 ++#define G5P_CONFIG_WRITES 27 ++ ++struct g5p_ldr_image { ++ u8 *data; ++ u32 config[G5P_CONFIG_WRITES][2]; ++ u32 config_count; ++ u32 t1_pc; ++ u32 t1_sp; ++}; ++ ++/* Exact C0 ranges derived from AGXMetaFirmware_layoutAndLoadLDR ...6c9f1bc. ++ * GPU virtual mappings and the boot config at +0x200 are intentionally separate. ++ */ ++static inline int g5p_image_offset(u32 addr, u32 size, u32 *offset) ++{ ++ static const struct { ++ u32 base, size, packed; ++ } segments[] = { ++ { 0x10000000, 0x18000, 0x04000 }, ++ { 0x20000000, 0x04000, 0x1c000 }, ++ { 0xc0000000, 0x04000, 0x00000 }, ++ { 0xe0001000, 0x04000, 0x20000 }, ++ { 0x80000080, 0x055dc, 0x24000 }, ++ }; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(segments); i++) { ++ if (addr >= segments[i].base && size <= segments[i].size && ++ addr - segments[i].base <= segments[i].size - size) { ++ *offset = segments[i].packed + addr - segments[i].base; ++ return 0; ++ } ++ } ++ return -EINVAL; ++} ++ ++static inline int g5p_pack_record(void *ctx, enum g5p_ldr_op op, u32 addr, ++ u32 size, const u8 *data) ++{ ++ struct g5p_ldr_image *image = ctx; ++ u32 i, offset; ++ int ret; ++ ++ switch (op) { ++ case G5P_LOAD: ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memcpy(image->data + offset, data, size); ++ break; ++ case G5P_ZERO: ++ /* Apple does not apply the local/core-memory ZeroMem records here. */ ++ if (addr >= 0x80000000 && addr < 0x88000000) ++ break; ++ ret = g5p_image_offset(addr, size, &offset); ++ if (ret) ++ return ret; ++ memset(image->data + offset, 0, size); ++ break; ++ case G5P_CONFIG: ++ for (i = 0; i < size; i += 12) { ++ if (image->config_count == G5P_CONFIG_WRITES) ++ return -EINVAL; ++ image->config[image->config_count][0] = ++ g5p_le32(data + i + 4); ++ image->config[image->config_count++][1] = ++ g5p_le32(data + i + 8); ++ } ++ break; ++ case G5P_START: ++ image->t1_sp = g5p_le32(data + 4); ++ image->t1_pc = g5p_le32(data + 8); ++ break; ++ } ++ return 0; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/perf-envelope.h b/drivers/gpu/apple-g5p/perf-envelope.h +new file mode 100644 +index 000000000..57387bcba +--- /dev/null ++++ b/drivers/gpu/apple-g5p/perf-envelope.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 clock: the inputs of the firmware's policy-state writer (0x10008218) and of its governor, ++ * so the firmware itself moves the GPC from pstate 1 to q = pin / 100. Both compute ++ * x = min(policy+0x6f0, policy+0x688, ctrl+0x102c) ++ * (META CS is unsigned lower; iOS's live policy+0x38 equals this min in every sample), store ++ * policy+8 = 0 when x % 100 == 0 (2 when x < 100), and request pstate q = x / 100 (policy+0x28). ++ * +0x6f0 and +0x688 are recomputed by the firmware from its coefficient block unless their ++ * overrides +0x6f4 / +0x68c are nonzero, and the firmware itself reloads those overrides from ++ * the control object on every pass, whatever the record held: +0x6f4 = ctrl+0x1040 (0x1000a1e4, ++ * 0x1000b388, 0x1000c568) and +0x68c = ctrl+0x1038 (0x1000a74c, 0x1000b10c, 0x1000c2fc). iOS ++ * leaves both control words 0 and arms the sampler (policy+4 = 16); this boot pins them ++ * (ctrl+0x1040 = pin, ctrl+0x1038 = 400) and leaves the sampler off (policy+4 = 0), so there is ++ * no idle parking at pstate 0. The pstate table and limits are iOS's (live j71t capture, ++ * 2026-11-01): meta+0x710/0x714 = 4/5, MHz {0, 340, 474, 550, 723}, ctrl+0x1028/0x1030 = ++ * 3348/400; the floor ctrl+0x102c is the pin. RAM only, before release. ++ */ ++#ifndef APPLE_G5P_PERF_ENVELOPE_H ++#define APPLE_G5P_PERF_ENVELOPE_H ++ ++#include "bits-compat.h" ++#define G5P_PERF_POLICY 0x4aU ++#define G5P_PERF_META 0x4dU ++#define G5P_PERF_CTL 0x5eU ++#define G5P_PERF_TOP 400U ++ ++static const u32 g5p_perf_mhz[] = { 0, 340, 474, 550, 723 }; ++ ++static inline bool g5p_perf_valid(u32 pin) ++{ ++ return pin <= G5P_PERF_TOP && !(pin % 100); ++} ++ ++static inline void g5p_perf_install(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 pin) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ u8 *meta = g5p_object(p, root, shared, G5P_PERF_META); ++ u8 *ctl = g5p_object(p, root, shared, G5P_PERF_CTL); ++ u32 i; ++ ++ g5p_put32(meta + 0x710, ARRAY_SIZE(g5p_perf_mhz) - 1); ++ g5p_put32(meta + 0x714, ARRAY_SIZE(g5p_perf_mhz)); ++ for (i = 0; i < ARRAY_SIZE(g5p_perf_mhz); i++) ++ g5p_put32(meta + 0x718 + 4 * i, g5p_perf_mhz[i]); ++ g5p_put32(ctl + 0x1028, 3348); ++ g5p_put32(ctl + 0x102c, pin); ++ g5p_put32(ctl + 0x1030, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1038, G5P_PERF_TOP); ++ g5p_put32(ctl + 0x1040, pin); ++ g5p_put32(policy + 0x688, G5P_PERF_TOP); ++ g5p_put32(policy + 0x68c, G5P_PERF_TOP); ++ g5p_put32(policy + 0x6f0, pin); ++ g5p_put32(policy + 0x6f4, pin); ++} ++ ++/* boot_gpu_oc_mhz (gpu-oc.h): state 5 = the overclock record; the firmware's table, ceilings and ++ * policy limits extended to index 5 so the pin 500 and the power-on request of state 5 are in range ++ * (the firmware derives its clock entry priv+0x2c4 = meta+0x718[state] from Fender, 0x1001c838). ++ */ ++ ++/* ++ * boot_gpu_perf_seed (perf60-clock.md "Clock-change drain RE" §3): in policy state 0 the ++ * firmware ++ * never changes pstate while powered; only its boot sampler (0x1000aa80, before ALIVE) moves the ++ * GPC, when policy+0x28 != Fender & 0xf. Seeding the policy record with the firmware's own ++ * steady-state output for the pin, +0x28 = +0x2c = q, +0x30 = 0, +8 = 0, makes that sampler ++ * request q through its setter (0x1001c9b8 -> 0x1001c88c: pol34 = q, R1+0x1000 = q | 0x10), ++ * which is ++ * iOS's park -> reboot -> sampler path. q = pin / 100 for pins 200..400, else 0 (no seed). ++ */ ++static inline u32 g5p_perf_seed_q(u32 pin) ++{ ++ return pin >= 200 && pin <= G5P_PERF_TOP && !(pin % 100) ? pin / 100 : ++ 0; ++} ++ ++/* Seed 2 adds what the sampler needs to get past its first-call init: 0x1000aa80 at 0x1000abb0 ++ * takes policy+0 == 0 as "never sampled" and only initialises (0x1000ae88: +0 = 1, its time words) ++ * without reaching the target compare at 0x1000ada4 or the setter; its sampling path divides by ++ * policy+4 (0x10022f88 at 0x1000ac28). The boot call at 0x10000b54 is its only caller, so a fresh ++ * record never moves Fender. iOS's record persists across firmware reboots: +0 = 1 and +4 = 16 in ++ * every sample of the 2026-11-01 capture (idle and load). Seed 2 writes those two as iOS has them. ++ * Seed 2 on hardware (2026-11-03): no ALIVE, Fender 0. With +0 = 1 the sampler takes its sampling ++ * path when (now - policy+0x8f8) / 24000 / policy+4 >= 1 (0x1000ac10..ac5c): +0x8f8 is the host's ++ * map-time stamp, ~3 s old at release, so it runs the load governor (0x1000afa4 -> 0x1000b050, busy ++ * accumulators policy+0x62c..0x68c, all 0 on a fresh record) and lands on pstate 0. Seed 3 ++ * (boot.inc ++ * g5p_boot_release) restamps +0x8f8 right before the release stream: < 1 period elapsed, the load ++ * update is skipped (0x1000ac64 -> 0x1000acb0 -> 0x1000ada4) and the compare against +0x28 = q ++ * reaches the setter (0x1000ae68 -> 0x1001c9b8). ++ */ ++static inline void g5p_perf_seed(const struct g5p_boot_layout *p, u8 *root, ++ u8 *shared, u32 q, bool ios_record) ++{ ++ u8 *policy = g5p_object(p, root, shared, G5P_PERF_POLICY); ++ ++ g5p_put64(policy + 0x28, (u64)q << 32 | q); ++ g5p_put32(policy + 0x30, 0); ++ g5p_put32(policy + 8, 0); ++ if (ios_record) { ++ g5p_put32(policy + 0, 1); ++ g5p_put32(policy + 4, 16); ++ } ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/power-r33.inc b/drivers/gpu/apple-g5p/power-r33.inc +new file mode 100644 +index 000000000..ce8a18f3c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r33.inc +@@ -0,0 +1,70 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* R33 (ds-r33-b knob A): the kexec GFX cycle holds ps_gfx off for ~10 us, and ++ * R0+0x418/0x41c (no reset) keeps the previous boot's value through it. At the ++ * first Fender wake, before any GPU MMIO and before any GPU IRQ is requested, ++ * hold the same PMGR gate (0x80340, gate124) off for boot_gfx_off_ms, then ++ * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset, ++ * auto). Every exit leaves the gate requested on. ++ */ ++#define G5P_R33_GFX_PS 0x80340U ++#define G5P_R33_GFX_OFF_MAX_MS 2000U ++#define G5P_R33_GFX_POLL_US 10000U ++ ++static inline bool g5p_r33_gfx_on(u32 raw) ++{ ++ return (raw & 0xf) == 0xf && ((raw >> 4) & 0xf) == 0xf; ++} ++ ++/* Only a plain, fully-on, software-controlled gate may be cycled. */ ++static inline bool g5p_r33_gfx_cyclable(u32 raw) ++{ ++ return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31) && ++ !((raw >> 28) & 1); ++} ++ ++static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) ++{ ++ u32 before, held, v = 0, after = 0; ++ u64 t0; ++ int ret, on; ++ ++ if (!ms) ++ return 0; ++ if (ms > G5P_R33_GFX_OFF_MAX_MS || !pmgr) ++ return -EINVAL; ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &before); ++ if (ret) ++ return ret; ++ if (!g5p_r33_gfx_cyclable(before)) { ++ dev_info( ++ dev, ++ "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset/auto or not on\n", ++ ms, before); ++ return -EBUSY; ++ } ++ ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0); ++ if (!ret) ++ ret = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, v, ++ !((v >> 4) & 0xf), 10, ++ G5P_R33_GFX_POLL_US); ++ t0 = ktime_get_ns(); ++ if (!ret) ++ msleep(ms); ++ held = (u32)((ktime_get_ns() - t0) / 1000); ++ if (!ret) ++ ret = regmap_read(pmgr, G5P_R33_GFX_PS, &v); ++ /* Never leave the GPU domain off, whatever happened above. */ ++ on = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0xf); ++ if (!on) ++ on = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, after, ++ g5p_r33_gfx_on(after), 10, ++ G5P_R33_GFX_POLL_US); ++ dev_info( ++ dev, ++ "POWER gfx-off ms=%u held_us=%u before=%08x end=%08x after=%08x off_ret=%d " ++ "on_ret=%d\n", ++ ms, held, before, v, after, ret, on); ++ if (!ret && (v >> 4) & 0xf) ++ ret = -EIO; ++ return ret ?: on; ++} +diff --git a/drivers/gpu/apple-g5p/power-r41.inc b/drivers/gpu/apple-g5p/power-r41.inc +new file mode 100644 +index 000000000..9e5c1e668 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/power-r41.inc +@@ -0,0 +1,135 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* J71t live ADT + ApplePMGR_restoreDeviceBridgeSettings (5a39918). ++ * Volatile fabric MMIO only. GFX selects bridge 8, then bridge 9; the two ++ * version-0 counter controls follow bridge 8's table, before bridge 9. ++ * Keep devm_ioremap/readl/writel identical to boot.inc's swirq mapping. ++ */ ++static const unsigned int gfx_bridge_replay = 0; ++ ++struct g5p_r41_write { ++ u32 bridge, offset, value; ++}; ++ ++static const struct g5p_r41_write g5p_r41_writes[] = { ++ { 8, 0x0000, 0x00000120 }, { 8, 0x0018, 0x80010005 }, ++ { 8, 0x001c, 0x00010001 }, { 8, 0x410c, 0x00000003 }, ++ { 8, 0x4100, 0x00000003 }, { 9, 0x0000, 0x00697101 }, ++ { 9, 0x0004, 0x00697101 }, { 9, 0x0008, 0x00697101 }, ++ { 9, 0x000c, 0x00697101 }, { 9, 0x0010, 0x00697101 }, ++ { 9, 0x0014, 0x00697101 }, { 9, 0x0018, 0x00697101 }, ++ { 9, 0x001c, 0x00697101 }, { 9, 0x0020, 0x00697101 }, ++ { 9, 0x0024, 0x00697101 }, { 9, 0x0028, 0x00697101 }, ++ { 9, 0x002c, 0x00697101 }, { 9, 0x0030, 0x00697101 }, ++ { 9, 0x0034, 0x00697101 }, { 9, 0x0038, 0x00697101 }, ++ { 9, 0x003c, 0x00697101 }, { 9, 0x0040, 0x00697101 }, ++ { 9, 0x0044, 0x00697101 }, { 9, 0x0048, 0x00697101 }, ++ { 9, 0x004c, 0x00697101 }, { 9, 0x0050, 0x00697101 }, ++ { 9, 0x0054, 0x00697101 }, { 9, 0x0058, 0x00697101 }, ++ { 9, 0x005c, 0x00697101 }, { 9, 0x0060, 0x00697101 }, ++ { 9, 0x0064, 0x00697101 }, { 9, 0x0068, 0x00697101 }, ++ { 9, 0x006c, 0x00697101 }, { 9, 0x0070, 0x00697101 }, ++ { 9, 0x0074, 0x00697101 }, { 9, 0x0078, 0x00697101 }, ++ { 9, 0x007c, 0x00697101 }, ++}; ++ ++static struct { ++ void __iomem *window[2]; ++ unsigned int mode; ++ int result; ++ bool attempted, ready, firmware_logged, ta_logged; ++} g5p_r41_bridge; ++ ++/* Reject anything outside the exact write allowlist, including unaligned ++ * offsets and other registers inside the two mapped ADT ranges. ++ */ ++static bool g5p_r41_valid(const struct g5p_r41_write *w) ++{ ++ if (w->offset & 3) ++ return false; ++ if (w->bridge == 8) ++ return w->offset < 0x5000 && ++ (w->offset == 0 || w->offset == 0x18 || ++ w->offset == 0x1c || w->offset == 0x410c || ++ w->offset == 0x4100); ++ return w->bridge == 9 && w->offset <= 0x7c; ++} ++ ++static void g5p_r41_bridge_snapshot(struct g5p_diag *g, const char *stage) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ u32 value = ++ readl(g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ ++ dev_info( ++ g->dev, ++ "BRIDGE r41 stage=%s mode=%u index=%u bridge=%u off=%04x value=%08x want=%08x\n", ++ stage, g5p_r41_bridge.mode, i, w->bridge, w->offset, ++ value, w->value); ++ } ++} ++ ++/* Called under g->lock after ACTIVE (and any r33 off/on), before Fender. ++ * Latch errors too: never retry a partial experiment in the same boot. ++ */ ++static int g5p_r41_bridge_prepare(struct g5p_diag *g) ++{ ++ unsigned int i, mode = gfx_bridge_replay; ++ ++ if (!mode) ++ return 0; ++ if (g5p_r41_bridge.attempted) ++ return g5p_r41_bridge.result; ++ g5p_r41_bridge.attempted = true; ++ g5p_r41_bridge.mode = mode; ++ g5p_r41_bridge.result = -EINVAL; ++ if (mode > 4) ++ return -EINVAL; ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) ++ if (!g5p_r41_valid(&g5p_r41_writes[i])) ++ return -EINVAL; ++ g5p_r41_bridge.result = -ENOMEM; ++ g5p_r41_bridge.window[0] = devm_ioremap(g->dev, 0x201f00000ULL, 0x5000); ++ if (!g5p_r41_bridge.window[0]) ++ return -ENOMEM; ++ g5p_r41_bridge.window[1] = devm_ioremap(g->dev, 0x201f80000ULL, 0x1000); ++ if (!g5p_r41_bridge.window[1]) ++ return -ENOMEM; ++ /* Read all 37 even in split modes; never read unmapped/partial state. */ ++ g5p_r41_bridge_snapshot(g, "before"); ++ for (i = 0; i < ARRAY_SIZE(g5p_r41_writes); i++) { ++ const struct g5p_r41_write *w = &g5p_r41_writes[i]; ++ ++ if (mode == 1 || (mode == 3 && w->bridge != 8) || ++ (mode == 4 && w->bridge != 9)) ++ continue; ++ writel(w->value, ++ g5p_r41_bridge.window[w->bridge - 8] + w->offset); ++ dev_info( ++ g->dev, ++ "BRIDGE r41 write mode=%u index=%u bridge=%u off=%04x value=%08x\n", ++ mode, i, w->bridge, w->offset, w->value); ++ } ++ g5p_r41_bridge_snapshot(g, "after"); ++ g5p_r41_bridge.ready = true; ++ g5p_r41_bridge.result = 0; ++ return 0; ++} ++ ++static void g5p_r41_bridge_firmware(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.firmware_logged) { ++ g5p_r41_bridge.firmware_logged = true; ++ g5p_r41_bridge_snapshot(g, "firmware-alive"); ++ } ++} ++ ++static void g5p_r41_bridge_ta(struct g5p_diag *g) ++{ ++ if (g5p_r41_bridge.ready && !g5p_r41_bridge.ta_logged) { ++ g5p_r41_bridge.ta_logged = true; ++ g5p_r41_bridge_snapshot(g, "first-TA-kick"); ++ } ++} +diff --git a/drivers/gpu/apple-g5p/r74e112.h b/drivers/gpu/apple-g5p/r74e112.h +new file mode 100644 +index 000000000..cacac73ac +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r74e112.h +@@ -0,0 +1,137 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R74E112_H ++#define G5P_R74E112_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++/* ++ * Wake112: the R74E GPC1 power proof when ALIVE preempts the on-poll. ++ * ++ * R74E (boot.inc) requests GPC=1 and polls R1+0x1000 until the state field reads 1. ++ * The poll stops as soon as ALIVE is decoded: after ALIVE only the first-ALIVE owner ++ * may read R1. On a warm, kexec-cycled boot the firmware can reach ALIVE before the ++ * poll's first read (Wake111 20261031T201022Z: off_ns=1500 on_ns=375 raw=ffffffff, ++ * the U32_MAX "not read" sentinel). Wake111 then latched a power abort although the ++ * state was never sampled. ++ * ++ * Wake112 defers that one proof to the owner instead, but only when the on-poll took ++ * no sample: the poll counts its reads, and the U32_MAX raw is never taken as "not read". ++ * The owner's existing first-ALIVE read of R1+0x1000 (g5p_r73e_irq_submit, before any ++ * register write) is the proof. State 1 sets release_ready and the unchanged cold ++ * sequence continues. Any other value (0, 0x10, 0x11, ffffffff, ...) is an absorbing ++ * power abort. An on-poll that reads state 1 never defers. An on-poll that sampled ++ * before ALIVE (0x11 in rounds 74e..79b, or any other word) and every other R74E failure ++ * (off-poll, timeout, GPC1 write refused) abort exactly as before. ++ * ++ * States (r73b->early_proof): ++ * NONE no deferral: the poll decided, or the release has not run ++ * DEFERRED ALIVE preempted the on-poll; one owner read is owed (generation recorded) ++ * TAKEN the once-owner claimed it; it settles with no exit in between ++ * PROVEN that read had state 1: release_ready ++ * FAILED any other value, another generation, a lost precondition, or an ALIVE ++ * wait that ended with the proof still owed: power abort, absorbing ++ * ++ * Pure policy: no register access, no lock, no kernel API. ++ */ ++ ++#define G5P_PX_STATE_MASK 0x1fU ++#define G5P_PX_STATE_ON 0x01U ++ ++enum g5p_px { ++ G5P_PX_NONE, ++ G5P_PX_DEFERRED, ++ G5P_PX_TAKEN, ++ G5P_PX_PROVEN, ++ G5P_PX_FAILED, ++ G5P_PX_N, ++}; ++ ++static const char *const g5p_px_name[G5P_PX_N] = { ++ "none", "deferred", "taken", "proven", "failed", ++}; ++ ++/* Why a deferred proof failed (NONE: it did not). */ ++enum g5p_px_why { ++ G5P_PXW_NONE, ++ G5P_PXW_STATE, /* the read's state field is not 1 */ ++ G5P_PXW_STALE, /* deferred in another reset generation */ ++ G5P_PXW_LOST, /* ALIVE or the release lost, stopped or failed since */ ++ G5P_PXW_RETIRED, /* a FINAL teardown forbids R1 */ ++ G5P_PXW_UNTAKEN, /* the ALIVE wait ended with the proof still owed */ ++ G5P_PXW_N, ++}; ++ ++static const char *const g5p_px_why_name[G5P_PXW_N] = { ++ "none", "state", "stale", "lost", "retired", "untaken", ++}; ++ ++/* What the R74E release path latches. */ ++enum g5p_pxp { ++ G5P_PXP_READY, /* the on-poll read state 1: release_ready (cold) */ ++ G5P_PXP_DEFER, /* ALIVE preempted the on-poll before its first read */ ++ G5P_PXP_ABORT, /* anything else: power abort (cold) */ ++}; ++ ++/* ++ * on_ran: the core request, the off-poll and the GPC1 write succeeded and the on-poll ++ * ran. ret: the last step's result. preempted: the on-poll stopped on ALIVE. ++ * samples: the on-poll's R1+0x1000 reads. Only a preemption with no read defers (no ++ * abort, no release_ready); a sampled one saw the state and aborts. ++ */ ++static inline enum g5p_pxp g5p_px_poll(bool on_ran, int ret, bool preempted, ++ u32 samples) ++{ ++ if (!on_ran) ++ return G5P_PXP_ABORT; ++ if (!ret) ++ return G5P_PXP_READY; ++ return preempted && !samples ? G5P_PXP_DEFER : G5P_PXP_ABORT; ++} ++ ++/* The first-ALIVE owner's admission. */ ++enum g5p_pxg { ++ G5P_PXG_READY, /* release_ready: the cold owner path */ ++ G5P_PXG_PROVE, /* a deferred proof is owed: the once-owner's read decides */ ++ G5P_PXG_WAIT, /* nothing proven yet, or another owner holds the proof */ ++ G5P_PXG_REFUSE, /* power abort or a failed proof: no register access */ ++}; ++ ++static inline enum g5p_pxg g5p_px_gate(bool power_abort, bool release_ready, ++ u32 px) ++{ ++ if (power_abort || px == G5P_PX_FAILED) ++ return G5P_PXG_REFUSE; ++ if (release_ready) ++ return G5P_PXG_READY; ++ if (px == G5P_PX_DEFERRED) ++ return G5P_PXG_PROVE; ++ return G5P_PXG_WAIT; ++} ++ ++/* RAM facts the once-owner samples after taking the proof, before its one read. */ ++struct g5p_px_pre { ++ u32 gen, gen_now; /* DS108 generation at the deferral and now */ ++ bool alive, released, stopped, failed; ++ bool r1_allowed; /* g5p_r8_r1_allowed() */ ++}; ++ ++static inline enum g5p_px_why g5p_px_check(const struct g5p_px_pre *p) ++{ ++ if (!p->gen || p->gen != p->gen_now) ++ return G5P_PXW_STALE; ++ if (!p->alive || !p->released || p->stopped || p->failed) ++ return G5P_PXW_LOST; ++ if (!p->r1_allowed) ++ return G5P_PXW_RETIRED; ++ return G5P_PXW_NONE; ++} ++ ++/* The proof: only the state field 1 is power-on. */ ++static inline bool g5p_px_on(u32 raw) ++{ ++ return G5P_GPC_ON(raw); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/r87b-attach.h b/drivers/gpu/apple-g5p/r87b-attach.h +new file mode 100644 +index 000000000..81600ac06 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/r87b-attach.h +@@ -0,0 +1,34 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_R87B_ATTACH_H ++#define G5P_R87B_ATTACH_H ++/* R87b DIAGNOSTIC ONLY, never production-safe: the DC3 attach policy selector. ++ * (docs/gpu/experiments/87b-build-report.md) ++ * ++ * Two campaigns (r85 stage 2, r87 stage 1) never reached GPU work: the DC3 attach pair was ++ * published, the immediate gate (R1+0x1000 & 0x1f == 1) read 0x10/0x11, the host GPC off/on ++ * repair timed out (-110) and attach stopped with -115 before the RAM-only consume wait ran, ++ * after which the machine froze. Every retained instance of that failure shows the firmware ++ * consuming the pair on its own (dc=3/3 at export, no bell rung). boot_r87b_attach selects: ++ * 0 the original attach policy, exactly r87: a closed gate is repaired by the host (R1+0x20000 ++ * clear, GPC off/on, ALIVE-owned doorbell). The matched control. ++ * 1 self-drain (the r82b design, docs/gpu/experiments/82-attach-reactivation-triage.md): a ++ * closed DC3 gate is only a phase sample of the firmware's GPC window. The host does no ++ * repair and rings no bell; it keeps the RAM-only 3 s consume wait of render-r74f.inc and ++ * continues through the firmware-MMU attach path (r74g) when the firmware consumed the pair. ++ * An active gate still rings the bell at once (the unchanged 80 % path). The TA kind is ++ * never affected. ++ * The selector is latched at map and read only through the latched copy. A non-zero value needs ++ * the r80 epoch (which implies r74h, r74g and r74f case 1): self-drain is only safe because r74g ++ * has the firmware own the first MMU pass, so no host R0 access is licensed by a rung bell. ++ * The TYPED_V3 production path latches policy 1 with boot_r74f_case = boot_r74g_case = 1 and NO ++ * r80 / r74h (boot-r73b.inc, "production path"); g5p_r87b_attach_valid() describes the r80 ++ * epoch only. ++ */ ++#include ++#include ++ ++#define G5P_R87B_ATTACH_ORIGINAL 0U ++#define G5P_R87B_ATTACH_SELF_DRAIN 1U ++#define G5P_R87B_ATTACH_MAX 1U ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/readback.h b/drivers/gpu/apple-g5p/readback.h +new file mode 100644 +index 000000000..fdd71ab30 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/readback.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_READBACK_H ++#define G5P_READBACK_H ++ ++#include "bits-compat.h" ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++enum g5p_readback_state { ++ G5P_READBACK_WRAPPER_ONLY, ++ G5P_READBACK_BASELINE, ++ G5P_READBACK_RESET_HELD, ++}; ++ ++/* Returning wrapper registers in exp03; never a META slave-port register. */ ++static const u32 g5p_fender_regs[] = { ++ 0, 8, 0x10, 0x14, 0x1000, 0x1004, 0x2010, 0x2014, 0x20000, ++}; ++ ++/* Reset halves first: the only post-arm R0 readback needed by the gate. */ ++static const u32 g5p_rgx_regs[] = { ++ 0x100, 0x104, 0, 4, 8, 0xc, 0x18, 0x20, 0x80, 0x84, 0x88, 0x8c, ++}; ++ ++static inline u32 g5p_readback_rgx_count(u32 gpc, enum g5p_readback_state state) ++{ ++ if (!G5P_GPC_ON(gpc)) ++ return 0; ++ if (state == G5P_READBACK_RESET_HELD) ++ return 2; ++ return state == G5P_READBACK_BASELINE ? ARRAY_SIZE(g5p_rgx_regs) : 0; ++} ++ ++static inline bool g5p_slave_access_allowed(bool reset_transition) ++{ ++ return !reset_transition; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/recover.inc b/drivers/gpu/apple-g5p/recover.inc +new file mode 100644 +index 000000000..f550d1010 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/recover.inc +@@ -0,0 +1,101 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * In-boot GPU recovery (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault isolation). ++ * ++ * A render job that stopped the firmware (its event stream went silent, or a fault) blocks the ++ * one TA/3D channel for every client, and the firmware has no abort. iOS ends that state with a ++ * firmware stop, a platform power cycle and a new firmware boot (kext 6c9e070, firmware-re README ++ * "Fault recovery"). This is the same, built from two proven pieces: ++ * - the reboot quiesce's freeze, IRQ mask and PMGR gfx gate cycle (teardown.inc g5p_kq_*), which ++ * on hardware cycled a wedged, failed device safely (deskfold reboot notifier: plan=-+cycle, ++ * then the next kernel came up warm-cycled); ++ * - the bootstrap's own stage functions (wake, map, arm, load, release, events, control), the ++ * same code the sysfs commands run after the next kernel's cycle on every warm hop. ++ * The old struct g5p_boot is frozen and left allocated (devm; a few hundred KiB per recovery), ++ * so a stale pointer anywhere still reads valid memory; a new one is built by the stages. id and ++ * sample are not rerun (they refuse once the boot saw a reset transition; map needs only the ++ * cached core id), nor is irq-test (refused in an r74 epoch). The firmware image is reused. ++ * ++ * Core lock held, no job in flight (the render scheduler's worker). Returns 0 with a live new ++ * epoch, or an error with the device left frozen and failed. ++ */ ++ ++/* The bootstrap's stages after the wake, as the sysfs runner orders them. */ ++static const char *const g5p_recover_stages[] = { ++ "map", ++ "arm", ++ "load", ++ "release", ++ "events", ++ "control-prepare", ++ "control-submit", ++ "control-wait", ++}; ++ ++static int __maybe_unused g5p_recover(struct g5p_diag *g, const char *why, ++ u32 *stage_out) ++{ ++ struct g5p_boot *old = g->boot; ++ struct g5p_kq_ctx kq = { .g = g }; ++ u64 t0 = ktime_get_ns(); ++ u32 stage = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ *stage_out = 0; ++ if (!old || !READ_ONCE(old->released) || g5p_r8_is_retired() || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || !g->pmgr || ++ !g->id_checked) ++ return -EPERM; ++ /* The quiesce's freeze and IRQ mask, then the line is given back for the new epoch. */ ++ g5p_kq_freeze(&kq); ++ g5p_kq_irq_off(&kq); ++ if (old->irq_ready && old->irq > 0) ++ devm_free_irq(g->dev, old->irq, g); ++ /* mmio-access.md G5: the GPU is marked unusable before its power is cut (Apple clears ++ * fender+0xc inside setPlatformPower(0), before the gate); g5p_kq_cycle() also sets mmio_off. ++ */ ++ WRITE_ONCE(g->fender_ready, false); ++ ret = g5p_kq_cycle(&kq); ++ ret = g5p_ds_hook_recover(g, ret) ?: ret; ++ if (ret) ++ goto out; ++ /* A new epoch: the stages build a new struct g5p_boot from here. */ ++ WRITE_ONCE(g->boot, NULL); ++ stage = 1; ++ ret = g5p_fender_wake(g); ++ for (u32 i = 0; !ret && i < ARRAY_SIZE(g5p_recover_stages); i++) { ++ const char *s = g5p_recover_stages[i]; ++ ++ stage = i + 2; ++ ret = strncmp(s, "control-", 8) ? g5p_boot_command(g, s) : ++ g5p_lifecycle_command(g, s); ++ if (ret == -EALREADY) ++ ret = 0; ++ } ++ if (!ret && (!g->boot || !READ_ONCE(g->boot->alive) || ++ !READ_ONCE(g->boot->control_complete))) ++ ret = -EIO; ++ /* The claim gate again: the held jobs need ALIVE proven in the new generation. */ ++ if (!ret) ++ ret = g5p_ds_hook_claim(g); ++ if (!ret) { ++ /* The render claim is kept: no diagnostic command may reset this epoch. */ ++ WRITE_ONCE(g->boot->user_compute_owned, true); ++ stage = 0; ++ } ++out: ++ if (ret && g->boot) ++ WRITE_ONCE(g->boot->failed, true); ++ g->last_result = ret; ++ *stage_out = stage; ++ dev_emerg( ++ g->dev, ++ "GPU-RECOVER why=%s ret=%d stage=%s ms=%llu gen=%u end=recover\n", ++ why, ret, ++ !stage ? (ret ? "cycle" : "done") : ++ stage == 1 ? "wake" : ++ g5p_recover_stages[stage - 2], ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC, g5p_ds_gen()); ++ return ret; ++} +diff --git a/drivers/gpu/apple-g5p/render-layout-r22.h b/drivers/gpu/apple-g5p/render-layout-r22.h +new file mode 100644 +index 000000000..8391a8320 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout-r22.h +@@ -0,0 +1,54 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_R22_H ++#define G5P_RENDER_LAYOUT_R22_H ++ ++#include "render-layout.h" ++ ++#define G5P_R22_TA 0x54000U ++#define G5P_R22_3D 0x58000U ++#define G5P_R22_PM 0x5c000U ++#define G5P_R22_W 0x400U ++#define G5P_R22_TA_SKU 0xa00U ++#define G5P_R22_3D_SKU 0xd00U ++#define G5P_R22_TQ 0x1800U ++#define G5P_R22_STAMP 0x1840U ++#define G5P_R22_DEP 0x1900U ++#define G5P_R22_BLOCKS 32U ++#define G5P_R22_PHYS_PAGES (G5P_R22_BLOCKS * 8) ++#define G5P_R22_AUX 0x101000000ULL ++#define G5P_R22_RT 0x101020000ULL ++#define G5P_R22_RT_AUX 0x101030000ULL ++ ++/* 6cb124c/13f4/14dc: PA PPNs, GPU F and META links are distinct. ++ * Caller provides complete 16KiB control/list/F allocations. Check the whole ++ * physical input before writing anything. No synthesized PM catalogue encoder. ++ */ ++ ++/* 6ca6d5c arithmetic oracle + 6cba480/6c7b64c. Native legacy mapping ++ * dialect has bit36 clear: L=H, Laux=Haux. No PA substitution or float math. ++ */ ++ ++static const u32 g5p_r22_wait_words[] = { ++ 0x201, 0x138204, 0x10, 0x8388204, 0, 0x104000a, 0x1f, ++ 0x1f, 0x8388204, 1, 0x104000a, 0x1f, 0x1f, 0xf48008, ++ 0x10000015, 4, 0, 0x14, 0x109f, 0x13000a, 0x4000, ++ 0x4000, 0x138204, 0x4000, 0x8388204, 0, 0x104000a, 8, ++ 8, 0x8388204, 1, 0x104000a, 8, 8, ++}; ++ ++/* Full normal timestamped recipes, independent stamp identities TA0/3D1. ++ * k is dependency+4 (zero on the proven fresh initializer), not MMU tag. ++ */ ++ ++/* PM bank0 save writes runtime qwords before the a0/a8/b0 backing anchors. */ ++static inline bool g5p_r22_pm_saved(const u8 *state) ++{ ++ for (u32 off = 8; off < 0xa0; off += 4) ++ if (g5p_le32(state + off)) ++ return true; ++ return false; ++} ++ ++/* Never trust a firmware pointer until it equals our retained stamp word. */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-layout.h b/drivers/gpu/apple-g5p/render-layout.h +new file mode 100644 +index 000000000..f2a0373da +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-layout.h +@@ -0,0 +1,59 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_LAYOUT_H ++#define G5P_RENDER_LAYOUT_H ++ ++#include "boot-layout.h" ++ ++/* Offline construction primitives only. No TA/3D publication path exists. ++ * 6caf9f8/6cb2378: caller must recover N/S1/S2/S3, not infer them from USC ++ * count or target dimensions. Sizes are bounded to u32 allocation extents. ++ */ ++struct g5p_pm_sizes { ++ u32 control, page_lists, backing; ++}; ++ ++static inline int g5p_pm_sizes(u32 n, u32 blocks, u64 s1, u64 s2, u64 s3, ++ struct g5p_pm_sizes *out) ++{ ++ u64 limit = 0xffffffffULL; ++ u64 a, b; ++ ++ if (!n || !blocks || !s1 || !s2 || !s3) ++ return -EINVAL; ++ if (n > (limit - 0x80) / 0xc0 || blocks > (limit - 0x44) / 0x24 || ++ s1 > limit / n || s2 > limit / n || s3 > limit) ++ return -EOVERFLOW; ++ a = s1 * n; ++ b = s2 * n; ++ if (a > limit - b || a + b > limit - s3) ++ return -EOVERFLOW; ++ out->control = 0x80 + n * 0xc0; ++ out->page_lists = 0x44 + blocks * 0x24; ++ out->backing = a + b + s3; ++ return 0; ++} ++ ++/* Only four recovered backing-pointer writes, not a complete PM record. ++ * Caller owns a fully initialized0xc0 record and validates GPU mappings. ++ */ ++static inline int g5p_pm_record_backing(u8 *record, u32 index, u32 n, u64 base, ++ u64 s1, u64 s2, u64 s3, u32 meta) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (index >= n) ++ return -EINVAL; ++ ret = g5p_pm_sizes(n, 1, s1, s2, s3, &sizes); ++ if (ret) ++ return ret; ++ if (base > ~0ULL - sizes.backing) ++ return -EOVERFLOW; ++ g5p_put64(record + 0xa8, base + index * s1); ++ g5p_put64(record + 0xa0, base + n * s1 + index * s2); ++ g5p_put64(record + 0xb0, base + n * s1 + n * s2); ++ g5p_put32(record + 0xb8, meta); ++ return 0; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-bind.h b/drivers/gpu/apple-g5p/render-user-bind.h +new file mode 100644 +index 000000000..3bbbb1e0f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-bind.h +@@ -0,0 +1,285 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Firmware slot pairs of render contexts (docs/gpu/multi-client-design.md, stage 1). ++ * ++ * Each render context owns a TA and a 3D firmware slot in 7..255, both pointing at its own ++ * page catalogue. The pair is bound once by two DC3 records {3, slot, 0, 0, 0}, 3D first, ++ * then TA, with the catalogue PA in 0x86[slot] and the host tag in 0x84[slot]. The firmware's ++ * DC3 handler installs CBASE and copies the host tag to 0x85[slot]. ++ * ++ * These helpers only read and write those RAM words, so they cannot reach a register and are ++ * safe after a job has retired. Pure functions, host-tested ++ * (scripts/gpu/multiclient1-host/test_render_bind.c). The caller provides g5p_le32(), ++ * g5p_put32() and g5p_put64(). ++ */ ++#ifndef G5P_RENDER_USER_BIND_H ++#define G5P_RENDER_USER_BIND_H ++ ++#define G5P_RU_SLOT_MIN 7U ++#define G5P_RU_SLOT_MAX 255U ++/* RAM-only wait for the firmware to consume a published DC3 batch. */ ++#define G5P_RU_BIND_MS 3000U ++/* Host tag of a pre-bound spare pair (render_user_spare_pairs): never zero. */ ++#define G5P_RU_SPARE_TAG 0x00400000U ++/* Host tag of a parked pair (render_user_close_park): never zero, never a spare's. */ ++#define G5P_RU_PARK_TAG 0x00500000U ++#define G5P_RU_MAX_SPARES 32U ++ ++/* The words one pair's binding touches. Pointers, because the DC records may straddle the ++ * shared pages. Index [e]: 0 TA slot, 1 3D slot. ++ */ ++struct g5p_ru_bind_mem { ++ u8 *rec[2][5]; /* the records at start and start + 1 */ ++ u8 *pa[2]; /* 0x86[slot], 64 bit */ ++ u8 *host[2]; /* 0x84[slot] */ ++ u8 *cached[2]; /* 0x85[slot] */ ++}; ++ ++/* One RAM sample of those words and of the DC ring control. */ ++struct g5p_ru_bind_sample { ++ u32 cons, prod; ++ u32 rec[2][5]; ++ u64 pa[2]; ++ u32 host[2], cached[2]; ++}; ++ ++/* g5p_ru_bind_busy() */ ++#define G5P_RU_BIND_BUSY_RING 0x1U /* the DC ring is not idle */ ++#define G5P_RU_BIND_BUSY_SLOT 0x2U /* a slot has a PA or a host tag */ ++ ++/* g5p_ru_bind_check(), the r74g criteria */ ++#define G5P_RU_BIND_BAD_RING 0x1U ++#define G5P_RU_BIND_BAD_RECORD 0x2U ++#define G5P_RU_BIND_BAD_PA 0x4U ++#define G5P_RU_BIND_BAD_HOST 0x8U ++#define G5P_RU_BIND_BAD_CACHED 0x10U ++ ++enum g5p_ru_bind_wait { ++ G5P_RU_BIND_PENDING, ++ G5P_RU_BIND_CONSUMED, ++ G5P_RU_BIND_PROTO, ++ G5P_RU_BIND_TIMEOUT, ++}; ++ ++static inline bool g5p_ru_slot_used(const unsigned long *map, u32 slot) ++{ ++ return (map[slot / BITS_PER_LONG] >> (slot % BITS_PER_LONG)) & 1; ++} ++ ++/* The slot pair of a new render context: the two lowest free slots in 7..255. The lower one ++ * is the 3D slot, or the TA slot with ta_first (render_user_slot_variant 1). False when fewer ++ * than two are free. slot[0] is TA, slot[1] is 3D. ++ */ ++static inline bool g5p_ru_slot_pair(const unsigned long *map, bool ta_first, ++ u32 *slot) ++{ ++ u32 pick[2], n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < 2; s++) ++ if (!g5p_ru_slot_used(map, s)) ++ pick[n++] = s; ++ if (n < 2) ++ return false; ++ slot[0] = pick[ta_first ? 0 : 1]; ++ slot[1] = pick[ta_first ? 1 : 0]; ++ return true; ++} ++ ++/* Word j of record i of a pair: {3, slot, 0, 0, 0}, the 3D slot's record first. */ ++static inline u32 g5p_ru_bind_record(const u32 *slot, u32 i, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot[1 - i] : 0; ++} ++ ++static inline void g5p_ru_bind_read(const struct g5p_ru_bind_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_bind_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ s->rec[i][j] = g5p_le32(m->rec[i][j]); ++ for (u32 e = 0; e < 2; e++) { ++ s->pa[e] = (u64)g5p_le32(m->pa[e] + 4) << 32 | ++ g5p_le32(m->pa[e]); ++ s->host[e] = g5p_le32(m->host[e]); ++ s->cached[e] = g5p_le32(m->cached[e]); ++ } ++} ++ ++/* Before a publish: the ring must be idle and both slots unused (zero PA and host tag). */ ++static inline u32 g5p_ru_bind_busy(const struct g5p_ru_bind_sample *s) ++{ ++ u32 busy = 0; ++ ++ if (s->prod > 255 || s->cons != s->prod) ++ busy |= G5P_RU_BIND_BUSY_RING; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] || s->host[e]) ++ busy |= G5P_RU_BIND_BUSY_SLOT; ++ return busy; ++} ++ ++/* ++ * A parked pair (render_user_close_park, release8 quiet2): a closed context's slots, bound to the ++ * kernel's park catalogue (it maps only the firmware window) with the park tag, so that the PM's ++ * late write-back of that context's free list resolves instead of faulting on an empty PC level. ++ * Both slots on `park`, both host tags the park tag. park 0: nothing is parked. ++ */ ++static inline bool g5p_ru_bind_parked(const struct g5p_ru_bind_sample *s, ++ u64 park) ++{ ++ if (!park) ++ return false; ++ for (u32 e = 0; e < 2; e++) ++ if (s->pa[e] != park || s->host[e] != G5P_RU_PARK_TAG) ++ return false; ++ return true; ++} ++ ++/* PA and tag of both slots, then the two records. The caller cleans the catalogue first and ++ * stores the producer afterwards, each behind a barrier. ++ */ ++static inline void g5p_ru_bind_publish(const struct g5p_ru_bind_mem *m, ++ const u32 *slot, u64 root, u32 tag) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ u32 e = 1 - i; ++ ++ g5p_put64(m->pa[e], root); ++ g5p_put32(m->host[e], tag); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[i][j], g5p_ru_bind_record(slot, i, j)); ++ } ++} ++ ++/* After the ring reached `target`: the records intact, both slots on `root`, and the firmware ++ * tag equal to the host tag (the DC3 handler ran for both). ++ */ ++static inline u32 g5p_ru_bind_check(const struct g5p_ru_bind_sample *s, ++ const u32 *slot, u64 root, u32 tag, ++ u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[i][j] != g5p_ru_bind_record(slot, i, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ for (u32 e = 0; e < 2; e++) { ++ if (s->pa[e] != root) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host[e] != tag) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached[e] != tag) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ } ++ return bad; ++} ++ ++/* One sample of a published batch. The caller takes `expired` before reading the ring, so ++ * the sample taken at the deadline still counts. ++ */ ++static inline int g5p_ru_bind_step(u32 cons, u32 prod, u32 target, bool expired) ++{ ++ if (prod != target || cons > 255) ++ return G5P_RU_BIND_PROTO; ++ if (cons == target) ++ return G5P_RU_BIND_CONSUMED; ++ return expired ? G5P_RU_BIND_TIMEOUT : G5P_RU_BIND_PENDING; ++} ++ ++/* ++ * Unbind (iOS AGXGart_detachSlotsAndDestroyAddressSpace, firmware-re README round 15 §1): PA ++ * and host tag of a slot go to zero and one DC3 record {3, slot, 0, 0, 0} follows. The DC3 ++ * handler then installs CBASE 0, copies the zero tag to 0x85[slot] and invalidates the slot's ++ * PC level. The slots of one batch take consecutive ring entries, one record each, behind a ++ * single producer store. The caller must have retired every use of the slots first. ++ */ ++#define G5P_RU_USER_SLOTS (G5P_RU_SLOT_MAX - G5P_RU_SLOT_MIN + 1) ++ ++struct g5p_ru_slot_mem { ++ u8 *rec[5]; /* the slot's record in the batch */ ++ u8 *pa, *host, ++ *cached; /* 0x86[slot] (64 bit), 0x84[slot], 0x85[slot] */ ++}; ++ ++struct g5p_ru_slot_sample { ++ u32 cons, prod; ++ u32 rec[5]; ++ u64 pa; ++ u32 host, cached; ++}; ++ ++/* The user slots the firmware may hold: those with a PA or a host tag. At most `max`. */ ++static inline u32 g5p_ru_held_slots(const u8 *pa, const u8 *host, u32 *slots, ++ u32 max) ++{ ++ u32 n = 0; ++ ++ for (u32 s = G5P_RU_SLOT_MIN; s <= G5P_RU_SLOT_MAX && n < max; s++) ++ if (g5p_le32(pa + s * 8) || g5p_le32(pa + s * 8 + 4) || ++ g5p_le32(host + s * 4)) ++ slots[n++] = s; ++ return n; ++} ++ ++/* A batch of n records fits the 256-entry ring behind an idle consumer. */ ++static inline bool g5p_ru_unbind_fits(u32 n) ++{ ++ return n && n <= G5P_RU_USER_SLOTS; ++} ++ ++static inline u32 g5p_ru_unbind_record(u32 slot, u32 j) ++{ ++ return j == 0 ? 3 : j == 1 ? slot : 0; ++} ++ ++static inline void g5p_ru_slot_read(const struct g5p_ru_slot_mem *m, u32 cons, ++ u32 prod, struct g5p_ru_slot_sample *s) ++{ ++ s->cons = cons; ++ s->prod = prod; ++ for (u32 j = 0; j < 5; j++) ++ s->rec[j] = g5p_le32(m->rec[j]); ++ s->pa = (u64)g5p_le32(m->pa + 4) << 32 | g5p_le32(m->pa); ++ s->host = g5p_le32(m->host); ++ s->cached = g5p_le32(m->cached); ++} ++ ++/* PA and host tag to zero, then the record. The caller stores the producer behind a barrier. */ ++static inline void g5p_ru_unbind_publish(const struct g5p_ru_slot_mem *m, ++ u32 slot) ++{ ++ g5p_put64(m->pa, 0); ++ g5p_put32(m->host, 0); ++ for (u32 j = 0; j < 5; j++) ++ g5p_put32(m->rec[j], g5p_ru_unbind_record(slot, j)); ++} ++ ++/* After the ring reached `target`: the record intact, PA and host tag still zero, and the ++ * firmware's tag zero too, which only its DC3 handler writes (the slot was bound with a ++ * non-zero tag). The G5P_RU_BIND_BAD_* bits. ++ */ ++static inline u32 g5p_ru_unbind_check(const struct g5p_ru_slot_sample *s, ++ u32 slot, u32 target) ++{ ++ u32 bad = 0; ++ ++ if (s->cons != target || s->prod != target) ++ bad |= G5P_RU_BIND_BAD_RING; ++ for (u32 j = 0; j < 5; j++) ++ if (s->rec[j] != g5p_ru_unbind_record(slot, j)) ++ bad |= G5P_RU_BIND_BAD_RECORD; ++ if (s->pa) ++ bad |= G5P_RU_BIND_BAD_PA; ++ if (s->host) ++ bad |= G5P_RU_BIND_BAD_HOST; ++ if (s->cached) ++ bad |= G5P_RU_BIND_BAD_CACHED; ++ return bad; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-close.h b/drivers/gpu/apple-g5p/render-user-close.h +new file mode 100644 +index 000000000..2aa461bce +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-close.h +@@ -0,0 +1,91 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Release of a closed render context (docs/gpu/multi-client-design.md, stage 2). ++ * ++ * A TYPED_V3 context whose handle is gone (DESTROY_CONTEXT or file close) gets one close item. ++ * It runs after the context's last job, in the job order, and decides from plain values what ++ * the firmware may still hold: ++ * ++ * RELEASE the firmware never held the pair: drop the references, free everything; ++ * UNBIND the pair is bound and every use retired: RAM-only DC3 unbind ++ * (g5p_rv3_unbind_slots()), then free on success, retain on failure; ++ * RETAIN the firmware may still use the pair or its memory: keep the context, its ++ * slots and its memory until reboot (the kexec quiesce detaches them then). ++ * ++ * The rule follows iOS AGXGart_detachSlotsAndDestroyAddressSpace (firmware-re README round 15 ++ * §1, "Linux sequence" step 8): stop new jobs, wait for every engine use, publish PA and tag ++ * zero, consume the DC3, and only then recycle the slots and free the translations. A slot ++ * returns to the allocator only after its unbind was consumed and verified, so an old unbind ++ * can never clear a new owner's PA, and the lowest free pair may be reused at once (iOS pops ++ * a freed slot off its free stack first). ++ * ++ * Pure function, host-tested (scripts/gpu/multiclient2-host/test_render_close.c). ++ */ ++#ifndef G5P_RENDER_USER_CLOSE_H ++#define G5P_RENDER_USER_CLOSE_H ++ ++enum g5p_ru_close_plan { ++ G5P_RU_CLOSE_RELEASE, ++ G5P_RU_CLOSE_UNBIND, ++ G5P_RU_CLOSE_RETAIN, ++}; ++ ++/* Why a bound or published pair is retained (bits). */ ++#define G5P_RU_CLOSE_UNVERIFIED \ ++ 0x01U /* a bind was published but never checked out */ ++#define G5P_RU_CLOSE_BAD \ ++ 0x02U /* a job of this context failed on the hardware */ ++#define G5P_RU_CLOSE_TERMINAL 0x04U /* the epoch noted a timeout or fault */ ++#define G5P_RU_CLOSE_FAILED 0x08U /* the device failed */ ++#define G5P_RU_CLOSE_STOPPED 0x10U /* the device stopped (teardown, quiesce) */ ++#define G5P_RU_CLOSE_SHUTDOWN \ ++ 0x20U /* admission closed for good (reboot quiesce) */ ++#define G5P_RU_CLOSE_QUEUED 0x40U /* a job of this context is still queued */ ++#define G5P_RU_CLOSE_PENDING 0x80U /* an earlier DC batch was never consumed */ ++#define G5P_RU_CLOSE_LOCK 0x100U /* the command lock could not be taken */ ++ ++struct g5p_ru_close_in { ++ bool retained; /* the firmware may hold the pair: a DC3 bind was published */ ++ bool bound; /* that bind was consumed and checked */ ++ bool bad, term_failed, dev_failed, stopped, shutdown, dc_pending; ++ u32 queued; ++}; ++ ++static inline const char *g5p_ru_close_name(int plan) ++{ ++ return plan == G5P_RU_CLOSE_RELEASE ? "release" : ++ plan == G5P_RU_CLOSE_UNBIND ? "unbind" : ++ "retain"; ++} ++ ++/* The plan, and in *why the G5P_RU_CLOSE_* bits that made it RETAIN (0 otherwise). */ ++static inline int g5p_ru_close_plan(const struct g5p_ru_close_in *in, u32 *why) ++{ ++ u32 w = 0; ++ ++ *why = 0; ++ if (!in->retained) ++ return in->queued ? (*why = G5P_RU_CLOSE_QUEUED, ++ G5P_RU_CLOSE_RETAIN) : ++ G5P_RU_CLOSE_RELEASE; ++ if (!in->bound) ++ w |= G5P_RU_CLOSE_UNVERIFIED; ++ if (in->bad) ++ w |= G5P_RU_CLOSE_BAD; ++ if (in->term_failed) ++ w |= G5P_RU_CLOSE_TERMINAL; ++ if (in->dev_failed) ++ w |= G5P_RU_CLOSE_FAILED; ++ if (in->stopped) ++ w |= G5P_RU_CLOSE_STOPPED; ++ if (in->shutdown) ++ w |= G5P_RU_CLOSE_SHUTDOWN; ++ if (in->queued) ++ w |= G5P_RU_CLOSE_QUEUED; ++ if (in->dc_pending) ++ w |= G5P_RU_CLOSE_PENDING; ++ *why = w; ++ return w ? G5P_RU_CLOSE_RETAIN : G5P_RU_CLOSE_UNBIND; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +new file mode 100644 +index 000000000..ccb7d2c8f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The phase wait's pause between two RAM samples (perf60-fillrate.md §11.3 cut 1): woken by the ++ * firmware event, not a fixed 1-2 ms sleep. Host test: scripts/gpu/perf60-host/test_evwait.c. ++ * ++ * Protocol (g5p_rv3_wait): drain -> seen = event_serial -> sample RAM -> decide -> pause. ++ * The IRQ thread's drain decodes each firmware event, bumps event_serial under status_lock and ++ * wakes the queue. The pause sleeps until event_serial != seen or the safety timeout, so an ++ * event decoded at any point after `seen` was read (before, during or after the sample) ends the ++ * pause at once: no lost wakeup. The timeout is the safety net for a RAM state that changes ++ * without an event; the predicate stays level-triggered on every sample, as before. ++ * Once the phase's own completion event is latched (s->event) but the RAM rules still wait ++ * (idle words, retire stamp, PM receipt land a little after the event), the pause is a short ++ * G5P_EW_POST_US poll: there is no further event to wait for. ++ */ ++#ifndef G5P_RENDER_USER_EVWAIT_H ++#define G5P_RENDER_USER_EVWAIT_H ++ ++#define G5P_EW_POST_US \ ++ 100U /* after the phase's own event: completion is imminent */ ++#define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ ++ ++enum g5p_ew_act { ++ G5P_EW_LEGACY, /* render_user_event_wake 0: usleep_range(1000, 2000), release8 */ ++ G5P_EW_SHORT, /* the phase's event seen: sleep G5P_EW_POST_US */ ++ G5P_EW_EVENT, /* sleep until any event since `seen`, at most *wait_us */ ++}; ++ ++static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, ++ u32 *wait_us) ++{ ++ if (!knob_us) { ++ *wait_us = 0; ++ return G5P_EW_LEGACY; ++ } ++ if (phase_event) { ++ *wait_us = G5P_EW_POST_US; ++ return G5P_EW_SHORT; ++ } ++ *wait_us = knob_us < G5P_EW_POST_US ? G5P_EW_POST_US : ++ knob_us > G5P_EW_MAX_US ? G5P_EW_MAX_US : ++ knob_us; ++ return G5P_EW_EVENT; ++} ++ ++/* The wait condition: an event was decoded since `seen` was read. */ ++static inline bool g5p_ew_woken(u64 seen, u64 serial) ++{ ++ return serial != seen; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-fault.h b/drivers/gpu/apple-g5p/render-user-fault.h +new file mode 100644 +index 000000000..2af9f7513 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-fault.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * A client job that faults (docs/gpu/experiments/faultiso.md). Pure decisions, host-tested ++ * (scripts/gpu/faultiso-host/test_fault_iso.c). ++ * ++ * On this firmware a GPU page fault is not survivable in place. The firmware acknowledges the ++ * MMU fault itself (R0+0x138), reports it as event 4 and parks (firmware-re README "Fault ++ * ownership"); iOS answers event 4 only with its restart tail (kext 6c81dac -> 6c834a0: firmware ++ * stop, context and dependency reset, shared state rebuilt, firmware boot), and has no ++ * per-context ACK, channel reset or slot unbind for it. The no-reset path iOS does have, the ++ * firmware's own job kill (event 8, then device-control command 0xa), is not a page fault. So a ++ * job that stops the firmware, by a fault event or by a silent event stream, loses its context ++ * and the GPU is recovered in the boot (render_user_recover), or the epoch ends. ++ */ ++#ifndef G5P_RENDER_USER_FAULT_H ++#define G5P_RENDER_USER_FAULT_H ++ ++/* The errno a lost job reports: a timeout's own, else the fault's (-EIO for a fault event or a ++ * silent stream, -EPROTO, or the drain's error). `result` is the phase's G5P_RU_WAIT_*. ++ */ ++static inline int g5p_ru_lost_errno(u32 result, int err) ++{ ++ return result == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++} ++ ++/* ++ * Does a job that failed after touching the hardware fail the device too? The legacy profiles ++ * always do. A TYPED_V3 job does not when it timed out (nothing is wrong with the protocol and ++ * the firmware may still finish), when its bind failed (the DC ring is not on the path of the ++ * contexts already bound), or when it was lost (the executor ruled: the channel drains, or a ++ * recovery replaces the epoch). Its context is quarantined in every case. ++ */ ++static inline bool g5p_ru_job_fails_device(bool v3, int ret, bool bind_failed, ++ bool lost) ++{ ++ return !v3 || (ret != -ETIMEDOUT && !bind_failed && !lost); ++} ++ ++/* ++ * A device failure noted while a job or a drain runs (a fault event decoded mid-phase), or while ++ * a recovery is pending, is not yet ruled on: the executor either loses that job's context and ++ * recovers, or ends the epoch. Until then the other contexts wait (submit accepted and held, ++ * VM_BIND busy, context status not lost) instead of failing. The executor sets `recovering` ++ * before it clears `executing`, so a reader that loads `executing` with acquire and then ++ * `recovering` always sees one of them through a recovery. ++ */ ++static inline bool g5p_ru_ruling(bool executing, bool recovering) ++{ ++ return executing || recovering; ++} ++ ++/* render_user_test_fault_job: the context's job about to start (it has run `jobs` before) gets ++ * the unmapped control stream. Never the epoch's first job: its attach and bell are special. ++ */ ++static inline bool g5p_ru_fault_due(u32 fault_job, u32 jobs, bool first) ++{ ++ return fault_job && jobs + 1 == fault_job && !first; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-faultreg.h b/drivers/gpu/apple-g5p/render-user-faultreg.h +new file mode 100644 +index 000000000..83cac51f7 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-faultreg.h +@@ -0,0 +1,124 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 fault address (render_user_fault_read, drm-render-v3.inc): decode of a GPU MMU fault ++ * status bank, R0+0xe150/e154 (bank 0) or R0+0xe158/e15c (bank 1), as high32:low32. ++ * ++ * The firmware's fault path (source 4 -> 1001a494) reads both banks with a 64-bit load, keeps ++ * only a class in its companion record and posts event 4 without the status, so the address ++ * exists only in these registers. iOS reads them itself in its fault handler (kext 6ca5c14, ++ * accelerator virtual+740) with exactly these fields (docs/gpu/firmware-re/README.md "Complete ++ * e150 decode"): bit 0 valid, 2:1 type (0 page invalid, 2 read-only, 3 PM/META protected), ++ * 3 read (0 write), 5:4 level (3 = catalogue), 11:6 requester (iOS table 6ff5ed8), 19:12 ++ * sideband, 27:20 MMU context (the slot), VA = (status >> 24) & 0xfffffffff0 (16-byte precision). ++ * Pure, host-tested (scripts/gpu/perf60-host/test_fault_decode.c). ++ */ ++#ifndef G5P_RENDER_USER_FAULTREG_H ++#define G5P_RENDER_USER_FAULTREG_H ++ ++#include "bits-compat.h" ++struct g5p_ru_mmu_fault { ++ bool valid, read; ++ u8 type, level, requester, sideband, context; ++ u64 va; ++}; ++ ++static inline struct g5p_ru_mmu_fault g5p_ru_mmu_fault_decode(u32 lo, u32 hi) ++{ ++ u64 s = (u64)hi << 32 | lo; ++ struct g5p_ru_mmu_fault f = { ++ .valid = s & 1, ++ .type = (s >> 1) & 3, ++ .read = (s >> 3) & 1, ++ .level = (s >> 4) & 3, ++ .requester = (s >> 6) & 0x3f, ++ .sideband = (s >> 12) & 0xff, ++ .context = (s >> 20) & 0xff, ++ .va = (s >> 24) & 0xfffffffff0ULL, ++ }; ++ ++ return f; ++} ++ ++static const char *const g5p_ru_mmu_fault_types[4] = { ++ "page-invalid", ++ "type1", ++ "read-only", ++ "pm-meta-protected", ++}; ++ ++/* Requester names, iOS kext table 6ff5ed8 (16-byte {name, class} entries; read from the j71t ++ * 16.7.10 kernelcache, 2026-11-03). 56 entries; a larger index prints its number only. ++ */ ++static const char *const g5p_ru_mmu_requesters[] = { ++ "META", ++ "TLA", ++ "HOST", ++ "VDMM", ++ "CDM", ++ "IPP", ++ "PM", ++ "TILING", ++ "MCU_TEXAS0", ++ "PDS_TEXAS0", ++ "PBE0_TEXAS0", ++ "PBE1_TEXAS0", ++ "VDMS_BLACKPEARL0", ++ "IPF_BLACKPEARL0", ++ "ISP_BLACKPEARL0", ++ "TPF_BLACKPEARL0", ++ "USCS_BLACKPEARL0", ++ "PPP_BLACKPEARL0", ++ "VCE_BLACKPEARL0", ++ "FBCDC_BLACKPEARL0", ++ "MCU_TEXAS1", ++ "PDS_TEXAS1", ++ "PBE0_TEXAS1", ++ "PBE1_TEXAS1", ++ "MCU_TEXAS2", ++ "PDS_TEXAS2", ++ "PBE0_TEXAS2", ++ "PBE1_TEXAS2", ++ "VDMS_BLACKPEARL1", ++ "IPF_BLACKPEARL1", ++ "ISP_BLACKPEARL1", ++ "TPF_BLACKPEARL1", ++ "USCS_BLACKPEARL1", ++ "PPP_BLACKPEARL1", ++ "VCE_BLACKPEARL1", ++ "FBCDC_BLACKPEARL1", ++ "MCU_TEXAS3", ++ "PDS_TEXAS3", ++ "PBE0_TEXAS3", ++ "PBE1_TEXAS3", ++ "MCU_TEXAS4", ++ "PDS_TEXAS4", ++ "PBE0_TEXAS4", ++ "PBE1_TEXAS4", ++ "VDMS_BLACKPEARL2", ++ "IPF_BLACKPEARL2", ++ "ISP_BLACKPEARL2", ++ "TPF_BLACKPEARL2", ++ "USCS_BLACKPEARL2", ++ "PPP_BLACKPEARL2", ++ "VCE_BLACKPEARL2", ++ "FBCDC_BLACKPEARL2", ++ "MCU_TEXAS5", ++ "PDS_TEXAS5", ++ "PBE0_TEXAS5", ++ "PBE1_TEXAS5", ++}; ++ ++static inline const char *g5p_ru_mmu_requester(u32 requester) ++{ ++ if (requester >= ARRAY_SIZE(g5p_ru_mmu_requesters)) ++ return "?"; ++ return g5p_ru_mmu_requesters[requester]; ++} ++ ++/* The range test of the VM edit log: does [va, va + size) hold the fault address? */ ++static inline bool g5p_ru_mmu_fault_in(u64 fault_va, u64 va, u64 size) ++{ ++ return fault_va >= va && fault_va - va < size; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-geometry.h b/drivers/gpu/apple-g5p/render-user-geometry.h +new file mode 100644 +index 000000000..2117d567b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-geometry.h +@@ -0,0 +1,144 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_GEOMETRY_H ++#define G5P_RENDER_USER_GEOMETRY_H ++ ++/* Kernel-derived render-target geometry: AGXRenderTargetMemoryG5's arithmetic ++ * leaf 6ca6d5c (firmware round16 section 3) for one sample at the pixel ++ * centre (8,8) or four samples (docs/gpu/experiments/msaa.md), in integer ++ * arithmetic only. It is Rogue's non-simple ++ * parameter format: 32x32 tiles, 4x4 macrotiles of ++ * ALIGN(DIV_ROUND_UP(tiles, 4), 4) tiles per axis, five-byte region headers ++ * per padded tile, a tail pointer cache of next_pow2(max(padded tiles))^2 ++ * eight-byte entries. The scale pairs are the leaf's IEEE singles ++ * 0x3e89309c / n, 0x3fddb3d9 / n (fdiv, round to nearest even) and n * 16. ++ * tools/gpu/test_rt_oracle.c compares every output with the unmodified leaf ++ * for every width and height in 1..G5P_RU_MAX_DIM. ++ * ++ * G5P_RU_MAX_DIM is rogue_get_render_size_max() for this tiling class; it ++ * is a memory bound (16 MiB private window, 8 MiB main ring), not a field ++ * limit: the 9-bit TE macrotile and 10-bit padded-size fields would hold ++ * twice as much. ++ */ ++#define G5P_RU_MAX_DIM 8192U ++#define G5P_RU_SCALE_A 0x3e89309cU ++#define G5P_RU_SCALE_B 0x3fddb3d9U ++ ++static inline u32 g5p_ru_fls(u32 v) ++{ ++ return v ? 32 - __builtin_clz(v) : 0; ++} ++ ++/* ucvtf of 1 <= n < 2^24: exact. */ ++static inline u32 g5p_ru_f32(u32 n) ++{ ++ u32 bits = g5p_ru_fls(n); ++ ++ return ((126 + bits) << 23) | ((n << (24 - bits)) & 0x7fffff); ++} ++ ++/* fdiv of a normal positive single by 1 <= n < 2^24, round to nearest even. ++ * The quotient q of the 24-bit significand shifted by fls(n) + 3 has 27 or ++ * 28 bits; s of them are rounding bits, the remainder is the sticky bit. ++ * Results stay normal for the two leaf constants and n <= G5P_RU_MAX_DIM. ++ */ ++static inline u32 g5p_ru_fdiv(u32 a, u32 n) ++{ ++ u32 shift = g5p_ru_fls(n) + 3; ++ u64 num = (u64)((a & 0x7fffff) | 0x800000) << shift; ++ u32 q = num / n, sticky = num % n != 0; ++ u32 s = g5p_ru_fls(q) - 24, m = q >> s, low = q & ((1U << s) - 1); ++ ++ if (low > 1U << (s - 1) || ++ (low == 1U << (s - 1) && (sticky || (m & 1)))) ++ m++; ++ if (m >> 24) { ++ m >>= 1; ++ s++; ++ } ++ return ((((a >> 23) & 0xff) - shift + s) << 23) | (m & 0x7fffff); ++} ++ ++/* Sample positions, one byte per sample: x in [3:0], y in [7:4], 1/16 px. ++ * One sample sits at the pixel centre; four use the standard Vulkan/D3D ++ * pattern, which is also Metal's (record+0x69c..0x6b8 of the iOS ++ * msaa4-resolve/-store captures: (6,2) (14,6) (2,10) (10,14)). ++ */ ++#define G5P_RU_POSITIONS_1X 0x88ULL ++#define G5P_RU_POSITIONS_4X 0xeaa26e26ULL ++ ++/* The leaf for one or four samples (6ca6d5c's 1, 2, 4, 8 table, of which ++ * Linux uses 1 and 4). Four samples are 2x2 per pixel: the region headers ++ * and the tail pointer cache are sized for twice the tiles in x and y, the ++ * ISP macrotile size (Rbc) doubles in y, Rb8 is log2(samples) (5940f40) ++ * and the TE word Rf0 is 5. Rb0 and Rf8 are the positions word, packed by ++ * the leaf as x0 | y0 << 4 | x1 << 8 | ... | y3 << 28. The scales, the TE ++ * tiling words and the tile count do not depend on the samples. ++ */ ++static inline int g5p_ru_rt_derive_samples(u32 width, u32 height, u32 samples, ++ u64 positions, struct g5p_ru_rt *rt) ++{ ++ u32 tx = (width + 31) / 32, ty = (height + 31) / 32; ++ u32 mx = (((tx + 3) >> 2) + 3) & ~3U, my = (((ty + 3) >> 2) + 3) & ~3U; ++ u32 s = samples == 4 ? 2 : 1; ++ u32 px = mx * s * 4, py = my * s * 4, tpc; ++ ++ if (!width || !height || width > G5P_RU_MAX_DIM || ++ height > G5P_RU_MAX_DIM || (samples != 1 && samples != 4) || ++ positions >> (samples * 8)) ++ return -EINVAL; ++ tpc = 1U << g5p_ru_fls((px > py ? px : py) - 1); ++ *rt = (struct g5p_ru_rt) { ++ .header = px * py * 5, ++ .region = mx * my * s * s * 5, ++ .packed = positions, ++ .state = s - 1 ? 2 : 0, ++ .padded = (mx & 0x3ff) << 16 | ((my * s) & 0x3ff), ++ .scale = { ++ g5p_ru_fdiv(G5P_RU_SCALE_A, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_A, height) << 32, ++ g5p_ru_fdiv(G5P_RU_SCALE_B, width) | ++ (u64)g5p_ru_fdiv(G5P_RU_SCALE_B, height) << 32, ++ (g5p_ru_f32(width) + (4U << 23)) | ++ (u64)(g5p_ru_f32(height) + (4U << 23)) << 32, ++ }, ++ .ta_state = { s - 1 ? 5 : 0, positions }, ++ .ta_tiling = { ++ (u64)((ty - 1) << 12 | (tx - 1)) << 32 | ++ ((height - 1) << 16 | (width - 1)), ++ (u64)(my << 18 | my << 10 | 3 * my) << 32 | ++ (mx << 18 | mx << 10 | 3 * mx), ++ }, ++ .ta_tiling_word = mx * my, ++ .tiles = tx * ty, ++ .secondary = (u64)tpc * tpc * 8, ++ .secondary_align = 0x40, ++ }; ++ return 0; ++} ++ ++/* g5p_ru_rt64 is this function's result for 64x64. */ ++ ++/* PM pool for a kernel-derived target. Linux policy, not hardware: Rogue's ++ * minimum free list (50 4KiB pages, non-XE cores) plus one 4KiB PM page per ++ * tile, in 128KiB blocks, never below the latched pool and at most one F ++ * page. 64x64 (4 tiles) needs 2, so the latched 32 stays. There is no PM ++ * growth during a job: a frame that exhausts the pool fails like any OOM. ++ */ ++static inline u32 g5p_ru_pm_blocks_for(u32 tiles, u32 latched) ++{ ++ u64 need = ((50ULL + tiles) * 0x1000 + G5P_RU_BLOCK_BYTES - 1) / ++ G5P_RU_BLOCK_BYTES; ++ ++ if (need < latched) ++ need = latched; ++ return need > G5P_RU_MAX_BLOCKS ? G5P_RU_MAX_BLOCKS : need; ++} ++ ++ ++ ++static inline int g5p_ru_rt_derive(u32 width, u32 height, struct g5p_ru_rt *rt) ++{ ++ return g5p_ru_rt_derive_samples(width, height, 1, G5P_RU_POSITIONS_1X, rt); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +new file mode 100644 +index 000000000..822709ed8 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -0,0 +1,651 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_LAYOUT_H ++#define G5P_RENDER_USER_LAYOUT_H ++ ++#include "bits-compat.h" ++/* Parameterized forms of the r22 fixed constructors for userspace render. ++ * Pure byte builders: no allocation, MMIO, locking or firmware publication. ++ * With r22's inputs every builder reproduces render-layout-r22.h exactly; ++ * tools/gpu/test_render_user.c checks that equivalence byte for byte. ++ */ ++#include "render-layout-r22.h" ++#include "render-variants-r25.h" ++#include "render-user-template.h" ++ ++/* Kernel-private window in every render VM. r22's aux/RT addresses are the ++ * first allocations, so a 64x64 target keeps the fixed path's W anchors. ++ * Root entry 0 carries the boot VA0 sink, so user maps start at 1 GiB. ++ */ ++#define G5P_RU_PRIVATE_VA G5P_R22_AUX ++#define G5P_RU_PRIVATE_SIZE 0x1000000ULL ++#define G5P_RU_USER_VA_MIN 0x40000000ULL ++/* Catalogue entry of the firmware GiB, borrowed from the boot tables. */ ++#define G5P_RU_FIRMWARE_CI ((G5P_IMAGE_VA >> 30) & 1023) ++#define G5P_RU_AUX_BYTES 0x10680U ++#define G5P_RU_AUX_PAGES 5U ++#define G5P_RU_RT_ALIGN 0x10000ULL ++#define G5P_RU_MAX_BLOCKS 128U ++#define G5P_RU_BLOCK_BYTES 0x20000U ++#define G5P_RU_MAX_HEADER 0x80000U ++#define G5P_RU_MAX_SECONDARY 0x100000ULL ++#define G5P_RU_TA_W_BYTES 0x560U ++#define G5P_RU_3D_W_BYTES 0x8c0U ++#define G5P_RU_CMD_BYTES 0x40U ++ ++/* Kernel mirror of drm_g5p_rt_geometry (firmware round16 R-offsets). */ ++struct g5p_ru_rt { ++ u32 header, region; /* R10, R20 */ ++ u64 packed; /* Rb0 */ ++ u32 state, padded; /* Rb8, Rbc */ ++ u64 scale[3]; /* Rc0, Rc8, Rd0 */ ++ u64 ta_state[2]; /* Rf0, Rf8 */ ++ u64 ta_tiling[2]; /* R100, R108 */ ++ u32 ta_tiling_word, tiles; /* R110, R118 */ ++ u64 secondary, secondary_align; /* R138, R178 */ ++}; ++ ++/* 6ca6d5c executed offline for 64,64,1,8,8 (round16 section 3). */ ++static const struct g5p_ru_rt g5p_ru_rt64 = { ++ .header = 0x500, ++ .region = 0x50, ++ .packed = 0x88, ++ .state = 0, ++ .padded = 0x40004, ++ .scale = { 0x3b89309c3b89309cULL, 0x3cddb3d93cddb3d9ULL, ++ 0x4480000044800000ULL }, ++ .ta_state = { 0, 0x88 }, ++ .ta_tiling = { 0x00001001003f003fULL, 0x0010100c0010100cULL }, ++ .ta_tiling_word = 0x10, ++ .tiles = 4, ++ .secondary = 0x800, ++ .secondary_align = 0x40, ++}; ++ ++#include "render-user-geometry.h" ++ ++struct g5p_ru_rt_sizes { ++ u64 slice, main, secondary_map; ++}; ++ ++/* 6cb3d68: slice=R10 rounded to 4KiB; main ring is the power-of-two capacity ++ * for sixteen slices. 6cb40e0 places the secondary extent at offset 0 of ++ * its own mapping. Bounds are Linux allocation policy, not hardware limits. ++ */ ++static inline int g5p_ru_rt_sizes(const struct g5p_ru_rt *rt, ++ struct g5p_ru_rt_sizes *out) ++{ ++ u64 main = 0x1000; ++ ++ if (!rt->header || rt->header > G5P_RU_MAX_HEADER || !rt->secondary || ++ rt->secondary > G5P_RU_MAX_SECONDARY || !rt->secondary_align || ++ (rt->secondary_align & (rt->secondary_align - 1)) || ++ rt->secondary_align > G5P_GPU_PAGE || !rt->tiles) ++ return -EINVAL; ++ out->slice = ((u64)rt->header + 0xfff) & ~0xfffULL; ++ while (main < out->slice * 16) ++ main <<= 1; ++ out->main = main < G5P_GPU_PAGE ? G5P_GPU_PAGE : main; ++ out->secondary_map = (rt->secondary + G5P_GPU_PAGE - 1) & ++ ~(u64)(G5P_GPU_PAGE - 1); ++ return 0; ++} ++ ++/* 6cba480/6c7b64c anchors. Native mapping dialect: L=H, Laux=Haux. ++ * Equal to g5p_r22_rt() for g5p_ru_rt64 at the r22 addresses. 3D W58 and ++ * Wa8 are the importer's literal Rb0 and R20>>2 for the verified geometry. ++ */ ++static inline void g5p_ru_rt_fill(u8 *ta, u8 *td, const struct g5p_ru_rt *rt, ++ u64 main_va, u64 secondary_va) ++{ ++ g5p_put64(ta + 0x28, main_va); ++ g5p_put64(ta + 0x30, secondary_va); ++ g5p_put64(ta + 0x38, rt->ta_state[0]); ++ g5p_put64(ta + 0x40, rt->ta_state[1]); ++ g5p_put64(ta + 0x444, secondary_va); ++ g5p_put32(ta + 0x44c, rt->secondary); ++ g5p_put64(ta + 0x3ac, rt->ta_tiling[0]); ++ g5p_put64(ta + 0x3b4, rt->ta_tiling[1]); ++ g5p_put32(ta + 0x3bc, rt->ta_tiling_word); ++ g5p_put64(td + 0x20, main_va); ++ g5p_put64(td + 0x28, rt->packed); ++ g5p_put32(td + 0x30, rt->state); ++ g5p_put32(td + 0x34, rt->padded); ++ g5p_put32(td + 0x38, rt->region); ++ g5p_put64(td + 0x3c, rt->scale[0]); ++ g5p_put64(td + 0x44, rt->scale[1]); ++ g5p_put64(td + 0x4c, rt->scale[2]); ++ g5p_put64(td + 0x58, rt->packed); ++ g5p_put32(td + 0xa8, rt->region >> 2); ++ g5p_put32(td + 0x3a0, rt->padded); ++ g5p_put64(td + 0x3d0, rt->scale[0]); ++ g5p_put64(td + 0x3d8, rt->scale[1]); ++ g5p_put32(td + 0x3e0, rt->scale[2]); ++ g5p_put32(td + 0x3e4, rt->scale[0]); ++} ++ ++/* Validated typed render inputs, already range-checked by the caller. */ ++struct g5p_ru_pass { ++ u64 vdm_va; ++ u32 vdm_size, vdm_control; ++ u64 scratch_va, scratch_size, scratch_ptr[6]; ++ u64 pool7_va, pool7_size, pool8_va, pool8_size; ++ u32 context_pds[2][6], resume_pds[2][4], ta_pds[4]; ++ u32 background[8], eot[4], eot_resume[4], restore[4]; ++}; ++ ++/* Importer chains 6cba8a0/6c7b100/6c7b34c -> 6ca9710/6ca7ae0 (offline ++ * differential map in make-render-user-template.py). Every user VA field ++ * and code-relative descriptor lands here; the rest of W is the frozen ++ * dialect-1 scalar table or a kernel RT/channel field. ++ */ ++static inline void g5p_ru_fill_ta(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_TA_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_ta_fixed); i++) ++ g5p_put32(w + g5p_ru_ta_fixed[i][0], g5p_ru_ta_fixed[i][1]); ++ g5p_put64(w + 0x48, p->scratch_ptr[4]); ++ g5p_put64(w + 0x50, p->scratch_ptr[5]); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put32(w + 0x58 + i * 4, p->context_pds[0][i]); ++ g5p_put32(w + 0x70 + i * 4, p->context_pds[1][i]); ++ } ++ g5p_put64(w + 0x88, p->vdm_control & ~0xfULL); ++ g5p_put64(w + 0x90, p->scratch_va & ~7ULL); ++ g5p_put64(w + 0x98, p->vdm_va & ~3ULL); ++ for (u32 i = 0; i < 4; i++) { ++ g5p_put32(w + 0xa0 + i * 4, p->resume_pds[0][i]); ++ g5p_put32(w + 0xb0 + i * 4, p->resume_pds[1][i]); ++ } ++ for (u32 i = 0; i < 3; i++) ++ g5p_put32(w + 0xc0 + i * 4, p->ta_pds[i]); ++ for (u32 i = 0; i < 4; i++) ++ g5p_put64(w + 0x468 + i * 8, p->scratch_ptr[i]); ++ g5p_put64(w + 0x488, p->scratch_ptr[4]); ++} ++ ++static inline void g5p_ru_fill_3d(u8 *w, const struct g5p_ru_pass *p) ++{ ++ memset(w, 0, G5P_RU_3D_W_BYTES); ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_ru_3d_fixed); i++) ++ g5p_put32(w + g5p_ru_3d_fixed[i][0], g5p_ru_3d_fixed[i][1]); ++ g5p_put64(w + 0x60, p->pool8_va & ~3ULL); ++ g5p_put64(w + 0x68, p->pool7_va & ~3ULL); ++ g5p_put64(w + 0x490, p->pool7_va); ++ g5p_put64(w + 0x4a0, p->pool8_va); ++ for (u32 i = 0; i < 8; i++) ++ g5p_put32(w + 0x5b8 + i * 4, p->background[i]); ++ g5p_put32(w + 0xb0, p->background[0]); ++ g5p_put32(w + 0xb4, p->background[1]); ++ for (u32 i = 4; i < 8; i++) ++ g5p_put32(w + 0xc0 + (i - 4) * 4, p->background[i]); ++ /* 6ca7ae0 swaps the first EOT qword's halves in its B upload. */ ++ g5p_put32(w + 0x3b4, p->eot[1]); ++ g5p_put32(w + 0x3b8, p->eot[0]); ++ g5p_put32(w + 0x3bc, p->eot[2]); ++ for (u32 i = 0; i < 3; i++) { ++ g5p_put32(w + 0x680 + i * 4, p->eot[i]); ++ g5p_put32(w + 0x694 + i * 4, p->eot_resume[i]); ++ } ++ g5p_put32(w + 0x600, p->restore[0]); ++ g5p_put32(w + 0x604, p->restore[1]); ++ g5p_put32(w + 0x610, p->restore[2]); ++ g5p_put32(w + 0x614, p->restore[3]); ++} ++ ++/* TYPED_V2 additions, already validated by g5p_ru_pass2_check(). */ ++struct g5p_ru_pass2 { ++ u64 scissor_va, dbias_va, background_texture, zls[6]; ++ u64 occlusion_query_va, occlusion_query_size; ++ u32 scissor_count, dbias_count, bgobj_depth, bgobj_vals; ++ u32 pixel_width, zls_mask; ++}; ++ ++/* Dialect-2 words over g5p_ru_fill_3d() (TYPED_V2 has zero pools). Each ++ * register image goes to its B-upload word and to the normal builder's ++ * copy (6c7a06c); make-render-user-dialect2.py proves every position by ++ * perturbing record 2d8/2e0 (pools), 2f0/310/338/360/388/308 (ZLS), 508 ++ * (background texture) and 4d0 (clears) through the r22 importer. ++ * Pixel output: 6ca7ae0 for D2f8 = width, one sample (D6e0 = 0), D630 = ++ * D6b2 = 0, six cores. Widths 2/4/8 keep two or more partitions, so W3c0 ++ * and W3c4 keep their dialect-1 values; width 2 is dialect 1 exactly. ++ */ ++static inline void g5p_ru_fill_3d_width(u8 *w, u32 width) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by D2f8 - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u32 count = 0x4000 / (width << 10); ++ u32 code = width_code[width - 2]; ++ ++ g5p_put32(w + 0x3c8, count & 0x1f); ++ g5p_put32(w + 0x3cc, code >> 4 | 0xf8); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(w + 0x108 + i * 8, code | (count - 1) << 7 | 0x43); ++ g5p_put32(w + 0x6b0, width); ++} ++ ++static inline void g5p_ru_fill_3d_v2(u8 *w, const struct g5p_ru_pass2 *e) ++{ ++ static const u16 zls_w[6][2] = { ++ { 0x80, 0x618 }, { 0x88, 0x628 }, { 0x90, 0x630 }, ++ { 0x98, 0x648 }, { 0xa0, 0x650 }, { 0x138, 0x6d0 }, ++ }; ++ ++ g5p_put64(w + 0x60, e->scissor_va & ~3ULL); ++ g5p_put64(w + 0x4a0, e->scissor_va); ++ g5p_put64(w + 0x68, e->dbias_va & ~3ULL); ++ g5p_put64(w + 0x490, e->dbias_va); ++ /* ++ * D318 is copied to both partition records. The normal 3D upload masks ++ * the low nibble before programming ISP_OCLQRY_BASE. ++ */ ++ g5p_put64(w + 0x70, e->occlusion_query_va); ++ g5p_put64(w + 0x78, e->occlusion_query_va); ++ g5p_put64(w + 0xb8, e->background_texture); ++ g5p_put64(w + 0x5f0, e->background_texture); ++ g5p_put32(w + 0x3a4, e->bgobj_depth); ++ g5p_put32(w + 0x6a8, e->bgobj_depth); ++ g5p_put32(w + 0x3a8, e->bgobj_vals); ++ g5p_put32(w + 0x6ac, e->bgobj_vals); ++ for (u32 i = 0; i < 6; i++) { ++ g5p_put64(w + zls_w[i][0], e->zls[i]); ++ g5p_put64(w + zls_w[i][1], e->zls[i]); ++ } ++ g5p_ru_fill_3d_width(w, e->pixel_width); ++} ++ ++/* TYPED_V3 additions, already validated by g5p_ru_pass3_check(). */ ++struct g5p_ru_pass3 { ++ u64 ta_program_va; ++ u32 ta_program_size, vdm_end; ++ u32 pixel_width, pad; /* V1 dialect output width, 0 = 2 */ ++}; ++ ++/* Dialect-3 word over g5p_ru_fill_ta(): W+0x88 (-> R0+0x470) is the exact ++ * byte offset of the TA PDS program from pds_base, a plain 64-bit store with ++ * no mask. g5p_ru_fill_ta() leaves it zero for a V3 pass (vdm_control is ++ * zero), so V1/V2 bytes are untouched; the stream length never reaches W. ++ */ ++static inline void g5p_ru_fill_ta_v3(u8 *w, u64 program_off) ++{ ++ g5p_put64(w + 0x88, program_off); ++} ++ ++/* 6cb2378 selected state S at control+80: explicit zeros plus the four ++ * recovered backing pointers. N=1, PM ID0/record0. aux_va covers 10680. ++ */ ++static inline int g5p_ru_pm_state(u8 *control, u64 aux_va, u64 list_va) ++{ ++ memset(control + 0x80, 0, 0xc0); ++ return g5p_pm_record_backing(control + 0x80, 0, 1, aux_va, 0x80, ++ 0x10000, 0x600, ++ g5p_meta_address(list_va + 0x40)); ++} ++ ++/* 6cb124c/13f4/14dc pool for `blocks` resident = maximum blocks. Rewrites ++ * control+0..7f, the whole list and F; S at control+80 is not touched. ++ * physical[] holds blocks*8 16KiB PAs. Validate everything before writing. ++ */ ++static inline int g5p_ru_pm_check(u64 list_va, u64 free_va, const u64 *physical, ++ u32 blocks) ++{ ++ struct g5p_pm_sizes sizes; ++ int ret; ++ ++ if (!blocks || blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ ret = g5p_pm_sizes(1, blocks, 0x80, 0x10000, 0x600, &sizes); ++ if (ret) ++ return ret; ++ if ((list_va | free_va) & (G5P_GPU_PAGE - 1) || ++ list_va >= (1ULL << 40) || free_va >= (1ULL << 40) || ++ 0x80 + blocks * 0x24 > G5P_GPU_PAGE || ++ blocks * 32 * 4 > G5P_GPU_PAGE) ++ return -EINVAL; ++ for (u32 i = 0; i < blocks * 8; i++) ++ if ((physical[i] & (G5P_GPU_PAGE - 1)) || ++ physical[i] >= (1ULL << 40)) ++ return -ERANGE; ++ return 0; ++} ++ ++static inline int g5p_ru_pm_pool(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks) ++{ ++ u32 previous = 0; ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ memset(control, 0, 0x80); ++ memset(list, 0, G5P_GPU_PAGE); ++ memset(free, 0, G5P_GPU_PAGE); ++ for (u32 j = 0; j < blocks; j++) { ++ u32 at = 0x80 + j * 0x24; ++ ++ g5p_put32(list + at, previous); ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[j * 8 + k] >> 12; ++ ++ g5p_put32(list + at + 4 + k * 4, ppn); ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * 4, ++ ppn + p); ++ } ++ previous = g5p_meta_address(list_va + at); ++ } ++ g5p_put32(list + 0x60, 1); ++ g5p_put32(control + 0x10, previous); ++ g5p_put64(control + 0x14, free_va); ++ g5p_put32(control + 0x1c, blocks * 32 * 4); ++ g5p_put32(control + 0x20, blocks * 32); ++ g5p_put32(control + 0x24, blocks * 32 - 1); ++ g5p_put32(control + 0x34, G5P_GPU_PAGE); ++ g5p_put32(control + 0x40, g5p_meta_address(list_va)); ++ g5p_put32(control + 0x54, blocks * 32); ++ g5p_put32(control + 0x58, blocks * 32); ++ return 0; ++} ++ ++/* Complete fresh PM object: g5p_r22_pm() when blocks=32, aux=G5P_R22_AUX. */ ++static inline int g5p_ru_pm(u8 *control, u8 *list, u8 *free, u64 list_va, ++ u64 free_va, const u64 *physical, u32 blocks, ++ u64 aux_va) ++{ ++ int ret = g5p_ru_pm_check(list_va, free_va, physical, blocks); ++ ++ if (ret) ++ return ret; ++ if ((aux_va & (G5P_GPU_PAGE - 1)) || ++ aux_va > (1ULL << 40) - G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ return -EINVAL; ++ memset(control, 0, G5P_GPU_PAGE); ++ ret = g5p_ru_pm_pool(control, list, free, list_va, free_va, physical, ++ blocks); ++ return ret ?: g5p_ru_pm_state(control, aux_va, list_va); ++} ++ ++/* The r25 latched selections that can still yield a frame. Context/host/SKU ++ * probes and the heap-relative RT dialect are diagnostic-only in the fixed ++ * path (pair rejected) and are not offered here; blocks is pm_blocks. ++ */ ++static inline bool g5p_ru_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return g5p_r25_valid(v) && !g5p_r25_diagnostic(v) && !v->blocks; ++} ++ ++/* r25 render_pm_variant 2/3 on a pool from g5p_ru_pm()/g5p_ru_pm_pool(): ++ * reverse only F's block order, or report F's whole mapped page as its ++ * extent. Equal to g5p_r25_pool() for 32, 64 and 128 blocks. ++ */ ++static inline void g5p_ru_pm_variant(u8 *control, u8 *free, const u64 *physical, ++ u32 blocks, u32 variant) ++{ ++ if (variant == 2) ++ for (u32 j = 0; j < blocks; j++) ++ for (u32 k = 0; k < 8; k++) { ++ u32 ppn = physical[(blocks - 1 - j) * 8 + k] >> ++ 12; ++ ++ for (u32 p = 0; p < 4; p++) ++ g5p_put32(free + ((j * 8 + k) * 4 + p) * ++ 4, ++ ppn + p); ++ } ++ if (variant == 3) ++ g5p_put32(control + 0x1c, G5P_GPU_PAGE); ++} ++ ++/* r25 render_pm_variant 1, after g5p_ru_job() built the TA page: tag6+10 ++ * names an owned holder with the linked head (outside CH/ring/W/SKU/TQ/ ++ * dependency); F/count/head start clear and last=ffffffff, so 10005368 ++ * rebuilds them. Equal to g5p_r25_inputs() for the same pool. ++ */ ++ ++/* Persistent per-channel objects (6c89420 layout): written once at attach. ++ * Control indices, TQ and the dependency object stay firmware-owned after. ++ */ ++static inline void g5p_ru_channel_init(u8 *p, u32 meta) ++{ ++ memset(p, 0, G5P_GPU_PAGE); ++ g5p_put32(p, meta + G5P_WORK_CTL); ++ g5p_put32(p + 4, meta + G5P_WORK_RING); ++ g5p_put32(p + 8, meta + G5P_R22_TQ); ++ g5p_put32(p + 0xc, 4); ++ g5p_put32(p + 0x18, ~0U); ++ g5p_put32(p + G5P_WORK_CTL + 0x50, G5P_WORK_SLOTS); ++ g5p_put32(p + G5P_R22_TQ + 4, meta + G5P_R22_TQ); ++ g5p_put64(p + G5P_R22_DEP + 0x30, 1); ++ g5p_put64(p + G5P_R22_DEP + 0x48, ~0ULL); ++ g5p_put32(p + G5P_R22_DEP + 0x50, ~0U); ++ for (u32 off = 0x70; off < 0x270; off += 0x10) ++ g5p_put32(p + G5P_R22_DEP + off + 8, 0xf); ++} ++ ++/* Pointer-ring positions of job `seq` (1-based): two entries per job. */ ++static inline u32 g5p_ru_ring_index(u32 seq) ++{ ++ return (2 * (seq - 1)) % G5P_WORK_SLOTS; ++} ++ ++static inline u32 g5p_ru_endpoint(u32 seq) ++{ ++ return (2 * seq) % G5P_WORK_SLOTS; ++} ++ ++/* perf60 job slots (render_user_job_slots, drm-render-v3.inc). A job's command, W record and ++ * microsequence ("SKU") live in slot 0 (the r22 offsets) or slot 1, in the channel page's free ++ * tail; the firmware reaches both through the job's two pointer-ring entries. The channel header, ++ * control, pointer ring, TQ, stamps and dependency object stay shared. Slot 1 clears TQ/STAMP/DEP ++ * (0x1800..0x1b70) and the r25 holder (G5P_R25_HEAD, 0x3000). ++ */ ++#define G5P_RU_SLOT1 0x1c00U ++#define G5P_RU_SLOT1_END \ ++ (G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES + 0x180U) ++ ++static inline u32 g5p_ru_cmd_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 : G5P_WORK_CMD; ++} ++ ++static inline u32 g5p_ru_w_at(u32 js) ++{ ++ return js ? G5P_RU_SLOT1 + G5P_RU_CMD_BYTES : G5P_R22_W; ++} ++ ++static inline u32 g5p_ru_sku_at(u32 js, bool td) ++{ ++ if (js) ++ return G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES; ++ return td ? G5P_R22_3D_SKU : G5P_R22_TA_SKU; ++} ++ ++/* Per-job objects for sequence `seq`: W image (fixed+typed+RT), channel W ++ * fields, the full timestamped microsequence, the tag6 PM bootstrap (TA) or ++ * tag4 TA-finish dependency (3D) and both pointer-ring entries. With seq=1 ++ * this equals g5p_r22_channel(). Stamp target and sequence are both seq. ++ * ta_finish is M(TA stamp word addressed by TA.W530), used by 3D only. ++ */ ++static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, ++ bool td, u32 seq, const u8 *image, ++ u32 ta_finish, bool zero_timestamps, u32 js, ++ u32 pmid) ++{ ++ u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; ++ u32 finish = meta + G5P_R22_STAMP + 4; ++ u32 sku = g5p_ru_sku_at(js, td), cmd = g5p_ru_cmd_at(js); ++ u32 ring = g5p_ru_ring_index(seq); ++ u8 *work = p + g5p_ru_w_at(js); ++ u32 cursor = sku, starts, finishes; ++ ++ memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); ++ memset(p + sku, 0, td ? 0x180 : 0xc0); ++ g5p_put32(work, td ? 1 : 0); ++ g5p_put32(work + 4, slot); ++ g5p_put32(work + (td ? 0x14 : 0xc), meta + G5P_R22_DEP); ++ g5p_put32(work + (td ? 0x18 : 0x14), pm); ++ if (!td && pmid) ++ g5p_put32(work + 0x10, ++ pmid); /* TA W+10: the PM identity (iOS Dcb0) */ ++ g5p_put32(work + (td ? 0x1c : 0x18), state); ++ g5p_put32(work + (td ? 0xc : 0x454), meta + sku); ++ g5p_put32(work + (td ? 0x10 : 0x458), td ? 0x180 : 0xc0); ++ g5p_put32(work + (td ? 0x888 : 0x52c), finish - 4); ++ g5p_put32(work + (td ? 0x88c : 0x530), finish); ++ g5p_put32(work + (td ? 0x890 : 0x534), seq); ++ g5p_put32(work + (td ? 0x894 : 0x538), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x898 : 0x53c), td ? 1 : 0); ++ g5p_put32(work + (td ? 0x8a0 : 0x544), seq); ++ { ++ u32 ta_start[] = { ++ 0x23, ++ w + 0x3a4, ++ w + 0x20, ++ state, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ pmid, ++ 0, ++ 0, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ 0, ++ seq, ++ }; ++ u32 td_start[] = { ++ 0x25, ++ w + 0x3a0, ++ w + 0x58, ++ state, ++ global + 0x724, ++ w + 0x7f0, ++ w + 0x7f4, ++ w + 0x878, ++ meta, ++ w, ++ slot, ++ 0, ++ pmid, ++ g5p_le32(work + 0x800), ++ 0, ++ 1, ++ w + 0x6d8, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ 0, ++ seq, ++ }; ++ u32 ta_finish[] = { ++ 0x24, ++ state, ++ pm, ++ global + 0x724, ++ meta, ++ slot, ++ 0, ++ w + 0x464, ++ g5p_le32(work + 0x51c), ++ g5p_le32(work + 0x520), ++ seq, ++ finish, ++ seq, ++ (u32)-0x70, ++ }; ++ u32 td_finish[] = { ++ 0x26, ++ seq, ++ finish, ++ seq, ++ state, ++ pm, ++ 1, ++ global + 0x724, ++ w + 0x878, ++ w + 0x7f0, ++ meta, ++ w, ++ slot, ++ w + 0x6d8, ++ 0, ++ g5p_le32(work + 0x87c), ++ g5p_le32(work + 0x884), ++ (u32)-0x104, ++ }; ++ u32 result = w + (td ? 0x8ac : 0x550); ++ u32 timestamp = w + (td ? 0x8b4 : 0x558); ++ u32 ts0[] = { 0x80000018, result, timestamp, timestamp, seq }; ++ u32 ts1[] = { 0x18, result, timestamp, timestamp + 4, seq }; ++ ++ starts = td ? ARRAY_SIZE(td_start) : ARRAY_SIZE(ta_start); ++ finishes = td ? ARRAY_SIZE(td_finish) : ARRAY_SIZE(ta_finish); ++ for (u32 i = 0; i < starts; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_start[i] : ta_start[i]); ++ for (u32 i = 0; i < ARRAY_SIZE(ts0); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts0[i]); ++ if (td) { ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_r22_wait_words); ++ i++, cursor += 4) ++ g5p_put32(p + cursor, g5p_r22_wait_words[i]); ++ } else { ++ g5p_put32(p + cursor, 0x801); ++ cursor += 4; ++ } ++ for (u32 i = 0; i < ARRAY_SIZE(ts1); i++, cursor += 4) ++ g5p_put32(p + cursor, zero_timestamps ? 1 : ts1[i]); ++ for (u32 i = 0; i < finishes; i++, cursor += 4) ++ g5p_put32(p + cursor, td ? td_finish[i] : ta_finish[i]); ++ g5p_put32(p + cursor, 0x40000017); ++ } ++ memset(p + cmd, 0, G5P_RU_CMD_BYTES); ++ g5p_put32(p + cmd, td ? 4 : 6); ++ g5p_put32(p + cmd + 4, td ? ta_finish : slot); ++ g5p_put32(p + cmd + 8, ++ td ? seq : pmid); /* tag6 {6, STA, PMid, M(control)} */ ++ g5p_put32(p + cmd + 12, td ? seq & 0x7fffffff : pm); ++ g5p_put32(p + G5P_WORK_RING + ring * 4, meta + cmd); ++ g5p_put32(p + G5P_WORK_RING + ((ring + 1) % G5P_WORK_SLOTS) * 4, w); ++} ++ ++/* Retirement of job `seq`: both W pointers still ours, finish word and ++ * retirement mirror at seq, all five channel watermarks at its endpoint. ++ * g5p_r22_retired() is the seq=1 case. ++ */ ++static inline bool g5p_ru_retired_at(const u8 *p, u32 meta, bool td, u32 seq, ++ u32 js) ++{ ++ const u8 *w = p + g5p_ru_w_at(js); ++ u32 endpoint = g5p_ru_endpoint(seq); ++ ++ if (g5p_le32(w + (td ? 0x888 : 0x52c)) != meta + G5P_R22_STAMP || ++ g5p_le32(w + (td ? 0x88c : 0x530)) != meta + G5P_R22_STAMP + 4 || ++ g5p_le32(w + (td ? 0x890 : 0x534)) != seq || ++ g5p_le32(p + G5P_R22_STAMP) != seq || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) != endpoint) ++ return false; ++ return true; ++} ++ ++/* A stamp beyond the published target, or a watermark not on the even ++ * two-entry grid, is a protocol error rather than "not yet retired". ++ */ ++static inline bool g5p_ru_stamp_sane(const u8 *p, u32 seq) ++{ ++ for (u32 i = 0; i < 2; i++) { ++ s32 ahead = (s32)(g5p_le32(p + G5P_R22_STAMP + i * 4) - seq); ++ ++ if (ahead > 0) ++ return false; ++ } ++ for (u32 off = 0; off <= 0x40; off += 0x10) ++ if (g5p_le32(p + G5P_WORK_CTL + off) >= G5P_WORK_SLOTS) ++ return false; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-loss.h b/drivers/gpu/apple-g5p/render-user-loss.h +new file mode 100644 +index 000000000..2aad49d88 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-loss.h +@@ -0,0 +1,120 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context fault isolation (docs/gpu/multi-client-design.md "Stage 2/3 as built", fault ++ * isolation). Pure decisions, host-tested (scripts/gpu/multiclient2-host/test_render_loss.c). ++ * ++ * The hardware gives no way to abandon a job: the firmware has no abort (iOS ends a ++ * non-progressing engine with a firmware stop, a platform power cycle and a new firmware boot, ++ * kext 6c9e070, firmware-re README "Fault recovery"), the TA and 3D channels are FIFO, and the ++ * one TA and one 3D engine serve every context. So one context's job can be *lost* (its client ++ * gets an error, its context is dead) without ending the device only while the firmware is ++ * alive, and only if the stuck phase still finishes: until then no other job may be published ++ * behind it (it would wait behind the same engine and be lost in turn). ++ * ++ * loss at the end of a phase that did not complete: ++ * NONE it completed; ++ * CONTEXT a phase timeout on a job other than the epoch's first, with the firmware ++ * alive (its event stream never went silent, no fault event, no protocol ++ * error) and isolation on: the job and its context are lost, the device is ++ * not; the channel is blocked behind the stuck phase; ++ * DEVICE anything else: the epoch is terminal, as before. ++ * drain while blocked, the scheduler runs no job and waits, RAM only, for the stuck phase: ++ * a late TA gets its 3D published (the channel and PM state then stay consistent), ++ * a late 3D ends it; the channel is then clear and every held job runs. A silent ++ * event stream, a fault, a refused 3D or the drain bound make the epoch terminal. ++ */ ++#ifndef G5P_RENDER_USER_LOSS_H ++#define G5P_RENDER_USER_LOSS_H ++ ++enum g5p_ru_loss { ++ G5P_RU_LOSS_NONE, ++ G5P_RU_LOSS_CONTEXT, ++ G5P_RU_LOSS_DEVICE, ++}; ++ ++struct g5p_ru_loss_in { ++ u32 result; /* G5P_RU_WAIT_* of the phase */ ++ bool first; /* the epoch's first job (its attach and bell) */ ++ bool isolate; /* render_user_drain_ms != 0 */ ++ bool silent; /* the event stream stopped: the firmware is dead */ ++ bool fault; /* a fault event, a failed boot, a protocol or drain error */ ++}; ++ ++static inline int g5p_ru_loss(const struct g5p_ru_loss_in *in) ++{ ++ if (in->result == G5P_RU_WAIT_COMPLETE) ++ return G5P_RU_LOSS_NONE; ++ if (in->result != G5P_RU_WAIT_TIMEOUT || in->first || !in->isolate || ++ in->silent || in->fault) ++ return G5P_RU_LOSS_DEVICE; ++ return G5P_RU_LOSS_CONTEXT; ++} ++ ++/* The drain of a blocked channel. */ ++enum g5p_ru_drain_state { ++ G5P_RU_DRAIN_TA, /* waiting for the stuck TA */ ++ G5P_RU_DRAIN_TD, /* waiting for the stuck job's 3D */ ++ G5P_RU_DRAIN_CLEAR, /* done: the channel is clear */ ++ G5P_RU_DRAIN_DEAD, /* the epoch is terminal */ ++}; ++ ++/* What the drain does next. */ ++enum g5p_ru_drain_action { ++ G5P_RU_DRAIN_WAIT, /* sample again */ ++ G5P_RU_DRAIN_PUBLISH_TD, /* publish the stuck job's 3D, wait for it */ ++ G5P_RU_DRAIN_UNBLOCK, /* release the held jobs */ ++ G5P_RU_DRAIN_TERMINAL, /* note the epoch terminal, release the held jobs (they fail) */ ++}; ++ ++/* One step from the result of the phase wait (G5P_RU_WAIT_*) in state *st. A refused 3D ++ * publish is reported by the caller as G5P_RU_WAIT_FAULT on the TD state. ++ */ ++static inline int g5p_ru_drain_step(u32 *st, u32 result) ++{ ++ if (*st >= G5P_RU_DRAIN_CLEAR) ++ return *st == G5P_RU_DRAIN_CLEAR ? G5P_RU_DRAIN_UNBLOCK : ++ G5P_RU_DRAIN_TERMINAL; ++ if (result == G5P_RU_WAIT_PENDING) ++ return G5P_RU_DRAIN_WAIT; ++ if (result != G5P_RU_WAIT_COMPLETE) { ++ *st = G5P_RU_DRAIN_DEAD; ++ return G5P_RU_DRAIN_TERMINAL; ++ } ++ if (*st == G5P_RU_DRAIN_TA) { ++ *st = G5P_RU_DRAIN_TD; ++ return G5P_RU_DRAIN_PUBLISH_TD; ++ } ++ *st = G5P_RU_DRAIN_CLEAR; ++ return G5P_RU_DRAIN_UNBLOCK; ++} ++ ++/* ++ * The drain bound must leave the held clients' own wait intact. A client waits for its job up to ++ * the job_timeout_ms GET_RENDER_INFO reports plus a margin (the ICD: + 3 s, so 2 * hard + obs + 3 s ++ * = 125 s with latency101 on, 33 s with it off), and a held job has already waited out the stuck ++ * phase (up to the hard bound, or render_user_wait_ms) before the drain starts, then runs after ++ * it. Valid: 0 (isolation off), or from 1000 up to hard + obs - 10000 with latency101 on (hard ++ * 60000, obs 2000: 52000), and up to 8000 with it off. An invalid value at the time of a loss ++ * turns isolation off for it. ++ */ ++static inline bool g5p_ru_drain_valid(u32 drain_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!drain_ms) ++ return true; ++ if (drain_ms < 1000) ++ return false; ++ if (!hard_ms) ++ return drain_ms <= 8000; ++ return hard_ms + obs_ms >= 11000 && ++ drain_ms <= hard_ms + obs_ms - 10000; ++} ++ ++ ++ ++static inline const char *g5p_ru_loss_name(int loss) ++{ ++ return loss == G5P_RU_LOSS_NONE ? "none" : ++ loss == G5P_RU_LOSS_CONTEXT ? "context" : "device"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-mmu.h b/drivers/gpu/apple-g5p/render-user-mmu.h +new file mode 100644 +index 000000000..5a05dccbf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-mmu.h +@@ -0,0 +1,60 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * perf60 MMU invalidation arm (render_user_mmu_inval, drm-render-v3.inc), pure helpers. ++ * ++ * The only invalidation of a user MMU context besides DC3 is the firmware's per-kick tag compare ++ * (10017ac4): diff = host_tag[S] ^ cached_tag[S]; bits 31:22 set -> e138 = (S << 3) | 4 (PC), ++ * else bits 21:12 -> | 2 (PD), else bits 11:0 -> | 1 (PT); then cached = host. One invalidate per ++ * kick, the highest changed field (docs/gpu/firmware-re/README.md "generation producer"). A PC ++ * field change therefore asks for the firmware's PC-level invalidate of that context, the level ++ * Apple's own producer (6cabe70) and DC3 (1000218c) use as the strongest one. ++ * ++ * g5p_ru_mmu_pc_tag(): a host tag whose PC field differs from both slots' cached PC fields, so ++ * the next kick of each slot takes the PC branch. The PD and PT fields are the given tag's. ++ * Never 0 (0 is the unbound tag) and never equal to either cached tag. Host-tested ++ * (scripts/gpu/perf60-host/test_mmu_inval.c). ++ */ ++#ifndef G5P_RENDER_USER_MMU_H ++#define G5P_RENDER_USER_MMU_H ++ ++#define G5P_RU_MMU_PC_MASK 0xffc00000U ++#define G5P_RU_MMU_PC_ONE 0x00400000U ++ ++/* render_user_mmu_inval bits */ ++#define G5P_RU_MMU_EDIT \ ++ 0x1U /* every VM edit publication (and spare adoption) asks for PC */ ++#define G5P_RU_MMU_SWITCH \ ++ 0x2U /* a bound context's job after another context's job: PC retag */ ++#define G5P_RU_MMU_TOUCH \ ++ 0x4U /* NC read of every table line before the tag store */ ++#define G5P_RU_MMU_EVERY \ ++ 0x8U /* every job of a bound context: PC retag (superset of SWITCH) */ ++ ++static inline u32 g5p_ru_mmu_pc_tag(u32 tag, u32 cached0, u32 cached1) ++{ ++ u32 pc = (tag & G5P_RU_MMU_PC_MASK) + G5P_RU_MMU_PC_ONE; ++ ++ /* At most two values are excluded, so three candidates always contain a free one. */ ++ for (u32 i = 0; i < 3; i++, pc += G5P_RU_MMU_PC_ONE) { ++ u32 t = (pc & G5P_RU_MMU_PC_MASK) | (tag & ~G5P_RU_MMU_PC_MASK); ++ ++ if (((t ^ cached0) & G5P_RU_MMU_PC_MASK) && ++ ((t ^ cached1) & G5P_RU_MMU_PC_MASK) && t) ++ return t; ++ } ++ return 0; /* unreachable: kept for the host test */ ++} ++ ++/* ++ * Retag on a job: the published host tag, or the cached one if none, moved to a fresh PC field. ++ */ ++static inline bool g5p_ru_mmu_retag_due(u32 mode, bool bound_before, ++ bool switched) ++{ ++ if (!bound_before) ++ return false; /* this job's DC3 or adoption publishes its own tag */ ++ return (mode & G5P_RU_MMU_EVERY) || ++ ((mode & G5P_RU_MMU_SWITCH) && switched); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-msaa.h b/drivers/gpu/apple-g5p/render-user-msaa.h +new file mode 100644 +index 000000000..f27f197a6 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-msaa.h +@@ -0,0 +1,88 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_MSAA_H ++#define G5P_RENDER_USER_MSAA_H ++ ++#include "bits-compat.h" ++/* TYPED_V3 4x multisampling: the per-job words a pass with a 4-sample render ++ * target needs on top of the V1/V2 3D words and the target's geometry ++ * (g5p_ru_rt_derive_samples()). Pure byte builders over the kernel's own ++ * work records: no allocation, MMIO, locking or firmware publication. ++ * ++ * Evidence (docs/gpu/experiments/msaa.md): the iOS gles3/gles3b captures ++ * 20261101T175626Z, msaa4-resolve, msaa4-store, g3b-a2c and g3b-samplemask, ++ * against g3-base, and the importer's 3D/TA fillers 6ca7ae0/6ca9710 executed ++ * with the 4-sample descriptor (build/gpu/msaa/filler-msaa.c): ++ * record+0x4d8 0xf -> 0x7 D6a8 -> 3D W3c0[3:0] (ISP_CTL; the draws' ++ * ISPCTL.upass drops 0xf -> 0x7 with it) ++ * record+0x640 0 -> 1 D6ad -> 3D W3c0 bit 17 and W86c (resolve only; ++ * bit 17 is ISP_CTL.PROCESS_EMPTY_TILES, which ++ * Linux's template sets for every pass since ++ * 0e05f993cf, so only W86c is new) ++ * RT Rb8 0 -> 2 D6e0 -> 3D Wd0 (AA mode, CR f30), and the ++ * pixel-output shift 10 -> 11 in W3c0 [21], ++ * W3c4, W3c8 and the six W108 words ++ * RT Rb0 / Rf8 3D W58 (CR fa8) / TA W40 (PPP_MULTISAMPLECTL) ++ * The rest of the target's geometry reaches W through g5p_ru_rt_fill(). ++ */ ++#define G5P_RU_MSAA_RESOLVE 1U /* DRM_G5P_RENDER_MSAA_RESOLVE */ ++ ++struct g5p_ru_msaa { ++ u32 samples, flags; /* 0 = 1 */ ++ u64 positions; /* 0 = the target's */ ++}; ++ ++/* samples 0 or 1 is the one-sample pass and must use a one-sample target; ++ * 4 must use a four-sample target. Positions are one byte per sample, so a ++ * word with bits above the sample count is malformed; zero takes the ++ * target's (the standard pattern). Resolve exists only with four samples. ++ * Nothing here is a hardware hazard: positions are a 4-bit grid and the ++ * resolve bit changes what the PBE writes, not where. ++ */ ++static inline int g5p_ru_msaa_check(const struct g5p_ru_msaa *m, u32 rt_samples) ++{ ++ u32 s = m->samples ? m->samples : 1; ++ ++ if ((s != 1 && s != 4) || s != rt_samples || ++ m->flags & ~G5P_RU_MSAA_RESOLVE || (m->flags && s == 1) || ++ m->positions >> (s * 8)) ++ return -EINVAL; ++ return 0; ++} ++ ++/* After g5p_ru_fill_3d(), the dialect's width words and g5p_ru_rt_fill(). ++ * A one-sample pass is left as it is, unless it names its position. ++ */ ++static inline void g5p_ru_fill_msaa(u8 *ta, u8 *td, const struct g5p_ru_msaa *m, ++ const struct g5p_ru_rt *rt) ++{ ++ /* 6ca7ae0's output-width code table 5942678, by width - 2. */ ++ static const u8 width_code[7] = { 0, 0, 0x10, 0, 0, 0, 0x20 }; ++ u64 positions = m->positions ? m->positions : rt->packed; ++ bool resolve = m->flags & G5P_RU_MSAA_RESOLVE; ++ u32 width, count, mode, w3c0; ++ ++ if (m->samples != 4 && !m->positions) ++ return; ++ g5p_put64(ta + 0x40, positions); ++ g5p_put64(td + 0x58, positions); ++ if (m->samples != 4) ++ return; ++ /* The width the dialect wrote (2, 4 or 8); 6ca7ae0 shifts by 11 with an ++ * AA mode, so each width has half the output partitions. ++ */ ++ width = g5p_le32(td + 0x6b0); ++ count = 0x4000 / (width << 11); ++ mode = count > 1 ? 2 : count; ++ w3c0 = g5p_le32(td + 0x3c0) & ~(0xfU | BIT(21)); ++ g5p_put32(td + 0x3c0, ++ w3c0 | 0x7 | (resolve ? BIT(17) : 0) | (mode - 1) << 21); ++ g5p_put32(td + 0x3c4, mode * 0x204); ++ g5p_put32(td + 0x3c8, count & 0x1f); ++ for (u32 i = 0; i < 6; i++) ++ g5p_put64(td + 0x108 + i * 8, ++ width_code[width - 2] | (count - 1) << 7 | 0x43); ++ g5p_put64(td + 0xd0, rt->state); ++ g5p_put32(td + 0x86c, resolve); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-pmset.h b/drivers/gpu/apple-g5p/render-user-pmset.h +new file mode 100644 +index 000000000..170738f1f +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-pmset.h +@@ -0,0 +1,77 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Per-context PM records (perf60 stage 4, docs/gpu/multi-client-design.md risk R3). ++ * ++ * iOS acquires a ParamBuffer ID per prepare from a 127-entry pool (6cb9afc) and builds a PM ++ * control/state/aux object and its block list per PM (6cb2378); one PM is never shared by two ++ * work queues. Linux built one device PM (control+S at shared+G5P_R22_PM, list and F in the ++ * two render_meta pages, aux at G5P_RU_PRIVATE_VA) for every render context; with the GPU ++ * never parked, one context's job then started from the PM state another context's job left. ++ * ++ * boot_pm_sets reserves N sets in the firmware GiB at boot, right after render_meta: set s is ++ * three pages {control + S, list, F} at G5P_PMSET_VA + s * 3 pages, mapped like the shared ++ * pages (they are firmware objects the firmware and the PM requesters address by meta VA). ++ * A render context takes one set, its own pool pages and its own aux pages. PM ID: 0 stays ++ * the device PM's; set s may carry s + 1. ++ * ++ * Pure helpers, host-tested (scripts/gpu/perf60-host/test_pmset.c). ++ */ ++#ifndef G5P_RENDER_USER_PMSET_H ++#define G5P_RENDER_USER_PMSET_H ++ ++#define G5P_PMSET_MAX 8U ++#define G5P_PMSET_PAGES 3U /* control + S, list, F */ ++#define G5P_PMSET_VA (G5P_SHARED_VA + G5P_SHARED_SIZE + 2 * G5P_GPU_PAGE) ++#define G5P_PMSET_END \ ++ (G5P_PMSET_VA + G5P_PMSET_MAX * G5P_PMSET_PAGES * G5P_GPU_PAGE) ++ ++enum { G5P_PMSET_CONTROL, G5P_PMSET_LIST, G5P_PMSET_FREE }; ++ ++static inline u64 g5p_pmset_va(u32 set, u32 part) ++{ ++ return G5P_PMSET_VA + ++ ((u64)set * G5P_PMSET_PAGES + part) * G5P_GPU_PAGE; ++} ++ ++static inline u32 g5p_pmset_id(u32 set) ++{ ++ return set + 1; ++} ++ ++/* ++ * The PM's write-back target (release8 bench-gaps3, 2026-11-03). After the GPU switches from ++ * context A's jobs to context B's, the firmware's PM keeps writing A's free list F back, through ++ * A's TA slot (its MMU context), on B's later jobs, until a job of a third context runs: two ++ * faults, one on the 2nd and one on the 3rd job of B after A's last, both the first job after A ++ * was unbound (page-invalid level 2 in A's MMU context, A's set F page; with A still bound the ++ * same writes landed). So a context's slots and set stay bound while it is the context of the ++ * last job or the previous *distinct* one: g5p_pmset_track() at every job end, g5p_pmset_holds() ++ * at close. That bounds when the slots may go, not when the PM stops: two later faults (release8 ++ * quiet2, seq 858 and 893) wrote the free list of a context two switches back, after this rule had ++ * released it. So a released pair is parked rather than left empty (render_user_close_park, ++ * drm-render-v3.inc), and a released set is reused last (g5p_pmset_pick()). ++ */ ++static inline bool g5p_pmset_switched(const void *last, const void *job) ++{ ++ return last != job; ++} ++ ++static inline bool g5p_pmset_holds(const void *last, const void *prev, ++ const void *ctx) ++{ ++ return ctx && (ctx == last || ctx == prev); ++} ++ ++/* A free set, round-robin from `next` (a released set is reused last). -1: none free. */ ++static inline int g5p_pmset_pick(const bool *used, u32 n, u32 next) ++{ ++ for (u32 i = 0; i < n; i++) { ++ u32 s = (next + i) % n; ++ ++ if (!used[s]) ++ return s; ++ } ++ return -1; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-quiesce.h b/drivers/gpu/apple-g5p/render-user-quiesce.h +new file mode 100644 +index 000000000..3671e415e +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-quiesce.h +@@ -0,0 +1,145 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The GPU quiesce before a kexec, reboot or power-off (docs/gpu/experiments/mcfold2-kexec.md). ++ * ++ * Without it the firmware keeps running from the outgoing kernel's RAM across the relocation: ++ * it still writes its event rings, and the MMU still holds a CBASE for every bound render slot ++ * pointing at page tables the next kernel reuses, until the next kernel's PMGR GFX cycle about ++ * 0.4 s into its boot. The quiesce ends that window on the outgoing side, in this order: ++ * ++ * 1 close job admission and wait (bounded) for submitters inside; ++ * 2 take the command lock (bounded trylock), so no job or command is in flight; ++ * 3 DETACH: every user slot the firmware may hold is unbound by a RAM-only DC3 batch ++ * (render-user-bind.h), only when the lock is ours, the firmware is healthy, no timeout ++ * or fault was noted and the DC ring is idle; ++ * 4 freeze the boot state and mask the GPU IRQ (no host register access after this); ++ * 5 CYCLE: one bounded PMGR GFX off/on, the same gate cycle the next kernel performs with ++ * apple_pmgr.kexec_gpu_cycle=1, only when no host thread can be inside a GPU register ++ * access: the lock is ours, or its holder is the TYPED_V3 executor in its RAM-only regime. ++ * ++ * Pure functions over the ops below, host-tested (scripts/gpu/mcfold2-host/test_kexec_quiesce.c). ++ */ ++#ifndef G5P_RENDER_USER_QUIESCE_H ++#define G5P_RENDER_USER_QUIESCE_H ++ ++/* kexec_quiesce module parameter */ ++#define G5P_RU_KQ_OFF 0U ++#define G5P_RU_KQ_RAM 1U /* detach only, no PMGR access */ ++#define G5P_RU_KQ_FULL 2U /* detach and cycle (default) */ ++ ++/* Plan */ ++#define G5P_RU_KQ_DETACH 0x1U ++#define G5P_RU_KQ_CYCLE 0x2U ++ ++/* Why a step was skipped */ ++#define G5P_RU_KQ_WHY_OFF 0x001U /* kexec_quiesce=0 */ ++#define G5P_RU_KQ_WHY_DOWN 0x002U /* the firmware never ran on this boot */ ++#define G5P_RU_KQ_WHY_BUSY 0x004U /* the command lock is not ours: no detach */ ++#define G5P_RU_KQ_WHY_FAILED \ ++ 0x008U /* fault or protocol failure: firmware state unknown */ ++#define G5P_RU_KQ_WHY_WEDGED \ ++ 0x010U /* a TYPED_V3 timeout or fault: a job may hold its slots */ ++#define G5P_RU_KQ_WHY_RING 0x020U /* the DC ring is not idle */ ++#define G5P_RU_KQ_WHY_NONE 0x040U /* no user slot is held */ ++#define G5P_RU_KQ_WHY_HOLDER \ ++ 0x080U /* unknown lock holder, maybe inside MMIO: no cycle */ ++#define G5P_RU_KQ_WHY_NO_PMGR 0x100U ++#define G5P_RU_KQ_WHY_RAM 0x200U /* kexec_quiesce=1: no cycle */ ++#define G5P_RU_KQ_WHY_NO_DRM 0x400U /* no render frontend: nobody can detach */ ++ ++struct g5p_ru_kq_in { ++ u32 mode; /* kexec_quiesce */ ++ bool booted; /* the firmware was released on this boot */ ++ bool locked; /* the command lock is ours */ ++ bool failed; /* b->failed or b->event_failed */ ++ bool frontend; /* a render frontend that can detach is registered */ ++ bool term_failed; /* the TYPED_V3 latch noted a timeout or fault */ ++ bool ram_only; /* the TYPED_V3 latch is quiet: its executor touches no register */ ++ bool executing; /* the TYPED_V3 executor is inside a job */ ++ bool ring_idle; /* DC consumer == producer <= 255 */ ++ bool pmgr; /* the PMGR regmap is present */ ++ u32 held; /* user slots with a PA or a host tag */ ++ u32 cons, prod; /* the DC ring when sampled */ ++}; ++ ++static inline u32 g5p_ru_kq_plan(const struct g5p_ru_kq_in *in, u32 *why) ++{ ++ u32 plan = 0, w = 0; ++ ++ if (in->mode == G5P_RU_KQ_OFF || !in->booted) { ++ *why = in->mode == G5P_RU_KQ_OFF ? G5P_RU_KQ_WHY_OFF : ++ G5P_RU_KQ_WHY_DOWN; ++ return 0; ++ } ++ if (!in->locked) ++ w |= G5P_RU_KQ_WHY_BUSY; ++ if (in->failed) ++ w |= G5P_RU_KQ_WHY_FAILED; ++ if (in->term_failed) ++ w |= G5P_RU_KQ_WHY_WEDGED; ++ if (!in->ring_idle) ++ w |= G5P_RU_KQ_WHY_RING; ++ if (!in->held) ++ w |= G5P_RU_KQ_WHY_NONE; ++ if (!in->frontend) ++ w |= G5P_RU_KQ_WHY_NO_DRM; ++ if (!w) ++ plan |= G5P_RU_KQ_DETACH; ++ if (in->mode < G5P_RU_KQ_FULL) ++ w |= G5P_RU_KQ_WHY_RAM; ++ else if (!in->pmgr) ++ w |= G5P_RU_KQ_WHY_NO_PMGR; ++ else if (!in->locked && !(in->executing && in->ram_only)) ++ w |= G5P_RU_KQ_WHY_HOLDER; ++ else ++ plan |= G5P_RU_KQ_CYCLE; ++ *why = w; ++ return plan; ++} ++ ++struct g5p_ru_kq_ops { ++ void *ctx; ++ u32 (*admit_close)(void *ctx); /* close admission; submitters still inside */ ++ bool (*lock)(void *ctx); /* bounded trylock of the command lock */ ++ void (*unlock)(void *ctx); ++ void (*inputs)(void *ctx, struct g5p_ru_kq_in *in); /* RAM only */ ++ int (*detach)(void *ctx); /* RAM only */ ++ void (*freeze)(void *ctx); /* RAM only */ ++ void (*irq_off)(void *ctx); /* AIC ack, mask, wait for the handler */ ++ int (*cycle)(void *ctx); /* PMGR only */ ++}; ++ ++struct g5p_ru_kq_out { ++ struct g5p_ru_kq_in in; ++ u32 inside, plan, why; ++ int detach_ret, cycle_ret; ++ bool froze; ++}; ++ ++/* One quiesce. detach_ret/cycle_ret stay 0 for a step that did not run (see plan). */ ++static inline void g5p_ru_kq_run(const struct g5p_ru_kq_ops *o, u32 mode, ++ struct g5p_ru_kq_out *out) ++{ ++ *out = (struct g5p_ru_kq_out){ .in.mode = mode }; ++ if (mode == G5P_RU_KQ_OFF) { ++ out->why = G5P_RU_KQ_WHY_OFF; ++ return; ++ } ++ out->inside = o->admit_close(o->ctx); ++ out->in.locked = o->lock(o->ctx); ++ o->inputs(o->ctx, &out->in); ++ out->plan = g5p_ru_kq_plan(&out->in, &out->why); ++ if (out->plan & G5P_RU_KQ_DETACH) ++ out->detach_ret = o->detach(o->ctx); ++ if (out->in.booted) { ++ o->freeze(o->ctx); ++ o->irq_off(o->ctx); ++ out->froze = true; ++ } ++ if (out->plan & G5P_RU_KQ_CYCLE) ++ out->cycle_ret = o->cycle(o->ctx); ++ if (out->in.locked) ++ o->unlock(o->ctx); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-recover.h b/drivers/gpu/apple-g5p/render-user-recover.h +new file mode 100644 +index 000000000..51982c5c5 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-recover.h +@@ -0,0 +1,107 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * In-boot GPU recovery: the firmware's recovery packet and the recovery decision. Pure, ++ * host-tested (scripts/gpu/perf60-host/test_recover.c). ++ * ++ * The recovery packet is firmware event type 4 (44 bytes). The AGX kext's event drain ++ * (AGXFirmware_drainFirmwareEventRing 6c99788, case 4) reads +0x0c (the faulting slot, 0x80 for ++ * -1), +0x10 (a flag, accel+0x1af8 bit 3), +0x18, +0x20 and +0x24 (accel+0x1a1c/0x1af4/0x1af0), ++ * marks a recovery pending (accel+0x1a24 = 2) and calls IOGPUScheduler_signalHardwareError(sched, ++ * 1) (6c67b4c): "GPURestartSignaled", the scheduler enqueues one GPU restart (a second signal ++ * before it runs is "Redundant hardware error"). The restart is ++ * AGXMetaFirmware_stopForRecoveryOptionalPowerCycle (6ca1e40) and ++ * AGXMetaFirmware_rebootAfterRecovery (6ca1efc), which panics only if the firmware does not boot ++ * again. There is no per-boot limit: every recovery packet restarts the GPU. The ++ * kext does not read +0x04. ++ * ++ * On A9/Linux the slot matches the phase the host waited in: 0 on every TA fault, 1 on the 3D fault ++ * of cf7 (2026-11-02); +0x20 = 8 and +0x24 = 0x81818105 (TA) / 0x81810581 (3D) so far. ++ */ ++#ifndef G5P_RENDER_USER_RECOVER_H ++#define G5P_RENDER_USER_RECOVER_H ++ ++#define G5P_FW_EVENT_RECOVERY 4 ++#define G5P_FW_RECOVERY_NOSLOT 0x80 ++ ++struct g5p_fw_recovery { ++ u32 code; /* +0x04, not read by the kext */ ++ u32 slot; /* +0x0c, G5P_FW_RECOVERY_NOSLOT for -1 */ ++ u32 flag; /* +0x10 != 0 */ ++ u32 w18, w20, w24; ++}; ++ ++/* `w` is the 11-word event entry, type 4. */ ++static inline struct g5p_fw_recovery g5p_fw_recovery_decode(const u32 *w) ++{ ++ struct g5p_fw_recovery r = { ++ .code = w[1], ++ .slot = w[3] == 0xffffffffU ? G5P_FW_RECOVERY_NOSLOT : w[3], ++ .flag = w[4] != 0, ++ .w18 = w[6], ++ .w20 = w[8], ++ .w24 = w[9], ++ }; ++ ++ return r; ++} ++ ++/* ++ * May a job that stopped the firmware (a recovery packet, a silent event stream, a hard timeout) ++ * be recovered from in this boot, the job's context lost and every other kept? ++ * ++ * budget == 0 never (render_user_recover 0: the epoch ends, as before); ++ * retired/closed never (the boot is shutting the GPU down); ++ * streak_max == 0 the old rule: `budget` recoveries per boot, never on the epoch's first job; ++ * streak_max != 0 Apple's rule, every time, with two guards: at most streak_max recoveries in a ++ * row with no job completing between them (a GPU that faults again before it ++ * finishes anything is not recovering), and at most G5P_RU_RECOVER_BOOT_MAX per ++ * boot (recover.inc leaves each old epoch allocated, ~0.8 MiB: 50 x 16 KiB pages ++ * and its struct, so a stale pointer still reads valid memory; this bounds that ++ * to ~26 MiB). The boot's first epoch's first job stays terminal (a failed ++ * bring-up, not a client); the first job of a later epoch (the attach after a ++ * recovery) is recoverable like any other. ++ */ ++#define G5P_RU_RECOVER_BOOT_MAX 32 ++ ++struct g5p_ru_recover_in { ++ bool first; /* the job runs its epoch's attach */ ++ bool retired; /* the GPU was retired (kexec / reboot quiesce) */ ++ bool closed; /* admission closed */ ++ u32 recoveries; /* recoveries this boot */ ++ u32 streak; /* recoveries since the last job completed */ ++ u32 budget; /* render_user_recover */ ++ u32 streak_max; /* render_user_recover_streak */ ++}; ++ ++static inline bool g5p_ru_recoverable(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget || in->retired || in->closed) ++ return false; ++ if (!in->streak_max) ++ return !in->first && in->recoveries < in->budget; ++ if (in->first && !in->recoveries) ++ return false; ++ return in->streak < in->streak_max && ++ in->recoveries < G5P_RU_RECOVER_BOOT_MAX; ++} ++ ++/* Why g5p_ru_recoverable() said no (for the terminal line), or "-" when it said yes. */ ++static inline const char * ++g5p_ru_recover_refusal(const struct g5p_ru_recover_in *in) ++{ ++ if (!in->budget) ++ return "recover-off"; ++ if (in->retired || in->closed) ++ return "shutdown"; ++ if (!in->streak_max) ++ return in->first ? "first-job" : ++ in->recoveries >= in->budget ? "per-boot-budget" : ++ "-"; ++ if (in->first && !in->recoveries) ++ return "first-job"; ++ if (in->streak >= in->streak_max) ++ return "streak"; ++ return in->recoveries >= G5P_RU_RECOVER_BOOT_MAX ? "boot-max" : "-"; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-retire.h b/drivers/gpu/apple-g5p/render-user-retire.h +new file mode 100644 +index 000000000..6e5729003 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-retire.h +@@ -0,0 +1,248 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 3D completion: target coherence and engine idle, decided separately. Pure ++ * functions, no kernel services beyond the integer types, so the host suite compiles this ++ * file as it is. Include render-user-wait.h first. ++ * ++ * combined107 P4R seq 4 (build/gpu/combined107/raw/20261031T145710Z): the 3D target state ++ * was coherent (ring consumer = producer = target 4, both stamps 4, W+0x888/0x88c ours, ++ * W+0x890 = 4, all five channel watermarks at endpoint 8, and the firmware's type-1 3D ++ * completion event with the endpoint pair 7/8 decoded at publish + 19999 ms), yet the ++ * wait timed out at 60002 ms. The wait is a 1-2 ms RAM poll (no waitqueue, no wake), so ++ * this was not a lost edge: the done predicate also required the 3D global's idle words ++ * (G+0x724/0x728/0x72c/0x734/0x738) to be zero, and they never cleared. ++ * ++ * The 3D phase is complete when either ++ * (a) the target is coherent and the idle words are zero (the flown r96 predicate), or ++ * (b) the target is coherent and the matching 3D completion event was decoded after this ++ * job's 3D publish (idle_stuck). The event is the firmware's completion report, the ++ * signal a fence follows on Asahi too; the idle words lag it by an arbitrary time. ++ * settle_ms (default 0) only adds a minimum length of the coherent run, or ++ * (c) the target has been coherent, unbroken, for G5P_RU_RT_HOLD_MS (at least settle_ms): ++ * tahang1, as the TA's rule (c), so a lost or late event does not hold a job whose ++ * stamps landed. The PM state needs no clause here: the TA phase, which the 3D ++ * publish waits for, required it saved. ++ * Target coherent = ring consumer == target && g5p_ru_retired(): both stamps == seq, the W ++ * stamp pointers ours, the W sequence word == seq, every channel watermark == endpoint. ++ * The stamps carry the identity (31-bit sequence, one per job); the watermarks and the ++ * event's endpoint pair repeat every eight jobs and only corroborate. The TA phase has its ++ * own rules (tahang1, below). ++ */ ++#ifndef G5P_RENDER_USER_RETIRE_H ++#define G5P_RENDER_USER_RETIRE_H ++ ++#define G5P_RU_RT_DEF_SETTLE_MS 0U /* default of render_user_idle_settle_ms */ ++#define G5P_RU_RT_MAX_SETTLE_MS \ ++ 5000U /* far below the smallest per-phase bound (20 s) */ ++#define G5P_RU_RT_GRID 16U /* the channel watermark grid, G5P_WORK_SLOTS */ ++#define G5P_RU_RT_HOLD_MS \ ++ 500U /* rule (c), raised to settle_ms when that is longer */ ++ ++/* Clauses of the 3D predicate, as the receipt's `unsat` bits (set = not satisfied). */ ++#define G5P_RU_RT_CONSUMED 0x1U /* ring consumer != target */ ++#define G5P_RU_RT_RETIRED 0x2U /* g5p_ru_retired() false */ ++#define G5P_RU_RT_IDLE 0x4U /* an idle word is non-zero */ ++#define G5P_RU_RT_EVENT 0x8U /* no matching completion event latched */ ++#define G5P_RU_RT_SETTLE \ ++ 0x10U /* coherent, run shorter than the settle or hold window */ ++#define G5P_RU_RT_PM 0x20U /* TA receipt only: the PM state is not saved */ ++ ++enum g5p_ru_rt_how { ++ G5P_RU_RT_PENDING, ++ G5P_RU_RT_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_RT_IDLE_STUCK, /* rule (b) */ ++ G5P_RU_RT_HOLD, /* rule (c) */ ++}; ++ ++struct g5p_ru_retire { ++ u64 settle_ns; /* minimum coherent run before rule (b); 0 = none */ ++ u64 hold_ns; /* rule (c): max(settle_ns, G5P_RU_RT_HOLD_MS) */ ++ u64 run_ns; /* first sample of the current coherent run; 0 = none */ ++ u64 coherent_ns; /* first coherent sample of the phase */ ++ u64 event_ns; /* first sample that saw the matching event latched */ ++ u64 busy_ns; /* first sample with an idle word set */ ++ u64 clear_ns; /* first all-clear sample after busy_ns */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample */ ++ u32 how; /* enum g5p_ru_rt_how of the last sample */ ++ u32 decided; /* enum g5p_ru_rt_how of the sample that decided COMPLETE */ ++}; ++ ++static inline bool g5p_ru_rt_settle_valid(u32 settle_ms) ++{ ++ return settle_ms <= G5P_RU_RT_MAX_SETTLE_MS; ++} ++ ++static inline void g5p_ru_rt_start(struct g5p_ru_retire *t, u32 settle_ms) ++{ ++ u32 hold_ms = settle_ms > G5P_RU_RT_HOLD_MS ? settle_ms : ++ G5P_RU_RT_HOLD_MS; ++ ++ *t = (struct g5p_ru_retire){ ++ .settle_ns = (u64)settle_ms * 1000000ULL, ++ .hold_ns = (u64)hold_ms * 1000000ULL, ++ }; ++} ++ ++/* The firmware's 3D completion is a type-1 event with notification bit 1 (engine 1) whose ++ * words 8/9 are the channel endpoint pair (2 seq - 1, 2 seq): 1/2, 3/4, 5/6, 7/8 for ++ * combined107 seq 1..4. The grid wraps at G5P_RU_RT_GRID, so the pair is compared modulo ++ * the grid whether the firmware reports it wrapped or as a running count: endpoint 0 ++ * (seq 8) matches 15/0 and 15/16 alike. ++ */ ++static inline bool g5p_ru_rt_event_match(const u32 words[11], u32 endpoint) ++{ ++ return words[0] == 1 && (words[1] & 2) && ++ words[9] % G5P_RU_RT_GRID == endpoint % G5P_RU_RT_GRID && ++ (words[9] - words[8]) % G5P_RU_RT_GRID == 1; ++} ++ ++/* One RAM sample of the 3D phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_rt_how); the caller passes `!= G5P_RU_RT_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_rt_step(struct g5p_ru_retire *t, u64 now_ns, ++ bool consumed, bool retired, bool idle, ++ bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ if (!idle && !t->busy_ns) ++ t->busy_ns = now_ns; ++ if (idle && t->busy_ns && !t->clear_ns) ++ t->clear_ns = now_ns; ++ t->how = G5P_RU_RT_PENDING; ++ if (!consumed || !retired) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ t->run_ns = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_RT_IDLE_CLEAR; ++ else if (event && now_ns - t->run_ns >= t->settle_ns) ++ t->how = G5P_RU_RT_IDLE_STUCK; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_RT_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_rt_note(struct g5p_ru_retire *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++/* ms from `from_ns` to `to_ns` for the receipt; -1 when either is missing or out of order. */ ++static inline s64 g5p_ru_rt_ms(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000000ULL); ++} ++ ++/* ++ * tahang1: the TA phase. mcfold1 desktop seq 251 (after T1) and seq 339 (one client): ring ++ * consumed at 2 ms, the firmware's type-1 TA completion event decoded at 7.7-8.4 ms, both ++ * stamps, the W words and the watermarks at the job and the PM state saved, yet a TA idle word ++ * (G+0x724/0x728/0x72c) set from 7-8 ms on never cleared, and the r74b predicate (which needs ++ * the idle words zero) timed out at 60 s (`unsat=0x4`). On 49 of 241 good TAs of the same ++ * boot that word was set at 7-12 ms and cleared 2 ms later. The 3D engine shows the same lag ++ * (rule (b) above). ++ * ++ * The TA phase is complete when the target is coherent (ring consumer == target and ++ * g5p_ru_retired()) with the PM state saved, and ++ * (a) the idle words are zero (the flown r74b predicate), or ++ * (b) a type-1 TA completion event was decoded after this job's TA publish, or ++ * (c) the target has stayed coherent, unbroken, for G5P_RU_TA_HOLD_MS. ++ * The stamps carry the job's identity; the TA event does not (its words are the same every ++ * job), it only reports that the firmware finished a TA after the publish. Rule (c) keeps a ++ * lost or late event from holding a job whose stamps already landed. ++ */ ++#define G5P_RU_TA_HOLD_MS 500U ++ ++enum g5p_ru_ta_how { ++ G5P_RU_TA_PENDING, ++ G5P_RU_TA_IDLE_CLEAR, /* rule (a) */ ++ G5P_RU_TA_EVENT, /* rule (b) */ ++ G5P_RU_TA_HOLD, /* rule (c) */ ++ G5P_RU_TA_HOWS, ++}; ++ ++struct g5p_ru_ta { ++ u64 hold_ns; /* rule (c) */ ++ u64 run_ns; /* first sample of the current coherent run */ ++ u64 coherent_ns; /* first coherent sample of the phase; 0 = none */ ++ u64 event_ns; /* first sample that saw the event latched; 0 = none */ ++ u32 run; /* samples in the current coherent run */ ++ u32 breaks; /* coherent runs that ended without a decision */ ++ u32 unsat; /* G5P_RU_RT_* of the last sample (SETTLE: run below the hold) */ ++ u32 how; /* enum g5p_ru_ta_how of the last sample */ ++ u32 decided; /* enum g5p_ru_ta_how of the sample that decided COMPLETE */ ++}; ++ ++static inline void g5p_ru_ta_start(struct g5p_ru_ta *t) ++{ ++ *t = (struct g5p_ru_ta){ .hold_ns = ++ (u64)G5P_RU_TA_HOLD_MS * 1000000ULL }; ++} ++ ++/* The firmware's TA completion: a type-1 event with notification bit 0 (engine 0). */ ++static inline bool g5p_ru_ta_event_match(const u32 words[11]) ++{ ++ return words[0] == 1 && (words[1] & 1); ++} ++ ++/* One RAM sample of the TA phase at now_ns. Returns the rule that completes it (enum ++ * g5p_ru_ta_how); the caller passes `!= G5P_RU_TA_PENDING` to g5p_ru_wait_step()/close(). ++ */ ++static inline u32 g5p_ru_ta_step(struct g5p_ru_ta *t, u64 now_ns, bool consumed, ++ bool retired, bool pm, bool idle, bool event) ++{ ++ t->unsat = (consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (retired ? 0 : G5P_RU_RT_RETIRED) | ++ (idle ? 0 : G5P_RU_RT_IDLE) | (event ? 0 : G5P_RU_RT_EVENT) | ++ (pm ? 0 : G5P_RU_RT_PM); ++ if (event && !t->event_ns) ++ t->event_ns = now_ns; ++ t->how = G5P_RU_TA_PENDING; ++ if (!consumed || !retired || !pm) { ++ if (t->run) ++ t->breaks++; ++ t->run = 0; ++ return t->how; ++ } ++ if (!t->coherent_ns) ++ t->coherent_ns = now_ns; ++ if (!t->run++) ++ t->run_ns = now_ns; ++ if (idle) ++ t->how = G5P_RU_TA_IDLE_CLEAR; ++ else if (event) ++ t->how = G5P_RU_TA_EVENT; ++ else if (now_ns - t->run_ns >= t->hold_ns) ++ t->how = G5P_RU_TA_HOLD; ++ else ++ t->unsat |= G5P_RU_RT_SETTLE; ++ return t->how; ++} ++ ++/* After g5p_ru_wait_step()/close() of the same sample: records which rule decided. */ ++static inline void g5p_ru_ta_note(struct g5p_ru_ta *t, u32 result) ++{ ++ if (result == G5P_RU_WAIT_COMPLETE && !t->decided) ++ t->decided = t->how; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sched.h b/drivers/gpu/apple-g5p/render-user-sched.h +new file mode 100644 +index 000000000..174a29e23 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sched.h +@@ -0,0 +1,155 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Render job scheduling across contexts (docs/gpu/multi-client-design.md, stage 2), shaped ++ * like drm_sched so that stage 4 can swap drm_sched in without a UAPI or structural change: ++ * ++ * entity = one render context (drm_sched_entity): a FIFO of its accepted jobs, and at ++ * most one close item, which runs after its last job (drm_sched_entity_destroy()); ++ * run list = the entities with work, served round-robin: each pick takes the first entity ++ * whose head job is ready and moves it to the tail (drm_sched's RR policy); ++ * ready = every dependency of the head job has signalled, it failed before the hardware, ++ * its input deadline passed, or admission closed for good (the job then fails ++ * without the hardware). Readiness is the caller's predicate; the kernel's adds ++ * dma-fence callbacks (drm_sched_job_add_dependency()) so that an unsignalled ++ * input of one client never blocks another client's jobs. ++ * ++ * One credit: the single shared TA/3D channel and PM record run one job at a time, so the ++ * caller runs each picked item to completion before the next pick. Only the head of an ++ * entity is looked at, so each context's fences signal in submission order. ++ * ++ * The lists are plain list_heads; the caller holds its scheduler lock around every call. ++ * Host-tested (scripts/gpu/multiclient2-host/test_render_sched.c). ++ */ ++#ifndef G5P_RENDER_USER_SCHED_H ++#define G5P_RENDER_USER_SCHED_H ++ ++struct g5p_rs_job { ++ struct list_head link; ++}; ++ ++struct g5p_rs_entity { ++ struct list_head link; /* in g5p_rs.run while it has work */ ++ struct list_head jobs; /* FIFO of g5p_rs_job */ ++ bool in_run, close_pending; ++}; ++ ++struct g5p_rs { ++ struct list_head run; ++}; ++ ++/* g5p_rs_pick() result */ ++enum g5p_rs_pick { ++ G5P_RS_IDLE, /* nothing ready; *wake holds the earliest deadline if any job waits */ ++ G5P_RS_JOB, /* *job was popped from *entity */ ++ G5P_RS_CLOSE, /* *entity has no job left and its close item is due */ ++}; ++ ++/* Head-job predicate: true when it may run now. *deadline is the time at which it becomes ++ * ready on its own (its input deadline); the predicate leaves it alone if there is none. ++ */ ++typedef bool (*g5p_rs_ready_fn)(struct g5p_rs_job *job, void *cookie, ++ unsigned long *deadline); ++ ++static inline void g5p_rs_init(struct g5p_rs *s) ++{ ++ INIT_LIST_HEAD(&s->run); ++} ++ ++static inline void g5p_rs_entity_init(struct g5p_rs_entity *e) ++{ ++ INIT_LIST_HEAD(&e->link); ++ INIT_LIST_HEAD(&e->jobs); ++ e->in_run = false; ++ e->close_pending = false; ++} ++ ++static inline void g5p_rs_enlist(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (!e->in_run) { ++ list_add_tail(&e->link, &s->run); ++ e->in_run = true; ++ } ++} ++ ++/* An accepted job joins the end of its entity's FIFO. */ ++static inline void g5p_rs_push(struct g5p_rs *s, struct g5p_rs_entity *e, ++ struct g5p_rs_job *job) ++{ ++ list_add_tail(&job->link, &e->jobs); ++ g5p_rs_enlist(s, e); ++} ++ ++/* The entity's close item: due once its FIFO is empty. False if one is already queued. */ ++static inline bool g5p_rs_close(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ if (e->close_pending) ++ return false; ++ e->close_pending = true; ++ g5p_rs_enlist(s, e); ++ return true; ++} ++ ++/* After the close item ran: the entity leaves the scheduler for good. */ ++static inline void g5p_rs_closed(struct g5p_rs *s, struct g5p_rs_entity *e) ++{ ++ (void)s; ++ e->close_pending = false; ++ if (e->in_run) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } ++} ++ ++/* ++ * The next item to run, round-robin over the entities with work. A picked entity moves to ++ * the tail; one left without a job and without a close item leaves the run list. Entities ++ * whose head is not ready keep their place. `has_wake` reports whether *wake is set. ++ */ ++static inline int g5p_rs_pick(struct g5p_rs *s, g5p_rs_ready_fn ready, ++ void *cookie, struct g5p_rs_entity **entity, ++ struct g5p_rs_job **job, unsigned long *wake, ++ bool *has_wake) ++{ ++ struct g5p_rs_entity *e; ++ ++ *has_wake = false; ++ list_for_each_entry(e, &s->run, link) { ++ struct g5p_rs_job *head; ++ unsigned long deadline; ++ bool timed = false; ++ ++ if (list_empty(&e->jobs)) { ++ /* Without a close item it left the run list at its last pick. */ ++ if (!e->close_pending) ++ continue; ++ *entity = e; ++ *job = NULL; ++ list_move_tail(&e->link, &s->run); ++ return G5P_RS_CLOSE; ++ } ++ head = list_first_entry(&e->jobs, struct g5p_rs_job, link); ++ deadline = 0; ++ if (!ready(head, cookie, &deadline)) { ++ timed = deadline != 0; ++ if (timed && ++ (!*has_wake || (long)(deadline - *wake) < 0)) { ++ *wake = deadline; ++ *has_wake = true; ++ } ++ continue; ++ } ++ list_del_init(&head->link); ++ *entity = e; ++ *job = head; ++ if (list_empty(&e->jobs) && !e->close_pending) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } else { ++ list_move_tail(&e->link, &s->run); ++ } ++ return G5P_RS_JOB; ++ } ++ return G5P_RS_IDLE; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-serial.h b/drivers/gpu/apple-g5p/render-user-serial.h +new file mode 100644 +index 000000000..e5cffa4e3 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-serial.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial jobs of every render context (a compositor and its clients: one TA+3D job ++ * per frame each), on the device's one TA/3D channel pair. ++ * Pure functions, host-tested (scripts/gpu/multiclient1-host/test_render_admit.c). ++ * ++ * Semantics: ++ * - jobs of all contexts run strictly one after another: the queue is FIFO, each job is ++ * TA publish -> TA complete -> 3D publish -> 3D complete before the next job starts, and ++ * there is never a TA(N+1) in flight with a 3D(N); ++ * - the epoch's first job brings the device up: PM, channels and its context's DC3 attach ++ * with the r74f/r74g bells (one-shot). Every later context binds its own slot pair at its ++ * first job with a DC3 pair published in RAM only (render-user-bind.h), no doorbell; ++ * - every job after the first only rebuilds the PM pool, builds its pages for the device's ++ * sequence number, publishes with NO doorbell and waits RAM-only; ++ * - the first job of a context that does not complete (timeout, fault, protocol error) ++ * marks that context bad: every job still queued behind it fails with -EIO without ++ * touching the hardware, and every later submit is refused (-EIO); ++ * - TYPED_V3 never mixes with the other profiles, on a context or on the device; ++ * - after any timeout or fault (render-user-terminal.h `failed`) no job starts at all, and ++ * once the epoch is quiet the device cannot be brought up (that rings a doorbell); binding ++ * another context is RAM only and stays allowed; ++ * - the sequence number is the device's, 31 bit; the channel watermark grid wraps ++ * (g5p_ru_endpoint). ++ */ ++#ifndef G5P_RENDER_USER_SERIAL_H ++#define G5P_RENDER_USER_SERIAL_H ++ ++enum g5p_ru_v3_refusal { ++ G5P_RU_V3_OK, ++ G5P_RU_V3_NO_TUPLE, /* boot tuple (r74g + r87b self-drain, no r74h/r80) absent */ ++ G5P_RU_V3_MIXED, /* another profile ran on this context or brought the device up */ ++ G5P_RU_V3_OVERFLOW, /* sequence space exhausted */ ++ G5P_RU_V3_SLOTS, /* not two distinct slots in 7..255 */ ++ G5P_RU_V3_VARIANT, /* r22/r25 variant, 3d_after_ta, timestamps or pm_policy */ ++ G5P_RU_V3_BOUND, /* per-phase bound outside 20000..30000 ms */ ++ G5P_RU_V3_STATE, /* executor only: bind state and job counts disagree */ ++ G5P_RU_V3_TERMINAL, /* terminal epoch: timeout/fault, or quiet and the device down */ ++ G5P_RU_V3_REFUSALS, ++}; ++ ++static const char *const g5p_ru_v3_why[G5P_RU_V3_REFUSALS] = { ++ "", ++ "transport is not ready for TYPED_V3 rendering", ++ "another profile ran on this context or brought the device up", ++ "sequence space exhausted", ++ "slots are not two distinct slots in 7..255", ++ "an r22/r25 variant, 3d_after_ta, timestamps or pm_policy is latched", ++ "per-phase bound outside 20000..30000 ms", ++ "bind state and job counts disagree", ++ "terminal epoch (timeout or fault, or quiet and the device not up): no hardware access", ++}; ++ ++/* Everything the admission needs, as plain values. */ ++struct g5p_ru_v3_state { ++ bool tuple; /* boot tuple present */ ++ bool v3_accepted; /* a V3 job was accepted on this context */ ++ bool bound; /* this context's slot pair was published */ ++ u32 jobs; /* jobs that started on this context */ ++ bool device_up; /* the epoch's first job built PM and channels */ ++ bool device_v3; /* ... through the TYPED_V3 attach */ ++ u32 sequence; /* the device's last job that started (0 = none) */ ++ u32 slot_ta, slot_3d; ++ bool variant; ++ u32 wait_ms; ++ bool quiet; /* a terminal note was made: mmio_ok is false */ ++ bool terminal_failed; /* ... and it was, or a later one is, a timeout or fault */ ++}; ++ ++/* Any two distinct user slots: unbinds leave holes, so a pair need not be adjacent. */ ++static inline bool g5p_ru_v3_slots_valid(u32 slot_ta, u32 slot_3d) ++{ ++ return slot_ta >= 7 && slot_3d >= 7 && slot_ta <= 255 && ++ slot_3d <= 255 && slot_ta != slot_3d; ++} ++ ++#define G5P_RU_V3_SEQ_MAX 0x7fffffffU ++ ++/* At submit time and again in the executor (`executing`): a refusal here touched nothing. */ ++static inline int g5p_ru_v3_admit(const struct g5p_ru_v3_state *s, ++ bool executing) ++{ ++ if (!s->tuple) ++ return G5P_RU_V3_NO_TUPLE; ++ /* After any timeout or fault nothing new starts (closes the window between a failing job's ++ * unlock and the context being marked bad). Once quiet, the device cannot be brought up, ++ * because that rings a doorbell; a job on an up device publishes with no doorbell and ++ * touches no register, and so does the bind of a new context. ++ */ ++ if (s->terminal_failed || (s->quiet && !s->device_up)) ++ return G5P_RU_V3_TERMINAL; ++ /* A job of another profile ran on this context, or another profile brought the device ++ * up: only V3 may follow V3. ++ */ ++ if ((s->jobs && !s->v3_accepted) || (s->device_up && !s->device_v3)) ++ return G5P_RU_V3_MIXED; ++ if (s->sequence >= G5P_RU_V3_SEQ_MAX) ++ return G5P_RU_V3_OVERFLOW; ++ if (!g5p_ru_v3_slots_valid(s->slot_ta, s->slot_3d)) ++ return G5P_RU_V3_SLOTS; ++ if (s->variant) ++ return G5P_RU_V3_VARIANT; ++ if (!g5p_ru_v3_bound_valid(s->wait_ms)) ++ return G5P_RU_V3_BOUND; ++ /* The executor: the device is up exactly when a job has started on it, and a context is ++ * bound exactly when it has run a job, always on an up device. ++ */ ++ if (executing && (s->device_up != !!s->sequence || ++ s->bound != !!s->jobs || (s->bound && !s->device_up))) ++ return G5P_RU_V3_STATE; ++ return G5P_RU_V3_OK; ++} ++ ++/* Verbose receipts for each context's first jobs and every failure; a compositor prints no ++ * line per frame. ++ */ ++static inline bool g5p_ru_v3_verbose(u32 jobs, int ret) ++{ ++ return jobs <= 8 || ret; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-sync.h b/drivers/gpu/apple-g5p/render-user-sync.h +new file mode 100644 +index 000000000..d212d3199 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-sync.h +@@ -0,0 +1,50 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * CPU cache maintenance by range (perf60-fillrate.md §11.3 cut 5, kernel half). Host test: ++ * scripts/gpu/perf60-host/test_sync_range.c. ++ * ++ * render_user_sync_range bit 0: BO_SYNC honours offset/size (already validated page aligned by ++ * g5p_bo_range_valid) instead of syncing the whole BO. Bit 1: a job's ACCESS_CACHE maintenance ++ * (clean before, invalidate after retirement) covers the resource's BO range [offset, offset + ++ * size), the bytes the job can reach through that resource, instead of the whole BO. The window ++ * is widened to whole GPU pages and clamped to the BO. Off: the whole BO, as release8. ++ * The window is walked over the sg_table's DMA segments in order (their lengths concatenate to ++ * the BO's bytes); each segment gets the intersection, relative to the segment. ++ */ ++#ifndef G5P_RENDER_USER_SYNC_H ++#define G5P_RENDER_USER_SYNC_H ++ ++#define G5P_SYNC_ALIGN 0x4000ULL ++ ++/* ++ * [lo, hi) = [off, off + size) widened to pages, clamped to the BO; false if empty or outside. ++ */ ++static inline bool g5p_sync_window(u64 off, u64 size, u64 bo_size, u64 *lo, ++ u64 *hi) ++{ ++ u64 end; ++ ++ if (!size || off >= bo_size) ++ return false; ++ end = size > bo_size - off ? bo_size : off + size; ++ *lo = off & ~(G5P_SYNC_ALIGN - 1); ++ *hi = (end + G5P_SYNC_ALIGN - 1) & ~(G5P_SYNC_ALIGN - 1); ++ if (*hi > bo_size) ++ *hi = bo_size; ++ return *hi > *lo; ++} ++ ++/* The part of segment [pos, pos + len) inside [lo, hi): *start relative to the segment. */ ++static inline bool g5p_sync_clip(u64 pos, u64 len, u64 lo, u64 hi, u64 *start, ++ u64 *n) ++{ ++ u64 a = pos > lo ? pos : lo, b = pos + len < hi ? pos + len : hi; ++ ++ if (b <= a) ++ return false; ++ *start = a - pos; ++ *n = b - a; ++ return true; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-template.h b/drivers/gpu/apple-g5p/render-user-template.h +new file mode 100644 +index 000000000..92bd1604a +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-template.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Generated by make-render-user-template.py from the hash-pinned r22 importer. */ ++#ifndef G5P_RENDER_USER_TEMPLATE_H ++#define G5P_RENDER_USER_TEMPLATE_H ++ ++/* Frozen dialect-1 W scalars {offset, value}: color-only, one sample, ++ * normal 3D, no depth/resource arrays. Typed and kernel fields excluded. ++ */ ++static const u32 g5p_ru_ta_fixed[][2] = { ++ { 0x020, 0x1e3ce508 }, { 0x024, 0x1e3ce508 }, { 0x3a4, 0x00000088 }, ++ { 0x3a8, 0x00000002 }, { 0x3c0, 0x00020000 }, { 0x514, 0x08002358 }, ++ { 0x528, 0x00000001 }, ++}; ++ ++static const u32 g5p_ru_3d_fixed[][2] = { ++ { 0x108, 0x000003c3 }, { 0x110, 0x000003c3 }, { 0x118, 0x000003c3 }, ++ { 0x120, 0x000003c3 }, { 0x128, 0x000003c3 }, { 0x130, 0x000003c3 }, ++ { 0x3a4, 0x3f800000 }, { 0x3a8, 0x00000300 }, { 0x3b0, 0x0000001f }, ++ { 0x3c0, 0x0023000f }, { 0x3c4, 0x00000408 }, { 0x3c8, 0x00000008 }, ++ { 0x3cc, 0x000000f8 }, { 0x6a8, 0x3f800000 }, { 0x6ac, 0x00000300 }, ++ { 0x6b0, 0x00000002 }, { 0x868, 0x08002358 }, ++}; ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-terminal.h b/drivers/gpu/apple-g5p/render-user-terminal.h +new file mode 100644 +index 000000000..4ced26eaf +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-terminal.h +@@ -0,0 +1,103 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 terminal state of the epoch. Pure functions over one small struct; the ++ * kernel supplies READ_ONCE/WRITE_ONCE/cmpxchg and the host suite shims them, so the ++ * host compiles this file as it is. ++ * ++ * One latch, first writer wins. The first terminal event of the epoch (the natural TA ++ * complete, a timeout, a fault or a protocol error) fixes `kind`; from then on `quiet` ++ * is set and never cleared, and g5p_ru_mmio_ok() is false: the host touches no register ++ * (R0, R1, AIC, doorbell) and no diagnostic bank. RAM stays accessible. A later ++ * failure (the 3D after a completed TA, job N+1 after job N) cannot change `kind` but ++ * sets the sticky `failed`, which is what the admission uses: after any timeout or fault ++ * no new job starts, from any queue, even one that passed the pre-lock context checks ++ * before the failing job marked the context bad. ++ * ++ * The recovery series (patches/gpu-recovery) is independent of this file. Its ++ * g5p_td_note_terminal() takes {1 complete, 2 timeout}; g5p_ru_td_kind() maps ours. ++ */ ++#ifndef G5P_RENDER_USER_TERMINAL_H ++#define G5P_RENDER_USER_TERMINAL_H ++ ++enum g5p_ru_term_kind { ++ G5P_RU_TERM_NONE, ++ G5P_RU_TERM_COMPLETE, /* the natural TA completed (RAM) */ ++ G5P_RU_TERM_TIMEOUT, /* a phase ran out its deadline */ ++ G5P_RU_TERM_FAULT, /* protocol error, drain error, attach or publish failure */ ++ G5P_RU_TERM_KINDS, ++}; ++ ++static const char *const g5p_ru_term_names[G5P_RU_TERM_KINDS] = { ++ "none", ++ "complete", ++ "timeout", ++ "fault", ++}; ++ ++struct g5p_ru_terminal { ++ int kind; /* first writer wins */ ++ int quiet; /* set with the first note, never cleared */ ++ int failed; /* sticky: any timeout or fault was noted */ ++ int captured; /* the CAPTURE entry (recovery hook) was taken, once */ ++}; ++ ++/* A new firmware epoch (the in-boot recovery, recover.inc) starts a new latch; nothing else ++ * clears it. ++ */ ++static inline void g5p_ru_term_reset(struct g5p_ru_terminal *t) ++{ ++ WRITE_ONCE(t->kind, 0); ++ WRITE_ONCE(t->quiet, 0); ++ WRITE_ONCE(t->failed, 0); ++ WRITE_ONCE(t->captured, 0); ++} ++ ++/* The kind a job's return value stands for (only called for ret < 0, ret != -ECANCELED). */ ++static inline int g5p_ru_term_from_ret(int ret) ++{ ++ return ret == -ETIMEDOUT ? G5P_RU_TERM_TIMEOUT : G5P_RU_TERM_FAULT; ++} ++ ++/* Returns true for the first note of the epoch. */ ++static inline bool g5p_ru_term_note(struct g5p_ru_terminal *t, int kind) ++{ ++ bool first; ++ ++ if (kind <= G5P_RU_TERM_NONE || kind >= G5P_RU_TERM_KINDS) ++ kind = G5P_RU_TERM_FAULT; ++ first = cmpxchg(&t->kind, G5P_RU_TERM_NONE, kind) == G5P_RU_TERM_NONE; ++ if (kind >= G5P_RU_TERM_TIMEOUT) ++ WRITE_ONCE(t->failed, 1); ++ WRITE_ONCE(t->quiet, 1); ++ return first; ++} ++ ++static inline int g5p_ru_term_kind(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->kind); ++} ++ ++static inline bool g5p_ru_term_failed(const struct g5p_ru_terminal *t) ++{ ++ return READ_ONCE(t->failed); ++} ++ ++/* The single predicate for register access: false from the first terminal note on. */ ++static inline bool g5p_ru_mmio_ok(const struct g5p_ru_terminal *t) ++{ ++ return !READ_ONCE(t->quiet); ++} ++ ++/* True once per epoch for a timeout or fault: the caller may take the recovery CAPTURE entry. */ ++static inline bool g5p_ru_term_capture_due(struct g5p_ru_terminal *t, int kind) ++{ ++ return kind >= G5P_RU_TERM_TIMEOUT && cmpxchg(&t->captured, 0, 1) == 0; ++} ++ ++/* Recovery's terminal encoding: 1 complete, 2 timeout (a fault is a no-completion exit: 2). */ ++static inline int g5p_ru_td_kind(int kind) ++{ ++ return kind == G5P_RU_TERM_COMPLETE ? 1 : 2; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-touch.h b/drivers/gpu/apple-g5p/render-user-touch.h +new file mode 100644 +index 000000000..145bdbf2d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-touch.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 RAM-touch latency A/B (release). Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. Include render-user-wait.h first. ++ * ++ * Question (docs/gpu/experiments/latency101-device-analysis.md section 9): is the host's bulk read ++ * of the receipt pages what releases a stalled phase (the +6 ms atom after the job-1 receipt)? ++ * ++ * render_user_touch_ms = T, default 0 = off. On, every phase of every job records the first wait ++ * sample at or after publish + T that left the phase undecided (still pending, deadline not ++ * reached). Odd sequences are the touch arm: right after that sample the host reads the ++ * receipt's read set once (RAM only, folded into a sum). Even sequences are the control arm: the ++ * same instant is recorded and nothing is read. Neither arm changes a sample, the sleep, the ++ * deadline, a latch or the result, and neither prints before the job ends. ++ * ++ * Off: g5p_ru_touch_start() leaves the record zero and g5p_ru_touch_due() is never true, so the ++ * wait loop runs the latency101 instructions unchanged. ++ */ ++#ifndef G5P_RENDER_USER_TOUCH_H ++#define G5P_RENDER_USER_TOUCH_H ++ ++#define G5P_RU_TOUCH_MAX_MS \ ++ 20000U /* never at or past the smallest soft bound */ ++ ++struct g5p_ru_touch { ++ u64 at_ns; /* publish + T; 0 = off for this phase */ ++ u64 seen_ns; /* the undecided sample at or after at_ns that took it; 0 = not reached */ ++ u64 done_ns; /* end of the read (touch arm), seen_ns (control arm) */ ++ u32 bytes; /* bytes read: 0 on the control arm */ ++ u32 sum; /* fold of the words read: the loads cannot be elided */ ++ bool sweep; /* this phase is in the touch arm */ ++}; ++ ++/* 0 (off), or 1..G5P_RU_TOUCH_MAX_MS with latency101 on (hard_ms != 0) and strictly before the ++ * effective soft bound (g5p_ru_lat_soft()), so the instant always falls in the plain wait. ++ */ ++static inline bool g5p_ru_touch_valid(u32 touch_ms, u32 soft_ms, u32 hard_ms) ++{ ++ if (!touch_ms) ++ return true; ++ return hard_ms && touch_ms <= G5P_RU_TOUCH_MAX_MS && touch_ms < soft_ms; ++} ++ ++/* Odd sequences are the touch arm, even ones the control. */ ++static inline bool g5p_ru_touch_arm(u32 seq) ++{ ++ return seq & 1; ++} ++ ++static inline void g5p_ru_touch_start(struct g5p_ru_touch *t, u64 start_ns, ++ u32 touch_ms, u32 seq) ++{ ++ *t = (struct g5p_ru_touch){ 0 }; ++ if (!touch_ms) ++ return; ++ t->at_ns = start_ns + (u64)touch_ms * 1000000ULL; ++ t->sweep = g5p_ru_touch_arm(seq); ++} ++ ++/* ++ * Asked after g5p_ru_wait_step() of the sample taken at now_ns: true once, for the first sample ++ * at ++ * or after the instant whose step left the phase undecided and not expired. A completion, a fault ++ * or the deadline in the same sample wins, and then the instant is never taken. ++ */ ++static inline bool g5p_ru_touch_due(const struct g5p_ru_touch *t, u64 now_ns, ++ u32 result, bool expired) ++{ ++ return t->at_ns && !t->seen_ns && !result && !expired && ++ now_ns >= t->at_ns; ++} ++ ++/* The instant was taken at seen_ns (a sample time, never 0); done_ns is when the read ended. */ ++static inline void g5p_ru_touch_mark(struct g5p_ru_touch *t, u64 seen_ns, ++ u64 done_ns, u32 bytes, u32 sum) ++{ ++ t->seen_ns = seen_ns; ++ t->done_ns = done_ns < seen_ns ? seen_ns : done_ns; ++ t->bytes = bytes; ++ t->sum = sum; ++} ++ ++/* Receipt helper: to - from in microseconds, -1 when an end is missing or the order is wrong. */ ++static inline s64 g5p_ru_touch_us(u64 from_ns, u64 to_ns) ++{ ++ if (!from_ns || !to_ns || to_ns < from_ns) ++ return -1; ++ return (s64)((to_ns - from_ns) / 1000); ++} ++ ++/* Receipt: how long after the instant (the end of the read) the phase's deciding sample saw it ++ * complete; -1 when the instant was not taken or the phase did not complete. ++ */ ++static inline s64 g5p_ru_touch_after_us(const struct g5p_ru_touch *t, ++ u32 result, u64 end_ns) ++{ ++ if (!t->seen_ns || result != G5P_RU_WAIT_COMPLETE) ++ return -1; ++ return g5p_ru_touch_us(t->done_ns, end_ns); ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wait.h b/drivers/gpu/apple-g5p/render-user-wait.h +new file mode 100644 +index 000000000..b9b5f1c49 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wait.h +@@ -0,0 +1,266 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial wait policy. Pure functions, no kernel services beyond the ++ * integer types, so the host suite compiles this file as it is. ++ * ++ * Evidence behind the numbers (r96 raw timelines, 2026-10-30): ++ * - the natural TA completion event arrived +12.0060 s and +12.0062 s after the ++ * publish in the two boots that a 12000 ms cap turned into timeouts; the whole ++ * delay is inside the firmware's start23, and once it returns the hardware and ++ * the retirement take less than ~1 ms; ++ * - nothing the host does shortens the delay, so the wait is RAM-only: after the ++ * DC3 attach the host touches no register until the job has ended. ++ * ++ * Policy: ++ * - one phase (TA, then 3D) waits at most `bound` from its own publish: 20 s by ++ * default, never below 20 s, never above the absolute cap; ++ * - the whole job (TA publish to the end of the 3D phase) never exceeds the ++ * absolute cap of 30 s; ++ * - the deadline is decided on RAM samples. A completion seen in the sample that ++ * reaches the deadline wins; a deadline is only final after one closing sample ++ * taken at or after it, which closes the gap between the last poll and the ++ * expiry; a completion after that is evidence only (`late_after`), never the ++ * result; ++ * - a fault wins over a completion in the same sample and over the deadline. ++ */ ++#ifndef G5P_RENDER_USER_WAIT_H ++#define G5P_RENDER_USER_WAIT_H ++ ++#define G5P_RU_V3_WAIT_MS 20000U /* default and minimum per-phase bound */ ++#define G5P_RU_V3_CAP_MS \ ++ 30000U /* absolute job bound, and the largest per-phase bound */ ++ ++/* ++ * latency101 (default off: render_user_soft_ms = render_user_hard_ms = 0 leaves every number above, ++ * and the behaviour of every function below, exactly as it was). When on, the per-phase deadline ++ * is the HARD bound and the job bound is two hard bounds; the SOFT bound marks a phase "slow" ++ * (RAM-only record, no print, no latch). A silent event stream means a dead firmware from the ++ * phase's publish on (faultiso; before it, only from the soft bound). See ++ * docs/gpu/experiments/prod98c-bimodal-latency.md and docs/gpu/experiments/faultiso.md. ++ */ ++#define G5P_RU_LAT_MIN_SOFT_MS \ ++ 20000U /* never below the proven per-phase bound */ ++#define G5P_RU_LAT_DEF_SOFT_MS 45000U ++#define G5P_RU_LAT_MAX_HARD_MS \ ++ 60000U /* largest hard bound; the job bound is twice this */ ++#define G5P_RU_LAT_DEF_OBS_MS \ ++ 2000U /* RAM-only look after a timeout, pre-receipt */ ++#define G5P_RU_LAT_MAX_OBS_MS 5000U ++#define G5P_RU_LAT_SILENT_MS \ ++ 1000U /* no event decoded for this long in a phase = silent */ ++/* The silence clock runs from the publish (the host suites key on this). */ ++#define G5P_RU_LAT_SILENT_FROM_PUBLISH 1 ++ ++enum g5p_ru_wait_result { ++ G5P_RU_WAIT_PENDING, ++ G5P_RU_WAIT_COMPLETE, ++ G5P_RU_WAIT_TIMEOUT, ++ G5P_RU_WAIT_FAULT, ++}; ++ ++struct g5p_ru_wait { ++ u64 start_ns; /* the publish (producer store) of this phase */ ++ u64 deadline_ns; /* start + bound, clamped to the job's absolute deadline */ ++ u64 end_ns; /* the sample that decided */ ++ u32 result; ++ bool expired; /* a sample at or after the deadline has been seen */ ++ bool late_gap; /* completion first seen by the closing sample */ ++ bool late_after; /* completion seen after a timeout was decided: evidence only */ ++ /* latency101; all zero (and never read) unless soft_ns != 0 */ ++ u64 soft_ns; /* start + soft bound; 0 = the grace is off */ ++ u64 soft_seen_ns; /* the first sample at or after soft_ns */ ++ u64 late_ns; /* the first retired sample after the timeout was decided */ ++ u64 event_ns; /* sample time of the last change of the decoded-event counter */ ++ u64 busy_ns; /* the first sample whose idle words showed the engine busy */ ++ u32 events; /* the decoded-event counter at that sample */ ++ bool slow; /* the phase passed its soft bound and is still pending */ ++ bool silent; /* the event stream stopped during the phase: firmware dead */ ++ bool late_soft; /* completed after the soft bound: a success, never a failure */ ++}; ++ ++/* latency101 parameters. Off is hard = soft = 0. On: hard in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000. The job bound the waiters must outlast is two ++ * hard bounds plus one observation window. ++ */ ++static inline u32 g5p_ru_lat_soft(u32 soft_ms, u32 hard_ms) ++{ ++ if (soft_ms) ++ return soft_ms; ++ return hard_ms > G5P_RU_LAT_DEF_SOFT_MS ? G5P_RU_LAT_DEF_SOFT_MS : ++ hard_ms; ++} ++ ++static inline bool g5p_ru_lat_valid(u32 soft_ms, u32 hard_ms, u32 obs_ms) ++{ ++ if (!hard_ms) ++ return !soft_ms; ++ soft_ms = g5p_ru_lat_soft(soft_ms, hard_ms); ++ return hard_ms <= G5P_RU_LAT_MAX_HARD_MS && ++ soft_ms >= G5P_RU_LAT_MIN_SOFT_MS && soft_ms <= hard_ms && ++ obs_ms <= G5P_RU_LAT_MAX_OBS_MS; ++} ++ ++static inline u32 g5p_ru_lat_job_ms(u32 hard_ms, u32 obs_ms) ++{ ++ return 2 * hard_ms + obs_ms; ++} ++ ++/* A per-phase bound is valid when it is in [G5P_RU_V3_WAIT_MS, G5P_RU_V3_CAP_MS]. */ ++static inline bool g5p_ru_v3_bound_valid(u32 ms) ++{ ++ return ms >= G5P_RU_V3_WAIT_MS && ms <= G5P_RU_V3_CAP_MS; ++} ++ ++/* The deadline of a phase: its own bound, clamped to the job's absolute deadline. ++ * A job deadline of 0 means "no job deadline yet" (the first phase). ++ */ ++static inline u64 g5p_ru_wait_deadline(u64 start_ns, u32 bound_ms, ++ u64 job_deadline_ns) ++{ ++ u64 d = start_ns + (u64)bound_ms * 1000000ULL; ++ ++ return job_deadline_ns && job_deadline_ns < d ? job_deadline_ns : d; ++} ++ ++static inline void g5p_ru_wait_start(struct g5p_ru_wait *w, u64 start_ns, ++ u32 bound_ms, u64 job_deadline_ns) ++{ ++ *w = (struct g5p_ru_wait){ ++ .start_ns = start_ns, ++ .deadline_ns = g5p_ru_wait_deadline(start_ns, bound_ms, ++ job_deadline_ns), ++ }; ++} ++ ++/* ++ * latency101: hard_ms is the per-phase deadline (clamped to the job's deadline as above), ++ * soft_ms ++ * (never past the deadline) the point where a still-pending phase is called slow. `events` seeds ++ * the silence clock. ++ */ ++static inline void g5p_ru_wait_start_lat(struct g5p_ru_wait *w, u64 start_ns, ++ u32 soft_ms, u32 hard_ms, ++ u64 job_deadline_ns, u32 events) ++{ ++ g5p_ru_wait_start(w, start_ns, hard_ms, job_deadline_ns); ++ w->soft_ns = start_ns + (u64)soft_ms * 1000000ULL; ++ if (w->soft_ns > w->deadline_ns) ++ w->soft_ns = w->deadline_ns; ++ w->events = events; ++ w->event_ns = start_ns; ++} ++ ++/* The first sample whose idle words (G+0x724..) show the engine busy, i.e. the firmware's ++ * start path has run. Recorded once; it splits a late start (busy_ns close to the ++ * completion) from slow hardware (busy_ns early). A busy window shorter than the poll ++ * period can be missed: 0 = not seen. ++ */ ++static inline void g5p_ru_wait_busy(struct g5p_ru_wait *w, u64 now_ns, ++ bool busy) ++{ ++ if (busy && !w->busy_ns) ++ w->busy_ns = now_ns; ++} ++ ++/* One look at the decoded-event counter, taken with every RAM sample, before g5p_ru_wait_step(). ++ * The ALIVE stream runs at about 290 events/s whatever the work does (r96c: 29442 polls, no gap ++ * over 3.4 ms; latency101: 287 events/s through every stall of 0.55-33 s), so a stream that has ++ * been silent for G5P_RU_LAT_SILENT_MS is a dead firmware, not a slow one, at any point of the ++ * phase. Returns true then; the caller reports it as a fault. The silence clock starts at the ++ * publish (vpds rung vk, 2026-11-01: the counter froze 1 s after the TA's publish, and a clock ++ * that only ran from the soft bound called it at 45 s, with every other client held behind it). ++ * The caller drains the ring before each sample, so a host that slept longer than the window ++ * sees the events it missed and is not silent. Never true when latency101 is off. ++ */ ++static inline bool g5p_ru_wait_grace(struct g5p_ru_wait *w, u64 now_ns, ++ u32 events) ++{ ++ if (!w->soft_ns) ++ return false; ++ if (events != w->events) { ++ w->events = events; ++ w->event_ns = now_ns; ++ } ++ if (now_ns >= w->soft_ns && !w->soft_seen_ns) ++ w->soft_seen_ns = now_ns; ++ w->silent = now_ns - w->event_ns >= ++ (u64)G5P_RU_LAT_SILENT_MS * 1000000ULL; ++ return w->silent; ++} ++ ++/* One RAM sample taken at `now_ns`. Returns the result so far (PENDING until ++ * decided). A sample at or after the deadline only marks the wait expired unless ++ * it shows completion or a fault; the caller must then take one closing sample and ++ * call g5p_ru_wait_close(). ++ */ ++static inline u32 g5p_ru_wait_step(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (w->soft_ns && now_ns >= w->soft_ns) ++ w->slow = true; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ w->end_ns = now_ns; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->end_ns = now_ns; ++ w->late_soft = w->slow; ++ } else if (now_ns >= w->deadline_ns) { ++ w->expired = true; ++ } ++ return w->result; ++} ++ ++/* The closing sample, taken after the loop left because of the deadline. This is ++ * the only place a timeout is decided. ++ */ ++static inline u32 g5p_ru_wait_close(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, bool fault) ++{ ++ if (w->result) ++ return w->result; ++ if (fault) { ++ w->result = G5P_RU_WAIT_FAULT; ++ } else if (complete) { ++ w->result = G5P_RU_WAIT_COMPLETE; ++ w->late_gap = true; ++ w->late_soft = w->slow; ++ } else { ++ w->result = G5P_RU_WAIT_TIMEOUT; ++ } ++ w->end_ns = now_ns; ++ return w->result; ++} ++ ++/* A sample after the result was decided: a completion after a timeout is recorded ++ * and nothing else happens (no 3D, no recovery, no register access). ++ */ ++static inline void g5p_ru_wait_after(struct g5p_ru_wait *w, bool complete) ++{ ++ if (w->result == G5P_RU_WAIT_TIMEOUT && complete) ++ w->late_after = true; ++} ++ ++/* latency101: the RAM-only observation window after a timeout was decided, before the receipt is ++ * printed and the terminal latch is taken. g5p_ru_wait_after() looks once, microseconds after the ++ * decision; a completion that arrives milliseconds later (prod98c seq 3: 5.49 ms) was never ++ * recorded, so `late_after=0/0` was a false negative. Returns true while the window is open. A ++ * retired sample ends it and is recorded in late_after/late_ns; it never changes the result (a ++ * deadline stays final). obs_ms = 0 closes it at once (the legacy behaviour). ++ */ ++static inline bool g5p_ru_wait_observe(struct g5p_ru_wait *w, u64 now_ns, ++ bool complete, u32 obs_ms) ++{ ++ if (w->result != G5P_RU_WAIT_TIMEOUT || w->late_after) ++ return false; ++ if (complete) { ++ w->late_after = true; ++ w->late_ns = now_ns; ++ return false; ++ } ++ return now_ns < w->end_ns + (u64)obs_ms * 1000000ULL; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +new file mode 100644 +index 000000000..64a51eaf4 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -0,0 +1,273 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 publish-time MTS wake (wake110). Pure functions over plain values; the one ++ * register sequence goes through a callback table (the fender.h pattern), so the host ++ * suite compiles this file as it is and records every access in order. ++ * ++ * Question (docs/gpu/experiments/wake110-apple-mts-dispatch.md): is the 0.5-33 s TA/3D ++ * start latency caused by the missing Apple submit kick? Apple's TA and 3D submit ++ * wrappers (iPad6,11 kernelcache 0xfffffff006ca00a0 / 0xfffffff006ca01f8) do, every time: ++ * ring entry, dmb ish, Fender vt+0x80 (R1+0x14 = 7, read back), Fender vt+0x18 ++ * (R1+0x1000 & 0xf), then R0+0xb00 = 0x10 (TA, DM0|COUNTED) or 0x11 (3D, DM1|COUNTED). ++ * The MTS store is reached from every arm; when the GPC nibble is 0 Apple first runs a ++ * power arm (up to forcePowerCycleAndReboot). Linux V3 publishes job N >= 2's TA and ++ * every 3D with no kick at all. ++ * ++ * render_user_wake (latched per context): 0 off (default: no call has any effect), ++ * 1 the A/B, 2 every eligible phase is a factor (the candidate production fix, for use ++ * only after the A/B proved it). The TA of job 1 is never a phase of this file (the r74g ++ * grace and its fenced fallback bell own it). In the A/B each engine's phases are paired ++ * in publish order, (0,1), (2,3), ...; exactly one of each pair is the factor, which one ++ * is fixed by render_user_wake_seed before the context exists, and bit 31 of the seed ++ * inverts the whole schedule (the mirror stage). The verdict recomputes the arm of every ++ * receipt from the seed. ++ * ++ * At the publish, right after the producer store and its barrier, both arms take the same ++ * lock and read the same RAM state and run the same guards; the control stops there (no ++ * register). The factor then runs the Apple order: R1+0x14 = 7, read back (any bit ++ * outside 7: stop), R1+0x1000 (state & 0x1f != 1, the GPC is not active: stop, never ++ * repaired, never a power arm), dma_wmb(), R0+0xb00 = 0x10 | engine. At most one attempt ++ * per phase. Never after a timeout or fault (the terminal latch's sticky `failed`), never ++ * with the boot failed or stopped, never unpowered, never with an r74f bell in flight, ++ * never once the phase shows any sign of retirement, never on a ring that is not exactly ++ * this publish. A consumer already at the target does not cancel (Apple never checks ++ * it); it is recorded. The natural TA completion (terminal kind COMPLETE, the epoch's ++ * first note) is not a failure: the factor's sequence is the one licensed exception to the ++ * r96 "no register after the TA complete" envelope, default off. ++ */ ++#ifndef G5P_RENDER_USER_WAKE_H ++#define G5P_RENDER_USER_WAKE_H ++ ++#ifndef G5P_GPC_ON ++#define G5P_GPC_ON(raw) (((raw) & 0x1f) == 1) /* core.inc overrides it */ ++#endif ++ ++#define G5P_WK_OFF 0U ++#define G5P_WK_AB 1U ++#define G5P_WK_ALWAYS 2U ++#define G5P_WK_MIRROR 0x80000000U /* seed bit 31: every arm inverted */ ++ ++#define G5P_WK_R1_FENDER 0x14U /* Fender vt+0x80: write 7, read back */ ++#define G5P_WK_R1_GPC 0x1000U /* GPC state: & 0x1f == 1 is active */ ++#define G5P_WK_R0_MTS 0xb00U /* MTS_SCHEDULE, write only */ ++#define G5P_WK_MTS_TA 0x10U /* DM0 | COUNTED */ ++#define G5P_WK_MTS_3D 0x11U /* DM1 | COUNTED */ ++ ++/* wake111: the publish-time power evidence, one bit per RAM fact, sampled under the event lock ++ * and printed per phase (power=TT/DD). Every required bit must be set. reset_transition is ++ * the permanent post-release slave-port lockout (set right before the release, never ++ * cleared), not a live reset state: Wake110 vetoed on it, so every phase skipped. It is ++ * recorded (SLAVE_LOCK) and is neither required nor a veto; the slave-port ban is unchanged. ++ */ ++#define G5P_WK_PWR_CORE 0x01U /* g->powered */ ++#define G5P_WK_PWR_FENDER 0x02U /* g->fender_ready */ ++#define G5P_WK_PWR_RELEASED 0x04U /* b->released */ ++#define G5P_WK_PWR_ALIVE 0x08U /* b->alive */ ++#define G5P_WK_PWR_CONTROL 0x10U /* b->control_complete */ ++#define G5P_WK_PWR_DS108 \ ++ 0x20U /* DS108 claim proof, current generation, no power abort */ ++#define G5P_WK_PWR_SLAVE_LOCK 0x40U /* g->reset_transition: evidence only */ ++#define G5P_WK_PWR_REQUIRED \ ++ (G5P_WK_PWR_CORE | G5P_WK_PWR_FENDER | G5P_WK_PWR_RELEASED | \ ++ G5P_WK_PWR_ALIVE | G5P_WK_PWR_CONTROL | G5P_WK_PWR_DS108) ++ ++enum g5p_wk_arm { ++ G5P_WK_ARM_NONE, /* wake off */ ++ G5P_WK_ARM_EXCLUDED, /* the TA of job 1 (r74g) */ ++ G5P_WK_ARM_CONTROL, ++ G5P_WK_ARM_FACTOR, ++ G5P_WK_ARMS, ++}; ++ ++enum g5p_wk_did { ++ G5P_WK_DID_NONE, /* off, or never reached */ ++ G5P_WK_DID_RANG, /* factor: fence, GPC active, 0xb00 written */ ++ G5P_WK_DID_SHAM, /* control: every guard passed, no register */ ++ G5P_WK_DID_EXCLUDED, ++ G5P_WK_SKIP_TWICE, /* this phase was already decided */ ++ G5P_WK_SKIP_LICENSE, /* a timeout or fault was noted (terminal failed) */ ++ G5P_WK_SKIP_FAULT, /* boot failed, event failed or stopped */ ++ G5P_WK_SKIP_POWER, /* one of G5P_WK_PWR_REQUIRED is absent */ ++ G5P_WK_SKIP_R74F, /* an r74f bell is published, open or firing */ ++ G5P_WK_SKIP_RING, /* the ring is not exactly this publish */ ++ G5P_WK_SKIP_RETIRED, /* a stamp == seq or a completion event since the publish */ ++ G5P_WK_SKIP_FENCE, /* R1+0x14 read back outside 7: no further register */ ++ G5P_WK_SKIP_GPC, /* GPC not active: no 0xb00 */ ++ G5P_WK_DIDS, ++}; ++ ++static const char *const g5p_wk_arm_names[G5P_WK_ARMS] = { ++ "none", ++ "excluded", ++ "control", ++ "factor", ++}; ++ ++static const char *const g5p_wk_did_names[G5P_WK_DIDS] = { ++ "none", "rang", "sham", "excluded", "skip-twice", ++ "skip-license", "skip-fault", "skip-power", "skip-r74f", "skip-ring", ++ "skip-retired", "skip-fence", "skip-gpc", ++}; ++ ++/* Everything the decision reads, sampled under the event lock right after the producer store. */ ++struct g5p_wk_in { ++ u32 mode, seed, seq, e; ++ bool attempted; /* this phase already went through g5p_wk_decide() */ ++ bool term_failed; /* g5p_ru_term_failed() */ ++ bool boot_failed; /* b->failed || b->event_failed || b->stopped */ ++ u32 power; /* g5p_wk_power_mask(); the slave lock is evidence, not a veto */ ++ bool r74f_pending; /* an r74f bell in PUBLISHED, OPEN or FIRING */ ++ u32 start, target, prod, ++ cons; /* ring: producer before this publish, our target, now */ ++ u32 stamp[2]; /* this engine's stamp words */ ++ u32 type1; /* this engine's completion events decoded since the phase armed */ ++}; ++ ++/* The register access of the factor, in order. The kernel binds them to writel/readl on the ++ * Fender window and the RGX window and to dma_wmb(); the host suite records them. ++ */ ++struct g5p_wk_io { ++ void *ctx; ++ void (*w1)(void *ctx, u32 reg, u32 value); ++ u32 (*r1)(void *ctx, u32 reg); ++ void (*wbarrier)(void *ctx); ++ void (*w0)(void *ctx, u32 reg, u32 value); ++}; ++ ++static inline bool g5p_wk_mode_valid(u32 mode) ++{ ++ return mode <= G5P_WK_ALWAYS; ++} ++ ++static inline u32 g5p_wk_mix(u32 x) ++{ ++ x ^= x >> 16; ++ x *= 0x7feb352dU; ++ x ^= x >> 15; ++ x *= 0x846ca68bU; ++ x ^= x >> 16; ++ return x; ++} ++ ++/* The phase's index among its engine's eligible phases: TA of job seq >= 2 -> seq - 2, ++ * 3D of job seq >= 1 -> seq - 1. ++ */ ++static inline u32 g5p_wk_index(u32 seq, u32 e) ++{ ++ return e ? seq - 1 : seq - 2; ++} ++ ++static inline u32 g5p_wk_arm(u32 mode, u32 seed, u32 seq, u32 e) ++{ ++ u32 i, flip; ++ ++ if (!mode || !g5p_wk_mode_valid(mode) || !seq || e > 1) ++ return G5P_WK_ARM_NONE; ++ if (!e && seq == 1) ++ return G5P_WK_ARM_EXCLUDED; ++ if (mode == G5P_WK_ALWAYS) ++ return G5P_WK_ARM_FACTOR; ++ i = g5p_wk_index(seq, e); ++ flip = g5p_wk_mix(g5p_wk_mix(seed & ~G5P_WK_MIRROR) + (e << 24) + ++ (i >> 1)) & ++ 1; ++ flip ^= !!(seed & G5P_WK_MIRROR); ++ return ((i & 1) ^ flip) ? G5P_WK_ARM_FACTOR : G5P_WK_ARM_CONTROL; ++} ++ ++static inline u32 g5p_wk_mts(u32 e) ++{ ++ return G5P_WK_MTS_TA | e; ++} ++ ++/* wake111: the evidence word, one bit per fact (the caller samples each fact once). */ ++static inline u32 g5p_wk_power_mask(bool powered, bool fender_ready, ++ bool released, bool alive, ++ bool control_complete, bool ds108, ++ bool slave_lock) ++{ ++ return (powered ? G5P_WK_PWR_CORE : 0) | ++ (fender_ready ? G5P_WK_PWR_FENDER : 0) | ++ (released ? G5P_WK_PWR_RELEASED : 0) | ++ (alive ? G5P_WK_PWR_ALIVE : 0) | ++ (control_complete ? G5P_WK_PWR_CONTROL : 0) | ++ (ds108 ? G5P_WK_PWR_DS108 : 0) | ++ (slave_lock ? G5P_WK_PWR_SLAVE_LOCK : 0); ++} ++ ++static inline bool g5p_wk_power_ready(u32 power) ++{ ++ return (power & G5P_WK_PWR_REQUIRED) == G5P_WK_PWR_REQUIRED; ++} ++ ++/* The RAM guards, common to both arms, in priority order. 0 = pass. */ ++static inline u32 g5p_wk_guard(const struct g5p_wk_in *in) ++{ ++ if (in->attempted) ++ return G5P_WK_SKIP_TWICE; ++ if (in->term_failed) ++ return G5P_WK_SKIP_LICENSE; ++ if (in->boot_failed) ++ return G5P_WK_SKIP_FAULT; ++ if (!g5p_wk_power_ready(in->power)) ++ return G5P_WK_SKIP_POWER; ++ if (in->r74f_pending) ++ return G5P_WK_SKIP_R74F; ++ if (in->start > 255 || in->target > 255 || in->prod != in->target || ++ in->cons > 255 || ((in->target - in->start) & 255) != 1 || ++ ((in->cons - in->start) & 255) > 1) ++ return G5P_WK_SKIP_RING; ++ if (in->stamp[0] == in->seq || in->stamp[1] == in->seq || in->type1) ++ return G5P_WK_SKIP_RETIRED; ++ return 0; ++} ++ ++/* The factor's register sequence (Apple's order). The caller set *fence and *gpc to U32_MAX ++ * (not read). ++ */ ++static inline u32 g5p_wk_ring(const struct g5p_wk_io *io, u32 e, u32 *fence, ++ u32 *gpc) ++{ ++ io->w1(io->ctx, G5P_WK_R1_FENDER, 7); ++ *fence = io->r1(io->ctx, G5P_WK_R1_FENDER); ++ if (*fence & ~7U) ++ return G5P_WK_SKIP_FENCE; ++ *gpc = io->r1(io->ctx, G5P_WK_R1_GPC); ++ if (!G5P_GPC_ON(*gpc)) ++ return G5P_WK_SKIP_GPC; ++ /* the publish's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ io->wbarrier(io->ctx); ++ io->w0(io->ctx, G5P_WK_R0_MTS, g5p_wk_mts(e)); ++ return G5P_WK_DID_RANG; ++} ++ ++/* One phase's decision and, for a factor that passed every guard, its register sequence ++ * (the control arm never touches io). ++ */ ++static inline u32 g5p_wk_decide(const struct g5p_wk_in *in, ++ const struct g5p_wk_io *io, u32 *fence, ++ u32 *gpc) ++{ ++ u32 arm = g5p_wk_arm(in->mode, in->seed, in->seq, in->e), skip; ++ ++ *fence = U32_MAX; ++ *gpc = U32_MAX; ++ if (arm == G5P_WK_ARM_NONE) ++ return G5P_WK_DID_NONE; ++ if (arm == G5P_WK_ARM_EXCLUDED) ++ return G5P_WK_DID_EXCLUDED; ++ skip = g5p_wk_guard(in); ++ if (skip) ++ return skip; ++ if (arm == G5P_WK_ARM_CONTROL) ++ return G5P_WK_DID_SHAM; ++ return g5p_wk_ring(io, in->e, fence, gpc); ++} ++ ++/* A decided phase never decides again (the caller stores this before it unlocks). */ ++static inline bool g5p_wk_attempted(u32 did) ++{ ++ return did != G5P_WK_DID_NONE && did != G5P_WK_DID_EXCLUDED; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-window.h b/drivers/gpu/apple-g5p/render-user-window.h +new file mode 100644 +index 000000000..bfbae628d +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-window.h +@@ -0,0 +1,74 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * The firmware window of a render VM (docs/gpu/multi-client-design.md §6 R4, stage 3). ++ * ++ * The firmware's own GPU memory lives in one GiB, catalogue entry G5P_RU_FIRMWARE_CI: its ++ * image, the root and shared pages (rings, the PA/tag arrays, W), the MMU table alias and the ++ * PIO pages (Fender, AIC). Stage 1 copied that catalogue entry into every render VM, so any ++ * render job could address all of it. A render VM now gets, under that entry, a directory ++ * the kernel built that maps only the pages the GPU's own requesters reach through the ++ * context's slot: the PM control page (S) and the PM list and free list (F). Their leaf ++ * entries are copied from the firmware's tables, so their translation is unchanged. ++ * ++ * render_user_fw_window = 0 (G5P_RU_FW_FULL) the stage 1 mapping: the whole firmware GiB; ++ * render contexts need CAP_SYS_RAWIO ++ * 1 (G5P_RU_FW_WINDOW) the window above (default) ++ * 2 (G5P_RU_FW_NONE) no firmware page at all (an experiment) ++ * ++ * Pure helpers, host-tested (scripts/gpu/multiclient2-host/test_render_window.c). The ++ * caller provides g5p_le32() and the boot layout constants. ++ */ ++#ifndef G5P_RENDER_USER_WINDOW_H ++#define G5P_RENDER_USER_WINDOW_H ++ ++#define G5P_RU_FW_FULL 0U ++#define G5P_RU_FW_WINDOW 1U ++#define G5P_RU_FW_NONE 2U ++#define G5P_RU_FW_MODES 3U ++ ++#define G5P_RU_FW_PAGES 3U ++/* The pages of the window, all in the firmware GiB. */ ++static const u64 g5p_ru_fw_window_va[G5P_RU_FW_PAGES] = { ++ G5P_SHARED_VA + G5P_R22_PM, /* PM control and S */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE, /* PM list */ ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, /* PM free list F */ ++}; ++ ++static inline u32 g5p_ru_fw_di(u64 va) ++{ ++ return (va >> 21) & 511; ++} ++ ++static inline u32 g5p_ru_fw_ti(u64 va) ++{ ++ return (va >> 14) & 127; ++} ++ ++/* The firmware's 64-bit leaf entry for `va`, from its leaf pages and their directory ++ * indices; 0 when the firmware does not map it. ++ */ ++static inline u64 g5p_ru_fw_leaf(u8 *const *leaf, const u32 *leaf_di, ++ u32 leaves, u64 va) ++{ ++ for (u32 i = 0; i < leaves; i++) { ++ const u8 *e = leaf[i] + g5p_ru_fw_ti(va) * 8; ++ ++ if (leaf_di[i] == g5p_ru_fw_di(va)) ++ return (u64)g5p_le32(e + 4) << 32 | g5p_le32(e); ++ } ++ return 0; ++} ++ ++/* The window's leaf page index for `va` among `n` distinct directory indices so far, or n ++ * (a new one) when none has its directory index. ++ */ ++static inline u32 g5p_ru_fw_slot(const u32 *di, u32 n, u64 va) ++{ ++ u32 i; ++ ++ for (i = 0; i < n && di[i] != g5p_ru_fw_di(va); i++) ++ ; ++ return i; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/render-variants-r25.h b/drivers/gpu/apple-g5p/render-variants-r25.h +new file mode 100644 +index 000000000..8469a2f4b +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-variants-r25.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_VARIANTS_R25_H ++#define G5P_RENDER_VARIANTS_R25_H ++ ++/* Finite, owned-input experiments. No raw W, register or firmware-patch API. */ ++#define G5P_R25_MAX_BLOCKS 128U ++#define G5P_R25_HEAD 0x3000U ++ ++struct g5p_r25_variants { ++ u32 pm, blocks, context, host, sku, memory, rt, slots; ++}; ++ ++static inline bool g5p_r25_valid(const struct g5p_r25_variants *v) ++{ ++ return v->pm <= 3 && ++ (v->blocks == 0 || v->blocks == 64 || v->blocks == 128) && ++ v->context <= 3 && v->host <= 2 && v->sku <= 3 && ++ v->memory <= 4 && v->rt <= 2 && v->slots <= 1; ++} ++ ++/* Event synthesis, removed context tasks, candidate clocks and the low-address ++ * dialect test cannot qualify rendering, even if firmware stamps retire. ++ */ ++static inline bool g5p_r25_diagnostic(const struct g5p_r25_variants *v) ++{ ++ return v->sku || v->context || v->host || v->rt == 2; ++} ++ ++/* PM payload frames are physical PPNs and have no user PTE attributes. This ++ * changes only PM auxiliary mappings (and, in mode4, other private mappings). ++ * Firmware/shared META mappings and the borrowed sink are never rewritten. ++ */ ++static inline u32 g5p_r25_attributes(const struct g5p_r25_variants *v, u64 va) ++{ ++ if (!v->memory) ++ return 0xc; ++ if (v->memory != 4 && ++ (va < G5P_R22_AUX || va >= G5P_R22_AUX + 5 * G5P_GPU_PAGE)) ++ return 0xc; ++ return v->memory == 2 ? 4 : v->memory == 3 ? 8 : 0; ++} ++ ++/* Preserve the original constructor exactly for the default pool. The extra ++ * blocks fit the existing 16KiB list and free allocations, including F's last ++ * dword at128 blocks. Validate every PA before mutating either allocation. ++ */ ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/retire108.h b/drivers/gpu/apple-g5p/retire108.h +new file mode 100644 +index 000000000..e944b92c9 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/retire108.h +@@ -0,0 +1,434 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RETIRE108_H ++#define G5P_RETIRE108_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P outgoing retire layer (series gpu-recovery108, patch 0006). ++ * ++ * Design: docs/gpu/experiments/recovery108-driver.md section 3. ++ * ++ * Wraps the recovery98 teardown ladder (teardown.h, g5p_td_run()) without ++ * changing it: the TD98 lines of a run are byte for byte the ones of the ++ * same run without this layer. It adds, around a run: ++ * ++ * ADMIT atomically close job admission (one cmpxchg word), then a ++ * bounded RAM-only wait for submitters already inside the ++ * admission window; still inside at the deadline = explicit ++ * ADMIT_OPEN flag, never a block. ++ * CLASSIFY the epoch class from RAM inputs only (struct g5p_r8_in). ++ * RECEIPT one TR108 line with the class and the generation, printed ++ * before the ladder makes its first AIC or R1 access. ++ * (ladder) g5p_td_run(FINAL) through a callback, unchanged. ++ * SLOTS three named post-ladder slots. Each runs only with a glue op ++ * present AND its own parameter on; every skip is named: ++ * fw-sleep validated firmware sleep before stable-off (G2: ++ * no recovered command; the glue has no op) ++ * stable-off one R1 read of isStableOff after a verified ++ * Fender stop leg (G6: default off) ++ * pmgr-gate outgoing PMGR power gate (G1: not implemented, ++ * there is no op and no parameter) ++ * RETIRE the retired latch: from here on g5p_r8_mmio_ok() refuses R0, ++ * R1 and every write except the always-on AIC ack of a late ++ * handler entry. The latch is set whatever the verdict. ++ * ++ * Generations: gen counts real runs (CAPTURE and FINAL) since boot and is ++ * monotonic; a REENTRY (a second FINAL after retire, e.g. notifier then ++ * .shutdown then remove) prints a line but does not advance it. ++ * ++ * Pure policy, like teardown.h: no register access, no kernel API except ++ * cmpxchg(), READ_ONCE()/WRITE_ONCE(), scnprintf() and the errno names. The ++ * same file is compiled by the kernel glue (teardown.inc) and by the offline ++ * tests (patches/gpu-recovery108/tests). Needs teardown.h first. ++ */ ++ ++#define G5P_R8_CLOSED 0x80000000u /* admission word: closed bit */ ++#define G5P_R8_INSIDE 0x7fffffffu /* admission word: submitters inside */ ++#define G5P_R8_POLL_US 100 ++#define G5P_R8_MAX_STEPS 512 /* ADMIT wait cap, independent of the clock */ ++#define G5P_R8_ADMIT_US 20000 /* ADMIT wait budget */ ++ ++/* Admission: one word, closed bit + count of submitters inside the window. */ ++struct g5p_r8_admit { ++ u32 word; ++}; ++ ++/* A submitter enters before its first check of driver state and leaves on ++ * every return path. False once admission is closed: the submit must fail ++ * with -ESHUTDOWN and touch nothing. ++ */ ++static inline bool g5p_r8_admit_enter(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if ((cur & G5P_R8_CLOSED) || ++ (cur & G5P_R8_INSIDE) == G5P_R8_INSIDE) ++ return false; ++ old = cmpxchg(&a->word, cur, cur + 1); ++ if (old == cur) ++ return true; ++ cur = old; ++ } ++} ++ ++static inline void g5p_r8_admit_exit(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ if (!(cur & G5P_R8_INSIDE)) ++ return; /* unbalanced exit: never wrap into the closed bit */ ++ old = cmpxchg(&a->word, cur, cur - 1); ++ if (old == cur) ++ return; ++ cur = old; ++ } ++} ++ ++/* Close (idempotent). Returns the submitters inside at the instant of closing. */ ++static inline u32 g5p_r8_admit_close(struct g5p_r8_admit *a) ++{ ++ u32 cur = READ_ONCE(a->word), old; ++ ++ for (;;) { ++ old = cmpxchg(&a->word, cur, cur | G5P_R8_CLOSED); ++ if (old == cur) ++ return cur & G5P_R8_INSIDE; ++ cur = old; ++ } ++} ++ ++static inline u32 g5p_r8_admit_inside(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_INSIDE; ++} ++ ++static inline bool g5p_r8_admit_closed(const struct g5p_r8_admit *a) ++{ ++ return READ_ONCE(a->word) & G5P_R8_CLOSED; ++} ++ ++/* Epoch classes. The dirty ones are never reported as clean (H10). */ ++enum g5p_r8_class { ++ G5P_R8C_NEVER_BOOTED, /* shared pages never mapped */ ++ G5P_R8C_UNRELEASED, /* mapped, firmware never released */ ++ G5P_R8C_IDLE, /* released and ALIVE, no context ever claimed (not-run) */ ++ G5P_R8C_CLEAN, /* claimed, every job signalled, no holder left */ ++ G5P_R8C_HELD, /* clean, but a process still holds the claimed context */ ++ G5P_R8C_PARKED, /* GPC=0 already verified by a park */ ++ G5P_R8C_IN_FLIGHT, /* a job accepted and not signalled, or a worker inside a wait */ ++ G5P_R8C_TIMEOUT, /* terminal latch = timeout (a late completion stays a timeout) */ ++ G5P_R8C_FAULT, /* failed, event failure or a fault terminal */ ++ G5P_R8C_ABORTED, /* bootstrap power abort latched (the r87b/C3 epoch) */ ++ G5P_R8C_UNKNOWN, /* released, never ALIVE; or an unverified earlier stop */ ++ G5P_R8C_N, ++}; ++ ++static const char *const g5p_r8_class_name[G5P_R8C_N] = { ++ "never-booted", "unreleased", "idle", "clean", "held", "parked", ++ "in-flight", "timeout", "fault", "aborted", "unknown", ++}; ++ ++static inline bool g5p_r8_class_dirty(enum g5p_r8_class c) ++{ ++ return c >= G5P_R8C_IN_FLIGHT; ++} ++ ++/* RAM-only inputs, read by the glue before the run (never from a register). */ ++struct g5p_r8_in { ++ bool booted; /* b->maps_ready */ ++ bool released; /* b->released */ ++ bool alive; /* b->alive before FREEZE, or ALIVE was proven this boot */ ++ bool failed; /* b->failed */ ++ bool fault_event; /* b->event_failed */ ++ int terminal; /* teardown terminal latch: 0 none, 1 complete, 2 timeout/fault */ ++ bool claimed; /* a context claimed the device (DRM quarantined_context) */ ++ bool held; /* ... and a render node file still references it */ ++ u32 queued; /* accepted render jobs not yet signalled */ ++ bool worker_busy; /* the command lock was busy: a worker may be inside a wait */ ++ bool parked; /* b->parked */ ++ bool stopped_unparked; /* b->stopped && !b->parked before this run */ ++ bool power_abort; /* r73b early_power_abort */ ++}; ++ ++static inline u32 g5p_r8_in_bits(const struct g5p_r8_in *i) ++{ ++ return i->booted | i->released << 1 | i->alive << 2 | i->failed << 3 | ++ i->fault_event << 4 | i->claimed << 5 | i->held << 6 | ++ (i->queued != 0) << 7 | i->worker_busy << 8 | i->parked << 9 | ++ i->stopped_unparked << 10 | i->power_abort << 11; ++} ++ ++/* First match wins; the order is the precedence (docs section 3.2). */ ++static inline enum g5p_r8_class g5p_r8_classify(const struct g5p_r8_in *i) ++{ ++ if (!i->booted) ++ return G5P_R8C_NEVER_BOOTED; ++ if (!i->released) ++ return G5P_R8C_UNRELEASED; ++ if (i->stopped_unparked) ++ return G5P_R8C_UNKNOWN; ++ if (i->power_abort) ++ return G5P_R8C_ABORTED; ++ if (i->failed || i->fault_event) ++ return G5P_R8C_FAULT; ++ if (i->terminal == 2) ++ return G5P_R8C_TIMEOUT; ++ if (i->queued || i->worker_busy) ++ return G5P_R8C_IN_FLIGHT; ++ if (i->parked) ++ return G5P_R8C_PARKED; ++ if (!i->alive) ++ return G5P_R8C_UNKNOWN; ++ if (i->claimed) ++ return i->held ? G5P_R8C_HELD : G5P_R8C_CLEAN; ++ return G5P_R8C_IDLE; ++} ++ ++enum g5p_r8_slot { ++ G5P_R8S_FW_SLEEP, ++ G5P_R8S_STABLE_OFF, ++ G5P_R8S_PMGR_GATE, ++ G5P_R8S_N, ++}; ++ ++enum g5p_r8_skip { ++ G5P_R8K_RAN, /* the slot ran (see its result) */ ++ G5P_R8K_NO_MECHANISM, /* the glue has no op for it */ ++ G5P_R8K_OFF, /* its parameter is off (default) */ ++ G5P_R8K_NOT_FINAL, /* CAPTURE never runs a slot */ ++ G5P_R8K_NEEDS_RT1, /* needs a verified Fender stop leg in this run */ ++ G5P_R8K_DIRTY, /* the ladder ended dirty: no register access */ ++ G5P_R8K_N, ++}; ++ ++static const char *const g5p_r8_skip_name[G5P_R8K_N] = { ++ "ran", "no-mechanism", "off", "not-final", "needs-rt1", "dirty", ++}; ++ ++#define G5P_R8F_ADMIT_OPEN \ ++ BIT(0) /* submitters still inside at the ADMIT deadline */ ++#define G5P_R8F_ADMIT_LATE BIT(1) /* ... or the step cap ended the wait */ ++#define G5P_R8F_REENTRY BIT(2) /* answered after retire: nothing ran */ ++#define G5P_R8F_BUSY BIT(3) /* a concurrent entrant was inside */ ++#define G5P_R8F_SLOT_FAIL BIT(4) /* a slot ran and failed */ ++#define G5P_R8F_STABLE_OFF BIT(5) /* stable-off slot read isStableOff == 1 */ ++/* jobs still queued at DETACH (abandoned to their worker) */ ++#define G5P_R8F_DETACH_OPEN BIT(6) ++ ++/* ++ * Ops. td_run runs the recovery98 ladder and returns its rc; td_flags/td_verdict ++ * read its result. slot[] may hold NULL (no mechanism). Only slot ops touch a ++ * register, and only after the ladder, never after RETIRE. ++ */ ++struct g5p_r8_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); ++ int (*td_run)(void *ctx, enum g5p_td_mode mode); ++ u32 (*td_flags)(void *ctx); ++ enum g5p_td_verdict (*td_verdict)(void *ctx); ++ int (*slot[G5P_R8S_N])(void *ctx, u32 *value); ++ void (*emit)(void *ctx, const char *line); ++}; ++ ++struct g5p_r8 { ++ struct g5p_r8_admit admit; ++ u32 admit_us; /* ADMIT budget */ ++ bool slot_on[G5P_R8S_N]; /* the slots' parameters (default all off) */ ++ /* latches */ ++ int running; ++ int retired; /* 0, or the generation that retired the device */ ++ u32 gen; /* monotonic: real runs since boot */ ++ /* last run */ ++ enum g5p_td_mode mode; ++ enum g5p_r8_class cls; ++ u32 in_bits, flags, inside0, inside1, steps, detach_queued; ++ enum g5p_r8_skip skip[G5P_R8S_N]; ++ int slot_rc[G5P_R8S_N]; ++ u32 slot_val[G5P_R8S_N]; ++ int rc; ++ enum g5p_td_verdict verdict; ++ u32 lines; /* TR108 lines emitted since boot */ ++}; ++ ++static inline void g5p_r8_init(struct g5p_r8 *s) ++{ ++ *s = (struct g5p_r8){ .admit_us = G5P_R8_ADMIT_US }; ++} ++ ++/* ++ * The predicate for every register access outside the ladder: after RETIRE only ++ * RAM, a PMGR read and the AIC (the always-on swirq ack of a late handler entry) ++ * remain. R0 and R1 are refused forever, whatever the verdict. ++ */ ++static inline bool g5p_r8_mmio_ok(const struct g5p_r8 *s, enum g5p_td_dom d) ++{ ++ if (d == G5P_TDD_RAM || d == G5P_TDD_PMGR || d == G5P_TDD_AIC) ++ return true; ++ return !READ_ONCE(s->retired); ++} ++ ++static inline bool g5p_r8_retired(const struct g5p_r8 *s) ++{ ++ return READ_ONCE(s->retired) != 0; ++} ++ ++/* The DRM frontend's DETACH callback reports what it left to the job workers. */ ++static inline void g5p_r8_note_detach(struct g5p_r8 *s, u32 queued) ++{ ++ WRITE_ONCE(s->detach_queued, queued); ++ if (queued) ++ s->flags |= G5P_R8F_DETACH_OPEN; ++} ++ ++static inline void g5p_r8_line(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ const char *line) ++{ ++ s->lines++; ++ if (o->emit) ++ o->emit(o->ctx, line); ++} ++ ++/* Bounded RAM-only wait for submitters inside the admission window. */ ++static inline void g5p_r8_admit_wait(struct g5p_r8 *s, ++ const struct g5p_r8_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)s->admit_us * 1000; ++ ++ s->inside0 = g5p_r8_admit_close(&s->admit); ++ s->inside1 = s->inside0; ++ s->steps = 0; ++ while (s->inside1) { ++ if (s->steps >= G5P_R8_MAX_STEPS) { ++ s->flags |= G5P_R8F_ADMIT_LATE; ++ break; ++ } ++ s->steps++; ++ if (o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_R8_POLL_US); ++ s->inside1 = g5p_r8_admit_inside(&s->admit); ++ } ++ if (s->inside1) ++ s->flags |= G5P_R8F_ADMIT_OPEN; ++} ++ ++static inline void g5p_r8_slots(struct g5p_r8 *s, const struct g5p_r8_ops *o) ++{ ++ u32 tdf = o->td_flags ? o->td_flags(o->ctx) : 0; ++ unsigned int i; ++ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ s->slot_rc[i] = 0; ++ s->slot_val[i] = 0; ++ if (s->mode != G5P_TD_FINAL) ++ s->skip[i] = G5P_R8K_NOT_FINAL; ++ else if (!o->slot[i]) ++ s->skip[i] = G5P_R8K_NO_MECHANISM; ++ else if (!s->slot_on[i]) ++ s->skip[i] = G5P_R8K_OFF; ++ else if (s->verdict == G5P_TDV_DIRTY) ++ s->skip[i] = G5P_R8K_DIRTY; ++ else if (!(tdf & G5P_TDF_RT1_OK)) ++ s->skip[i] = G5P_R8K_NEEDS_RT1; ++ else ++ s->skip[i] = G5P_R8K_RAN; ++ } ++ /* One attempt each, in slot order, no retry; a failed slot ends the slots. */ ++ for (i = 0; i < G5P_R8S_N; i++) { ++ if (s->skip[i] != G5P_R8K_RAN) ++ continue; ++ if (s->flags & G5P_R8F_SLOT_FAIL) { ++ s->skip[i] = G5P_R8K_DIRTY; ++ continue; ++ } ++ s->slot_rc[i] = o->slot[i](o->ctx, &s->slot_val[i]); ++ if (s->slot_rc[i]) ++ s->flags |= G5P_R8F_SLOT_FAIL; ++ else if (i == G5P_R8S_STABLE_OFF && s->slot_val[i] == 1) ++ s->flags |= G5P_R8F_STABLE_OFF; ++ } ++} ++ ++/* ++ * One run. mode is the teardown mode the glue chose (teardown=1 CAPTURE, ++ * teardown=2 FINAL). Returns the ladder's rc, -EBUSY for a concurrent entrant, ++ * and the cached rc after retire. Lines: ++ * TR108 gen= phase=classify why= mode= class= dirty= in= inside= after= flags= admit_us= ++ * (TD98 lines of the ladder, unchanged) ++ * TR108 gen= phase=retired|captured verdict= class= flags= fw-sleep= stable-off= pmgr-gate= ++ * so= detach_queued= rc= ++ * TR108 gen= phase=reentry why= retired_gen= rc= (after retire; no gen step) ++ */ ++static inline int g5p_r8_run(struct g5p_r8 *s, const struct g5p_r8_ops *o, ++ enum g5p_td_mode mode, const struct g5p_r8_in *in, ++ const char *why) ++{ ++ char line[256]; ++ u64 t0; ++ int rc; ++ ++ if (cmpxchg(&s->running, 0, 1)) { ++ /* A concurrent entrant touches nothing; it does not reach the ladder either. */ ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=busy why=%s mode=%s rc=%d", ++ READ_ONCE(s->gen), why, g5p_td_mode_name[mode], ++ -EBUSY); ++ if (o->emit) ++ o->emit(o->ctx, line); ++ return -EBUSY; ++ } ++ if (READ_ONCE(s->retired)) { ++ /* The ladder answers from its cache too (REENTRY); nothing else runs. */ ++ rc = o->td_run(o->ctx, mode); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=reentry why=%s mode=%s retired_gen=%d rc=%d", ++ s->gen, why, g5p_td_mode_name[mode], ++ READ_ONCE(s->retired), rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++ } ++ s->gen++; ++ s->mode = mode; ++ s->flags = 0; ++ s->inside0 = 0; ++ s->inside1 = 0; ++ s->steps = 0; ++ WRITE_ONCE(s->detach_queued, 0); ++ t0 = o->now_ns(o->ctx); ++ if (mode == G5P_TD_FINAL) ++ g5p_r8_admit_wait(s, o); ++ s->cls = g5p_r8_classify(in); ++ s->in_bits = g5p_r8_in_bits(in); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=classify why=%s mode=%s class=%s dirty=%u in=%03x term=%d queued=%u inside=%u after=%u flags=%02x admit_us=%u", ++ s->gen, why, g5p_td_mode_name[mode], g5p_r8_class_name[s->cls], ++ g5p_r8_class_dirty(s->cls), s->in_bits, in->terminal, ++ in->queued, s->inside0, s->inside1, s->flags, ++ (u32)((o->now_ns(o->ctx) - t0) / 1000)); ++ /* The receipt is out before the ladder's first AIC or R1 access. */ ++ g5p_r8_line(s, o, line); ++ s->rc = o->td_run(o->ctx, mode); ++ s->verdict = o->td_verdict ? o->td_verdict(o->ctx) : G5P_TDV_NONE; ++ g5p_r8_slots(s, o); ++ if (mode == G5P_TD_FINAL) ++ WRITE_ONCE(s->retired, (int)s->gen); ++ scnprintf(line, sizeof(line), ++ "TR108 gen=%u phase=%s verdict=%s class=%s flags=%02x fw-sleep=%s stable-off=%s pmgr-gate=%s so=%u detach_queued=%u rc=%d", ++ s->gen, mode == G5P_TD_FINAL ? "retired" : "captured", ++ g5p_td_verdict_name[s->verdict], g5p_r8_class_name[s->cls], ++ s->flags, g5p_r8_skip_name[s->skip[G5P_R8S_FW_SLEEP]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_STABLE_OFF]], ++ g5p_r8_skip_name[s->skip[G5P_R8S_PMGR_GATE]], ++ s->slot_val[G5P_R8S_STABLE_OFF], READ_ONCE(s->detach_queued), ++ s->rc); ++ g5p_r8_line(s, o, line); ++ WRITE_ONCE(s->running, 0); ++ return s->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.h b/drivers/gpu/apple-g5p/teardown.h +new file mode 100644 +index 000000000..b536f428c +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.h +@@ -0,0 +1,740 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_TEARDOWN_H ++#define G5P_TEARDOWN_H ++ ++#include "bits-compat.h" ++ ++/* ++ * G5P GPU teardown state machine (series gpu-recovery98). ++ * ++ * Design and evidence: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md. ++ * ++ * Pure policy: no register access, no kernel API except cmpxchg(), scnprintf() ++ * and the errno names. Every hardware or firmware-visible access goes through ++ * struct g5p_td_ops, and every call is checked against g5p_td_allow[phase][domain] ++ * before it is made. The same file is compiled by the kernel glue (teardown.inc) ++ * and by the offline fault-injection tests (patches/gpu-recovery/tests). ++ * ++ * Two modes: ++ * CAPTURE timeout / fault / attach failure, and (gpu-recovery99) any number of looks at a ++ * live epoch. Observation only: reads RAM rings, ++ * changes nothing the host could still want to export. Never writes ++ * driver or firmware-shared state, never masks the IRQ, never touches ++ * a register. Arms the quiet latch for diagnostics (g5p_td_mmio_ok()). ++ * FINAL shutdown / reboot notifier / remove. Runs the whole ladder and leaves ++ * a known state for the next kernel. ++ * ++ * Domains: RAM (DMA pages + driver structs), AIC (swirq 0x20e104000 ack and the ++ * IRQ-chip mask), R1 (Fender wrapper window), R0 (core register window), PMGR ++ * (0x20e080340 read-only). R0 has no op at all and is forbidden in every phase. ++ * PMGR is read-only in every phase. R1 is reachable only in the RESET phase. ++ * ++ * Nothing here frees or reuses memory the GPU might still access. ++ */ ++ ++#define G5P_TD_NCMD 4 /* command rings: shared objects 0x60 0x64 0x68 0x6c */ ++#define G5P_TD_NRING (G5P_TD_NCMD + 1) /* + event ring (object 0x70) */ ++#define G5P_TD_MAX_RCPT 12 ++#define G5P_TD_MAX_STEPS 4096 /* hard stop independent of the clock */ ++#define G5P_TD_POLL_US 100 ++#define G5P_TD_EV_SLACK \ ++ 16 /* events produced during the phase above this = storm */ ++ ++enum g5p_td_mode { ++ G5P_TD_CAPTURE = 1, ++ G5P_TD_FINAL = 2, ++}; ++ ++enum g5p_td_phase { ++ G5P_TDP_FREEZE, /* stopped=1 alive=0, watchdog disarmed (RAM) */ ++ G5P_TDP_ACCOUNT, /* RAM-only ring/event accounting */ ++ G5P_TDP_DRAIN, /* bounded RAM-only wait for the firmware to self-drain */ ++ G5P_TDP_EVENTS, /* bounded RAM-only event-ring quiesce */ ++ G5P_TDP_IRQ, /* AIC ack, IRQ mask, settle */ ++ G5P_TDP_DETACH, /* host detach, RAM only */ ++ G5P_TDP_RESET, /* RT1 Fender stop leg when eligible; PMGR read */ ++ G5P_TDP_DONE, ++ G5P_TDP_N, ++}; ++ ++enum g5p_td_dom { ++ G5P_TDD_RAM, ++ G5P_TDD_AIC, ++ G5P_TDD_R1, ++ G5P_TDD_R0, ++ G5P_TDD_PMGR, ++ G5P_TDD_N, ++}; ++ ++#define G5P_TDA_R 1 ++#define G5P_TDA_W 2 ++#define G5P_TDA_RW 3 ++ ++/* Hard stop boundaries. A phase may only use the domains marked here. */ ++static const unsigned char g5p_td_allow[G5P_TDP_N][G5P_TDD_N] = { ++ /* RAM AIC R1 R0 PMGR */ ++ [G5P_TDP_FREEZE] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_ACCOUNT] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_DRAIN] = { G5P_TDA_R, 0, 0, 0, 0 }, ++ [G5P_TDP_EVENTS] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_IRQ] = { G5P_TDA_R, G5P_TDA_W, 0, 0, 0 }, ++ [G5P_TDP_DETACH] = { G5P_TDA_RW, 0, 0, 0, 0 }, ++ [G5P_TDP_RESET] = { G5P_TDA_R, 0, G5P_TDA_RW, 0, G5P_TDA_R }, ++ [G5P_TDP_DONE] = { G5P_TDA_R, 0, 0, 0, 0 }, ++}; ++ ++#define G5P_TDF_NOT_BOOTED \ ++ BIT(0) /* shared pages not mapped: nothing to tear down */ ++#define G5P_TDF_UNRELEASED BIT(1) /* firmware never released */ ++#define G5P_TDF_PROTOCOL \ ++ BIT(2) /* ring index > 255, read error, producer moved */ ++#define G5P_TDF_TERMINAL BIT(3) /* TA complete/timeout latched before entry */ ++#define G5P_TDF_LATE_TERMINAL BIT(4) /* ... or during the teardown (race) */ ++#define G5P_TDF_ABANDON BIT(5) /* >=1 ring pending after the drain */ ++#define G5P_TDF_SELF_DRAINED \ ++ BIT(6) /* >=1 ring consumed by the firmware, no bell */ ++#define G5P_TDF_FAULT_EVENT \ ++ BIT(7) /* event type other than 0/1 left unconsumed */ ++#define G5P_TDF_EV_UNQUIET \ ++ BIT(8) /* event ring still non-empty after the budget */ ++#define G5P_TDF_IRQ_UNSETTLED BIT(9) /* IRQ count still moving after the mask */ ++#define G5P_TDF_RT1_OK BIT(10) /* Fender stop leg verified GPC=0 */ ++#define G5P_TDF_RT1_FAIL BIT(11) /* stop leg attempted, not verified */ ++#define G5P_TDF_RT1_SKIPPED BIT(12) /* no register access; see skip reason */ ++#define G5P_TDF_OVERRUN BIT(13) /* step or time budget exhausted */ ++#define G5P_TDF_VIOLATION BIT(14) /* a gate refused an access (machine bug) */ ++#define G5P_TDF_CAPTURE_ONLY BIT(15) /* CAPTURE run: nothing was changed */ ++#define G5P_TDF_REENTRY BIT(16) /* second entry answered from the cache */ ++ ++enum g5p_td_skip { ++ G5P_TDS_NONE, ++ G5P_TDS_NOT_BOOTED, ++ G5P_TDS_NOT_RELEASED, ++ G5P_TDS_NOT_POWERED, ++ G5P_TDS_ALREADY_PARKED, /* clean: GPC=0 already verified by park */ ++ G5P_TDS_STOPPED_UNPARKED, /* dirty: an earlier stop leg did not verify */ ++ G5P_TDS_POWER_ABORT, ++ G5P_TDS_FW_POLICY, /* meta+0x6c8 != 0: firmware also writes R1+0x1000 */ ++ G5P_TDS_LOCK_BUSY, ++ G5P_TDS_CAPTURE_MODE, ++ G5P_TDS_BAD_STATE, /* protocol/overrun/unsettled IRQ: no register access */ ++ G5P_TDS_N, ++}; ++ ++enum g5p_td_verdict { ++ G5P_TDV_NONE, ++ G5P_TDV_OBSERVED, /* CAPTURE: nothing changed */ ++ G5P_TDV_UNTOUCHED, /* never booted or never released: nothing to do */ ++ G5P_TDV_QUIESCED, /* rings clear, events clear, IRQ masked, GPC=0 verified */ ++ G5P_TDV_ABANDONED, /* as QUIESCED, but >=1 command abandoned after the drain */ ++ G5P_TDV_RAM_ONLY, /* IRQ masked, no register access: GPU left as found */ ++ G5P_TDV_DIRTY, /* a step failed: warm continue forbidden, cold DFU */ ++ G5P_TDV_N, ++}; ++ ++static const char *const g5p_td_phase_name[G5P_TDP_N] = { ++ "freeze", "account", "drain", "events", ++ "irq", "detach", "reset", "done", ++}; ++ ++static const char *const g5p_td_mode_name[3] = { "none", "capture", "final" }; ++ ++static const char *const g5p_td_verdict_name[G5P_TDV_N] = { ++ "none", "observed", "untouched", "quiesced", ++ "abandoned", "ram-only", "dirty", ++}; ++ ++struct g5p_td_budget { ++ u32 drain_us; /* self-drain wait; DC3 took 2.34 ms (r87b) */ ++ u32 events_us; ++ u32 settle_us; /* IRQ count must stay constant this long after the mask */ ++ u32 events_pass_max; /* entries per pass; the ring has 256 slots */ ++ u32 events_passes; ++ u32 total_us; /* RESET is skipped when the RAM phases already overran this */ ++}; ++ ++#define G5P_TD_BUDGET_DEFAULT { 20000, 20000, 2000, 256, 4, 100000 } ++ ++/* Inputs the glue reads from RAM before the run (never from a register). */ ++struct g5p_td_pre { ++ bool booted; /* maps_ready: shared pages exist */ ++ bool released; /* firmware was released */ ++ bool powered; /* ps_gfx held on by the retained PM reference */ ++ bool fender_ready; ++ bool already_parked; /* b->parked: GPC=0 verified by park */ ++ bool stopped_unparked; /* b->stopped && !b->parked: an earlier stop did not verify */ ++ bool policy_host_owned; /* init meta+0x6c8 == 0 */ ++ bool power_abort; /* r73b early_power_abort latched */ ++ bool lock_held; /* the glue holds g->lock: no worker is inside a hardware wait */ ++}; ++ ++/* ++ * Every op is bounded by its own contract; the machine additionally bounds how ++ * often and for how long it calls them. Ops marked RAM never touch a register. ++ */ ++struct g5p_td_ops { ++ void *ctx; ++ u64 (*now_ns)(void *ctx); ++ void (*relax_us)(void *ctx, u32 us); /* sleep, no register access */ ++ void (*freeze)(void *ctx); /* RAM: stopped=1 alive=0, disarm */ ++ /* RAM: idx 0..NCMD-1 command rings, NCMD = event ring. -ENODEV: not mapped. */ ++ int (*ring_read)(void *ctx, u32 idx, u32 *cons, u32 *prod); ++ /* RAM: consume at most max entries of type 0/1, stop at the first other type. */ ++ int (*event_pass)(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault); ++ bool (*terminal_seen)(void *ctx); /* RAM: TA complete/timeout latch */ ++ void (*detach)(void *ctx); /* RAM: signal/forget host jobs */ ++ void (*aic_ack)(void *ctx); /* AIC: swirq+0x94 = 0x4000 */ ++ void (*irq_mask)(void *ctx); /* AIC: ack stays, line masked */ ++ u32 (*irq_total)(void *ctx); /* RAM: handler entry counter */ ++ int (*fender_stop)(void *ctx); /* R1: g5p_fender_idle() */ ++ int (*pmgr_actual)(void *ctx, u32 *target, ++ u32 *actual); /* PMGR read; may be NULL */ ++ void (*emit)(void *ctx, const char *line); /* printk; may be NULL */ ++}; ++ ++struct g5p_td_receipt { ++ unsigned char seq, phase, mode, verdict; ++ s32 rc; ++ u32 us, steps, flags; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [dom][0]=reads [dom][1]=writes, op level */ ++ u32 a, b; /* phase specific, see g5p_td_run() */ ++}; ++ ++struct g5p_td { ++ struct g5p_td_budget budget; ++ /* latches shared with the render path; touched by cmpxchg only */ ++ int running; /* 0 idle, 1 inside g5p_td_run() */ ++ int terminal; /* 0, 1 complete, 2 timeout: first writer wins */ ++ int quiet; /* diagnostics MMIO forbidden from here on */ ++ /* 0 idle, 1 captured (informational), 2 finalized: only 2 answers from the cache */ ++ int state; ++ u32 snaps; /* CAPTURE runs since init: the n= of the TD99 snap line */ ++ /* per run */ ++ enum g5p_td_mode mode; ++ enum g5p_td_phase phase; ++ enum g5p_td_skip skip; ++ enum g5p_td_verdict verdict; ++ bool eligible; ++ u32 flags, steps, violations, seq, nrcpt; ++ u64 t0_ns; ++ unsigned char cnt[G5P_TDD_N] ++ [2]; /* [enum g5p_td_dom][0=reads,1=writes], op level */ ++ u32 pending_mask, pending_initial, abandoned, drained; ++ u32 ring_cons[G5P_TD_NRING], ring_prod[G5P_TD_NRING]; ++ u32 ev_consumed, ev_left, ev_initial, irq_before, irq_after; ++ u32 us_total; ++ u32 pmgr_target, pmgr_actual; ++ int rc; ++ struct g5p_td_receipt rcpt[G5P_TD_MAX_RCPT]; ++}; ++ ++static inline void g5p_td_init(struct g5p_td *t) ++{ ++ const struct g5p_td_budget b = G5P_TD_BUDGET_DEFAULT; ++ ++ *t = (struct g5p_td){ .budget = b }; ++} ++ ++/* One latch for "the TA reached a terminal state"; the first writer wins. */ ++static inline bool g5p_td_note_terminal(struct g5p_td *t, int kind) ++{ ++ if (cmpxchg(&t->terminal, 0, kind) != 0) ++ return false; ++ WRITE_ONCE(t->quiet, 1); ++ return true; ++} ++ ++/* ++ * The single predicate for diagnostic register access (R0 snapshots, end banks, ++ * exports): false once the epoch is terminal or a teardown has started. Nothing ++ * in this series calls a diagnostic; this exists so the render path can. ++ */ ++ ++/* RT1 (Fender stop leg) eligibility. Pure function of RAM-derived inputs. */ ++static inline bool g5p_td_eligible(const struct g5p_td_pre *p, ++ enum g5p_td_mode mode, enum g5p_td_skip *why) ++{ ++ enum g5p_td_skip s = G5P_TDS_NONE; ++ ++ if (!p->booted) ++ s = G5P_TDS_NOT_BOOTED; ++ else if (!p->released) ++ s = G5P_TDS_NOT_RELEASED; ++ else if (!p->powered || !p->fender_ready) ++ s = G5P_TDS_NOT_POWERED; ++ else if (p->already_parked) ++ s = G5P_TDS_ALREADY_PARKED; ++ else if (p->stopped_unparked) ++ s = G5P_TDS_STOPPED_UNPARKED; ++ else if (p->power_abort) ++ s = G5P_TDS_POWER_ABORT; ++ else if (!p->policy_host_owned) ++ s = G5P_TDS_FW_POLICY; ++ else if (!p->lock_held) ++ s = G5P_TDS_LOCK_BUSY; ++ else if (mode != G5P_TD_FINAL) ++ s = G5P_TDS_CAPTURE_MODE; ++ if (why) ++ *why = s; ++ return s == G5P_TDS_NONE; ++} ++ ++static inline int g5p_td_format(const struct g5p_td_receipt *r, char *buf, ++ size_t n) ++{ ++ return scnprintf(buf, n, ++ "TD98 seq=%u phase=%s mode=%s rc=%d us=%u steps=%u flags=%08x aic=%u/%u r1=%u/%u r0=%u/%u pmgr=%u/%u a=%u b=%u verdict=%s", ++ r->seq, g5p_td_phase_name[r->phase], g5p_td_mode_name[r->mode], ++ r->rc, r->us, r->steps, r->flags, r->cnt[G5P_TDD_AIC][0], ++ r->cnt[G5P_TDD_AIC][1], r->cnt[G5P_TDD_R1][0], ++ r->cnt[G5P_TDD_R1][1], r->cnt[G5P_TDD_R0][0], ++ r->cnt[G5P_TDD_R0][1], r->cnt[G5P_TDD_PMGR][0], ++ r->cnt[G5P_TDD_PMGR][1], r->a, r->b, ++ g5p_td_verdict_name[r->verdict]); ++} ++ ++static inline u32 g5p_td_us_since(const struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ u64 d = o->now_ns(o->ctx) - t->t0_ns; ++ ++ return d > 0xffffffffULL * 1000 ? 0xffffffffu : (u32)(d / 1000); ++} ++ ++static inline void g5p_td_receipt(struct g5p_td *t, const struct g5p_td_ops *o, ++ int rc, u32 a, u32 b) ++{ ++ struct g5p_td_receipt *r = ++ &t->rcpt[t->nrcpt < G5P_TD_MAX_RCPT ? t->nrcpt : ++ G5P_TD_MAX_RCPT - 1]; ++ char line[224]; ++ unsigned int d; ++ ++ r->seq = (unsigned char)t->seq++; ++ r->phase = t->phase; ++ r->mode = t->mode; ++ r->verdict = t->verdict; ++ r->rc = rc; ++ r->us = g5p_td_us_since(t, o); ++ r->steps = t->steps; ++ r->flags = t->flags; ++ for (d = 0; d < G5P_TDD_N; d++) { ++ r->cnt[d][0] = t->cnt[d][0]; ++ r->cnt[d][1] = t->cnt[d][1]; ++ } ++ r->a = a; ++ r->b = b; ++ if (t->nrcpt < G5P_TD_MAX_RCPT) ++ t->nrcpt++; ++ if (o->emit) { ++ g5p_td_format(r, line, sizeof(line)); ++ o->emit(o->ctx, line); ++ } ++} ++ ++/* Count and check one access. A refused access is a bug in the machine itself. */ ++static inline int g5p_td_gate(struct g5p_td *t, enum g5p_td_dom d, ++ unsigned int acc) ++{ ++ if ((g5p_td_allow[t->phase][d] & acc) != acc) { ++ t->violations++; ++ t->flags |= G5P_TDF_VIOLATION; ++ return -EACCES; ++ } ++ if ((acc & G5P_TDA_R) && t->cnt[d][0] < 255) ++ t->cnt[d][0]++; ++ if ((acc & G5P_TDA_W) && t->cnt[d][1] < 255) ++ t->cnt[d][1]++; ++ return 0; ++} ++ ++/* One loop iteration or hardware op. False once the hard step cap is hit. */ ++static inline bool g5p_td_step(struct g5p_td *t) ++{ ++ if (t->steps >= G5P_TD_MAX_STEPS) { ++ t->flags |= G5P_TDF_OVERRUN; ++ return false; ++ } ++ t->steps++; ++ return true; ++} ++ ++static inline bool g5p_td_ring_ok(u32 cons, u32 prod) ++{ ++ return cons <= 255 && prod <= 255; ++} ++ ++static inline void g5p_td_note_late(struct g5p_td *t, ++ const struct g5p_td_ops *o) ++{ ++ if (o->terminal_seen && o->terminal_seen(o->ctx) && ++ !(t->flags & G5P_TDF_TERMINAL)) ++ t->flags |= G5P_TDF_LATE_TERMINAL; ++} ++ ++static inline int g5p_td_account(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 i, bad = 0, pending = 0, cons, prod; ++ int rc; ++ ++ for (i = 0; i < G5P_TD_NRING; i++) { ++ if (!g5p_td_step(t)) ++ return -ETIMEDOUT; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return -EACCES; ++ rc = o->ring_read(o->ctx, i, &cons, &prod); ++ if (rc == -ENODEV) { ++ t->flags |= G5P_TDF_NOT_BOOTED; ++ break; ++ } ++ if (rc || !g5p_td_ring_ok(cons, prod)) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ bad++; ++ continue; ++ } ++ t->ring_cons[i] = cons; ++ t->ring_prod[i] = prod; ++ if (i < G5P_TD_NCMD && cons != prod) { ++ t->pending_mask |= 1u << i; ++ pending++; ++ } ++ } ++ t->pending_initial = pending; ++ t->ev_left = (t->ring_prod[G5P_TD_NCMD] - t->ring_cons[G5P_TD_NCMD]) & ++ 255; ++ t->ev_initial = t->ev_left; ++ g5p_td_receipt(t, o, bad ? -EPROTO : 0, pending, bad); ++ return 0; ++} ++ ++/* ++ * gpu-recovery99 0004: one extra line per CAPTURE run, after the ACCOUNT receipt, so a live ++ * epoch can be looked at ++ * several times (S1a). RAM only: it formats what ACCOUNT stored in struct g5p_td and reads the ++ * handler entry counter ++ * (RAM) and the clock; no register, no state change. The prefix is not TD98 on purpose: receipt ++ * parsers keyed on ++ * "TD98 seq=" / "TD98 begin" ignore it. ++ * n CAPTURE runs since init (1 based) ++ * t_ns ops->now_ns(), for the IRQ rate between two lines ++ * irq handler entry counter (ops->irq_total), 0 if there is no such op ++ * term terminal latch (0 none, 1 complete, 2 timeout); quiet = the diagnostics latch ++ * own bit0 booted 1 released 2 powered 3 fender_ready 4 already_parked 5 stopped_unparked ++ * 6 policy_host_owned 7 power_abort 8 lock_held ++ * cN/pN command ring N consumer / producer; ce/pe the event ring. Zero when the shared pages ++ * are not mapped. ++ */ ++static inline void g5p_td_snap(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ char line[224]; ++ u32 own, irq = 0; ++ ++ t->snaps++; ++ if (!o->emit) ++ return; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R) && o->irq_total) ++ irq = o->irq_total(o->ctx); ++ own = pre->booted | pre->released << 1 | pre->powered << 2 | ++ pre->fender_ready << 3 | pre->already_parked << 4 | ++ pre->stopped_unparked << 5 | pre->policy_host_owned << 6 | ++ pre->power_abort << 7 | pre->lock_held << 8; ++ scnprintf(line, sizeof(line), ++ "TD99 snap n=%u t_ns=%llu irq=%u term=%d quiet=%d own=%03x c0=%u p0=%u c1=%u p1=%u c2=%u p2=%u c3=%u p3=%u ce=%u pe=%u", ++ t->snaps, (unsigned long long)o->now_ns(o->ctx), irq, ++ READ_ONCE(t->terminal), READ_ONCE(t->quiet), own, ++ t->ring_cons[0], t->ring_prod[0], t->ring_cons[1], ++ t->ring_prod[1], t->ring_cons[2], t->ring_prod[2], ++ t->ring_cons[3], t->ring_prod[3], t->ring_cons[G5P_TD_NCMD], ++ t->ring_prod[G5P_TD_NCMD]); ++ o->emit(o->ctx, line); ++} ++ ++static inline void g5p_td_drain(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.drain_us * 1000; ++ u32 i, cons, prod; ++ ++ while (t->pending_mask) { ++ if (!g5p_td_step(t)) ++ break; ++ if (o->now_ns(o->ctx) >= deadline) { ++ t->flags |= G5P_TDF_ABANDON; ++ break; ++ } ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ for (i = 0; i < G5P_TD_NCMD; i++) { ++ if (!(t->pending_mask & (1u << i))) ++ continue; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_R)) ++ return; ++ if (o->ring_read(o->ctx, i, &cons, &prod) || ++ !g5p_td_ring_ok(cons, prod) || ++ prod != t->ring_prod[i]) { ++ /* The producer is host-written and the host is frozen. */ ++ t->flags |= G5P_TDF_PROTOCOL; ++ t->pending_mask &= ~(1u << i); ++ continue; ++ } ++ if (cons == prod) { ++ t->pending_mask &= ~(1u << i); ++ t->flags |= G5P_TDF_SELF_DRAINED; ++ } ++ } ++ } ++ for (i = 0; i < G5P_TD_NCMD; i++) ++ if (t->pending_mask & (1u << i)) ++ t->abandoned++; ++ if (t->abandoned) ++ t->flags |= G5P_TDF_ABANDON; ++ t->drained = t->pending_initial - t->abandoned; ++} ++ ++static inline void g5p_td_events(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u64 deadline = o->now_ns(o->ctx) + (u64)t->budget.events_us * 1000; ++ u32 pass, consumed, left = t->ev_left; ++ bool fault = false; ++ int rc; ++ ++ for (pass = 0; pass < t->budget.events_passes && left; pass++) { ++ if (!g5p_td_step(t)) ++ break; ++ if (g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_RW)) ++ return; ++ consumed = 0; ++ rc = o->event_pass(o->ctx, t->budget.events_pass_max, &consumed, ++ &left, &fault); ++ t->ev_consumed += consumed; ++ if (rc) { ++ t->flags |= G5P_TDF_PROTOCOL; ++ break; ++ } ++ if (fault) { ++ /* Retained for the host; stop consuming, the IRQ is masked next. */ ++ t->flags |= G5P_TDF_FAULT_EVENT; ++ break; ++ } ++ if (!left || o->now_ns(o->ctx) >= deadline) ++ break; ++ o->relax_us(o->ctx, G5P_TD_POLL_US); ++ } ++ t->ev_left = left; ++ /* ++ * A ring holds at most 255 entries, so consuming more than the backlog plus a ++ * few ALIVE ticks means the producer outran us: an empty ring at the last ++ * look is a snapshot, not a quiesce. ++ */ ++ if (!fault && (left || (s32)(t->ev_consumed + left - t->ev_initial) > ++ G5P_TD_EV_SLACK)) ++ t->flags |= G5P_TDF_EV_UNQUIET; ++} ++ ++static inline void g5p_td_irq(struct g5p_td *t, const struct g5p_td_ops *o) ++{ ++ u32 tries, n0, n1; ++ ++ if (g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W) || ++ g5p_td_gate(t, G5P_TDD_AIC, G5P_TDA_W)) ++ return; ++ t->irq_before = o->irq_total(o->ctx); ++ /* Same pair as the IRQ-cap path: ack the software source, then mask the line. */ ++ o->aic_ack(o->ctx); ++ o->irq_mask(o->ctx); ++ for (tries = 0; tries < 3; tries++) { ++ if (!g5p_td_step(t)) ++ break; ++ n0 = o->irq_total(o->ctx); ++ o->relax_us(o->ctx, t->budget.settle_us); ++ n1 = o->irq_total(o->ctx); ++ t->irq_after = n1; ++ if (n0 == n1) ++ return; ++ } ++ t->flags |= G5P_TDF_IRQ_UNSETTLED; ++} ++ ++static inline void g5p_td_reset(struct g5p_td *t, const struct g5p_td_ops *o, ++ const struct g5p_td_pre *pre) ++{ ++ enum g5p_td_skip why = G5P_TDS_NONE; ++ bool bad = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_VIOLATION); ++ int rc = 0; ++ ++ t->eligible = g5p_td_eligible(pre, t->mode, &why); ++ if (t->eligible && ++ (bad || g5p_td_us_since(t, o) > t->budget.total_us)) { ++ t->eligible = false; ++ why = G5P_TDS_BAD_STATE; ++ } ++ t->skip = why; ++ if (!t->eligible) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ } else if (!g5p_td_step(t) || g5p_td_gate(t, G5P_TDD_R1, G5P_TDA_RW)) { ++ t->flags |= G5P_TDF_RT1_SKIPPED; ++ t->skip = G5P_TDS_BAD_STATE; ++ } else { ++ /* ++ * Apple stopFirmwareForPowerDown 6c9fa10, R1 only; flown here after a DC ++ * timeout (round 10). Never retried, even on failure. ++ */ ++ rc = o->fender_stop(o->ctx); ++ t->flags |= rc ? G5P_TDF_RT1_FAIL : G5P_TDF_RT1_OK; ++ } ++ if (o->pmgr_actual && !g5p_td_gate(t, G5P_TDD_PMGR, G5P_TDA_R)) ++ o->pmgr_actual(o->ctx, &t->pmgr_target, &t->pmgr_actual); ++ g5p_td_receipt(t, o, rc, t->eligible ? 0 : (u32)t->skip, ++ t->pmgr_actual); ++} ++ ++static inline void g5p_td_decide(struct g5p_td *t, const struct g5p_td_pre *pre) ++{ ++ bool dirty = t->flags & (G5P_TDF_PROTOCOL | G5P_TDF_EV_UNQUIET | ++ G5P_TDF_IRQ_UNSETTLED | G5P_TDF_RT1_FAIL | ++ G5P_TDF_OVERRUN | G5P_TDF_VIOLATION) || ++ t->skip == G5P_TDS_STOPPED_UNPARKED || ++ t->skip == G5P_TDS_BAD_STATE; ++ ++ if (t->mode == G5P_TD_CAPTURE) ++ t->verdict = (t->flags & G5P_TDF_VIOLATION) ? G5P_TDV_DIRTY : ++ G5P_TDV_OBSERVED; ++ else if (dirty) ++ t->verdict = G5P_TDV_DIRTY; ++ else if (t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED)) ++ t->verdict = G5P_TDV_UNTOUCHED; ++ else if (t->flags & G5P_TDF_RT1_OK || t->skip == G5P_TDS_ALREADY_PARKED) ++ t->verdict = (t->flags & G5P_TDF_ABANDON) ? G5P_TDV_ABANDONED : ++ G5P_TDV_QUIESCED; ++ else ++ t->verdict = G5P_TDV_RAM_ONLY; ++ (void)pre; ++} ++ ++/* ++ * Run the teardown once per mode. Returns 0 for QUIESCED/ABANDONED/UNTOUCHED/OBSERVED, ++ * -EAGAIN for RAM_ONLY (clean, but the GPU was left as found), -EIO for DIRTY, ++ * -EBUSY if another caller is inside the machine. Every phase ends in a receipt. ++ * Receipt a/b: account pending/bad, drain drained/abandoned, events consumed/left, ++ * irq before/after, reset skip-reason/pmgr-actual, done skip-reason/abandoned. ++ * Counters in the receipts are op-level (one count per gate): the AIC pair is ++ * 2 writes, the whole Fender stop leg is one R1 read+write op. ++ */ ++static inline int g5p_td_run(struct g5p_td *t, const struct g5p_td_ops *o, ++ enum g5p_td_mode mode, ++ const struct g5p_td_pre *pre) ++{ ++ if (cmpxchg(&t->running, 0, 1)) ++ return -EBUSY; ++ WRITE_ONCE(t->quiet, 1); ++ if (t->state == 2) { ++ /* Answered from the cache (after FINAL only; CAPTURE re-runs, gpu- ++ * recovery99 0004): no phase runs, nothing is accessed. ++ */ ++ u32 saved_flags = t->flags; ++ ++ t->t0_ns = o->now_ns(o->ctx) - (u64)t->us_total * 1000; ++ t->flags |= G5P_TDF_REENTRY; ++ t->phase = G5P_TDP_DONE; ++ g5p_td_receipt(t, o, t->rc, t->flags, t->steps); ++ t->flags = saved_flags; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++ } ++ t->mode = mode; ++ t->phase = G5P_TDP_FREEZE; ++ t->t0_ns = o->now_ns(o->ctx); ++ t->seq = 0; ++ t->nrcpt = 0; ++ t->flags = mode == G5P_TD_CAPTURE ? G5P_TDF_CAPTURE_ONLY : 0; ++ t->steps = 0; ++ t->violations = 0; ++ t->abandoned = 0; ++ t->drained = 0; ++ t->pending_mask = 0; ++ t->pending_initial = 0; ++ t->ev_consumed = 0; ++ t->ev_left = 0; ++ t->ev_initial = 0; ++ t->irq_before = 0; ++ t->irq_after = 0; ++ t->skip = G5P_TDS_NONE; ++ t->verdict = G5P_TDV_NONE; ++ memset(t->cnt, 0, sizeof(t->cnt)); ++ /* a CAPTURE may now run many times: nothing of the previous look may leak into this one */ ++ memset(t->ring_cons, 0, sizeof(t->ring_cons)); ++ memset(t->ring_prod, 0, sizeof(t->ring_prod)); ++ if (READ_ONCE(t->terminal)) ++ t->flags |= G5P_TDF_TERMINAL; ++ if (!pre->released) ++ t->flags |= G5P_TDF_UNRELEASED; ++ ++ if (mode == G5P_TD_FINAL) { ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->freeze(o->ctx); ++ g5p_td_receipt(t, o, 0, pre->released, pre->policy_host_owned); ++ } ++ t->phase = G5P_TDP_ACCOUNT; ++ if (g5p_td_account(t, o)) ++ t->flags |= G5P_TDF_OVERRUN; ++ g5p_td_note_late(t, o); ++ if (mode == G5P_TD_CAPTURE) ++ g5p_td_snap(t, o, pre); ++ ++ if (mode == G5P_TD_FINAL && ++ !(t->flags & (G5P_TDF_NOT_BOOTED | G5P_TDF_UNRELEASED | ++ G5P_TDF_PROTOCOL | G5P_TDF_OVERRUN))) { ++ t->phase = G5P_TDP_DRAIN; ++ g5p_td_drain(t, o); ++ g5p_td_note_late(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_ABANDON) ? -ETIMEDOUT : 0, ++ t->drained, t->abandoned); ++ ++ t->phase = G5P_TDP_EVENTS; ++ g5p_td_events(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_EV_UNQUIET) ? -ETIMEDOUT : 0, ++ t->ev_consumed, t->ev_left); ++ } ++ if (mode == G5P_TD_FINAL) { ++ /* The IRQ is masked even after an overrun: ack + mask is the safe exit. */ ++ t->phase = G5P_TDP_IRQ; ++ if (!(t->flags & G5P_TDF_NOT_BOOTED)) ++ g5p_td_irq(t, o); ++ g5p_td_receipt(t, o, ++ (t->flags & G5P_TDF_IRQ_UNSETTLED) ? -ETIMEDOUT : ++ 0, ++ t->irq_before, t->irq_after); ++ ++ t->phase = G5P_TDP_DETACH; ++ if (!g5p_td_gate(t, G5P_TDD_RAM, G5P_TDA_W)) ++ o->detach(o->ctx); ++ g5p_td_receipt(t, o, 0, 0, 0); ++ ++ t->phase = G5P_TDP_RESET; ++ g5p_td_reset(t, o, pre); ++ } ++ g5p_td_note_late(t, o); ++ t->phase = G5P_TDP_DONE; ++ g5p_td_decide(t, pre); ++ switch (t->verdict) { ++ case G5P_TDV_DIRTY: ++ t->rc = -EIO; ++ break; ++ case G5P_TDV_RAM_ONLY: ++ t->rc = -EAGAIN; ++ break; ++ default: ++ t->rc = 0; ++ } ++ t->us_total = g5p_td_us_since(t, o); ++ g5p_td_receipt(t, o, t->rc, (u32)t->skip, t->abandoned); ++ t->state = mode == G5P_TD_FINAL ? 2 : 1; ++ WRITE_ONCE(t->running, 0); ++ return t->rc; ++} ++ ++#endif +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +new file mode 100644 +index 000000000..e3875bc59 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -0,0 +1,694 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Kernel glue for the GPU teardown state machine (teardown.h). ++ * Design, evidence and device plan: docs/gpu/experiments/gpu-warm-kexec-hang-proof.md ++ * ++ * Included from core.inc after lifecycle.inc, so it may use g5p_dc_root_word(), ++ * g5p_fender_write()/poll() and g5p_r73b_disarm(). No state is added to struct ++ * g5p_boot or struct g5p_diag: one GPU, one file-static teardown instance. ++ * ++ * Compiled with W=1 against a9/tri96 (patches/gpu-recovery/README.md); NOT run on hardware. ++ * ++ * gpu-recovery108 (0006): every run goes through the retire layer (retire108.h): ++ * admission close, RAM-only epoch class and a TR108 receipt before the ladder, the ++ * named post-ladder slots, and the retired latch after a FINAL. The TD98 lines are ++ * unchanged. docs/gpu/experiments/recovery108-driver.md. ++ */ ++#include ++#include ++#include ++#include "teardown.h" ++#include "retire108.h" ++#include "render-user-bind.h" ++#include "render-user-quiesce.h" ++ ++/* 0 off (default; the notifier and .shutdown do nothing), 1 observe-only receipts ++ * (CAPTURE: RAM reads, no change), 2 full ladder (FINAL). ++ */ ++ ++/* gpu-recovery108 G6 slot: one R1 read of isStableOff after a verified Fender stop leg. */ ++ ++/* Defined in render-r96.inc (same translation unit, included later by render-fixed.inc). */ ++ ++static struct g5p_td g5p_td_state; ++static struct g5p_diag *g5p_td_dev; ++static void (*g5p_td_detach_cb)(void *cookie); ++static void *g5p_td_detach_cookie; ++ ++/* gpu-recovery108: the retire layer around the ladder (retire108.h), one GPU, file static. */ ++static struct g5p_r8 g5p_r8_state = { .admit_us = G5P_R8_ADMIT_US }; ++static void (*g5p_r8_epoch_cb)(void *cookie, struct g5p_r8_in *in); ++static void *g5p_r8_epoch_cookie; ++ ++/* The DRM frontend reports its share of the epoch inputs (claim, holders, queued jobs), RAM only. */ ++static void __maybe_unused g5p_r8_register_epoch(void (*cb)(void *, ++ struct g5p_r8_in *), ++ void *cookie) ++{ ++ g5p_r8_epoch_cookie = cookie; ++ smp_store_release(&g5p_r8_epoch_cb, cb); ++} ++ ++/* Job admission: a submitter enters before its first look at driver state and ++ * leaves on every return; false once a FINAL teardown closed admission. ++ */ ++static bool __maybe_unused g5p_r8_enter(void) ++{ ++ return g5p_r8_admit_enter(&g5p_r8_state.admit); ++} ++ ++static void __maybe_unused g5p_r8_exit(void) ++{ ++ g5p_r8_admit_exit(&g5p_r8_state.admit); ++} ++ ++/* The post-retire register predicate for the R1 helpers outside the ladder. */ ++static bool __maybe_unused g5p_r8_r1_allowed(void) ++{ ++ return g5p_r8_mmio_ok(&g5p_r8_state, G5P_TDD_R1); ++} ++ ++static bool __maybe_unused g5p_r8_is_retired(void) ++{ ++ return g5p_r8_retired(&g5p_r8_state); ++} ++ ++/* The DRM frontend registers the callback that fails its queued/in-flight job ++ * fences (exactly once each, -ENODEV) without touching hardware. ++ */ ++static void __maybe_unused g5p_td_register_detach(void (*cb)(void *), ++ void *cookie) ++{ ++ g5p_td_detach_cookie = cookie; ++ smp_store_release(&g5p_td_detach_cb, cb); ++} ++ ++static u64 g5p_td_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_td_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++static void g5p_td_op_freeze(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ struct g5p_boot *b = g->boot; ++ ++ if (!b) ++ return; ++ /* The IRQ-side plateau banks (g5p_r80_irq_bank/g5p_r85_irq_bank) read R0 until this ++ * latch is set; arm it before anything else (RAM only, no-op without an r96 config). ++ */ ++ /* Same freeze as g5p_boot_park(): nothing may publish after this. */ ++ if (b->r73b) ++ g5p_r73b_disarm(b->r73b); ++ WRITE_ONCE(b->stopped, true); ++ WRITE_ONCE(b->alive, false); ++} ++ ++static int g5p_td_op_ring_read(void *ctx, u32 idx, u32 *cons, u32 *prod) ++{ ++ static const u32 slots[G5P_TD_NRING] = { 0x60, 0x64, 0x68, 0x6c, 0x70 }; ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *ctl; ++ ++ if (!b || !READ_ONCE(b->maps_ready)) ++ return -ENODEV; ++ ctl = (u32 *)g5p_boot_shared(b, slots[idx]); ++ *cons = READ_ONCE(ctl[0]); ++ *prod = READ_ONCE(ctl[4]); ++ dma_rmb(); ++ return 0; ++} ++ ++/* RAM only. Consumes ALIVE (0) and stamp (1) entries; the first other type is ++ * left in place for the host, as g5p_boot_drain() does. ++ */ ++static int g5p_td_op_event_pass(void *ctx, u32 max, u32 *consumed, u32 *left, ++ bool *fault) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ u32 *control = (u32 *)g5p_boot_shared(b, 0x70); ++ u32 consumer, producer, n = 0; ++ unsigned long flags; ++ int ret = 0; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ consumer = READ_ONCE(control[0]); ++ producer = READ_ONCE(control[4]); ++ if (consumer > 255 || producer > 255) { ++ ret = -EPROTO; ++ goto out; ++ } ++ dma_rmb(); ++ while (n < max && consumer != producer) { ++ u32 off = b->layout.offset[0x74] + consumer * 44; ++ ++ if (g5p_le32(b->shared[off / G5P_GPU_PAGE].cpu + ++ off % G5P_GPU_PAGE) > 1) { ++ *fault = true; ++ break; ++ } ++ consumer = (consumer + 1) & 255; ++ mb(); ++ WRITE_ONCE(control[0], consumer); ++ n++; ++ } ++ producer = READ_ONCE(control[4]); ++ *left = producer > 255 ? 255 : (producer - consumer) & 255; ++out: ++ *consumed = n; ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ return ret; ++} ++ ++/* The render path latches terminal states through g5p_td_note_terminal(); until ++ * prod98 wires that call this reports "not seen" and the race is only visible as ++ * ring state. ++ */ ++static bool g5p_td_op_terminal_seen(void *ctx) ++{ ++ return READ_ONCE(g5p_td_state.terminal) != 0; ++} ++ ++static void g5p_td_op_detach(void *ctx) ++{ ++ void (*cb)(void *) = smp_load_acquire(&g5p_td_detach_cb); ++ ++ if (cb) ++ cb(g5p_td_detach_cookie); ++} ++ ++static void g5p_td_op_aic_ack(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* The same software-source ACK the IRQ-cap path uses. */ ++ if (b && b->irq_ready && b->swirq) ++ writel(0x4000, b->swirq + 0x94); ++} ++ ++static void g5p_td_op_irq_mask(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ /* Line masked at the interrupt controller; the handler stays registered for ++ * a late entry, which still ACKs. Same call as the IRQ-cap path. ++ */ ++ if (b && b->irq_ready && b->irq > 0) ++ disable_irq_nosync(b->irq); ++} ++ ++static u32 g5p_td_op_irq_total(void *ctx) ++{ ++ struct g5p_boot *b = ((struct g5p_diag *)ctx)->boot; ++ ++ return b ? (u32)atomic_read(&b->irqs) : 0; ++} ++ ++static int g5p_td_op_fender_stop(void *ctx) ++{ ++ struct g5p_diag *g = ctx; ++ const struct g5p_fender_io io = { ++ .ctx = g, ++ .write = g5p_fender_write, ++ .poll = g5p_fender_poll, ++ }; ++ ++ /* Apple stopFirmwareForPowerDown 6c9fa10; no retry, no PMGR off. */ ++ return g5p_fender_idle(&io); ++} ++ ++static int g5p_td_op_pmgr(void *ctx, u32 *target, u32 *actual) ++{ ++ struct g5p_diag *g = ctx; ++ u32 raw; ++ int ret; ++ ++ if (!g->pmgr) ++ return -ENODEV; ++ ret = regmap_read(g->pmgr, 0x80340, &raw); ++ if (!ret) { ++ *target = raw & 0xf; ++ *actual = (raw >> 4) & 0xf; ++ } ++ return ret; ++} ++ ++/* dev_emerg so the receipt reaches netconsole at any console loglevel. */ ++static void g5p_td_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_diag *)ctx)->dev, "%s\n", line); ++} ++ ++static void g5p_td_pre_fill(struct g5p_diag *g, struct g5p_td_pre *p, ++ bool locked) ++{ ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(p, 0, sizeof(*p)); ++ p->lock_held = locked; ++ p->powered = READ_ONCE(g->powered); ++ p->fender_ready = READ_ONCE(g->fender_ready); ++ if (!b) ++ return; ++ p->booted = READ_ONCE(b->maps_ready); ++ p->released = READ_ONCE(b->released); ++ p->already_parked = READ_ONCE(b->parked); ++ p->stopped_unparked = READ_ONCE(b->stopped) && !READ_ONCE(b->parked); ++ p->power_abort = b->r73b && READ_ONCE(b->r73b->early_power_abort); ++ /* RAM-only read of the host-written init word (same as g5p_dc_snapshot()). */ ++ p->policy_host_owned = ++ p->booted && ++ !g5p_dc_root_word(b, b->layout.offset[0x4d] + 0x6c8); ++} ++ ++/* gpu-recovery108: the ladder as the retire layer's td_run op. The TD98 begin line ++ * and the ladder's own lines are exactly the recovery98/99 ones. ++ */ ++struct g5p_r8_ctx { ++ struct g5p_diag *g; ++ const struct g5p_td_ops *ops; ++ struct g5p_td_pre *pre; ++ const char *why; ++ bool locked; ++}; ++ ++static int g5p_r8_op_td_run(void *ctx, enum g5p_td_mode mode) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ dev_emerg(c->g->dev, ++ "TD98 begin why=%s mode=%s teardown=%u locked=%u\n", c->why, ++ g5p_td_mode_name[mode], (0), c->locked); ++ return g5p_td_run(&g5p_td_state, c->ops, mode, c->pre); ++} ++ ++static u32 g5p_r8_op_td_flags(void *ctx) ++{ ++ return g5p_td_state.flags; ++} ++ ++static enum g5p_td_verdict g5p_r8_op_td_verdict(void *ctx) ++{ ++ return g5p_td_state.verdict; ++} ++ ++static u64 g5p_r8_op_now(void *ctx) ++{ ++ return ktime_get_mono_fast_ns(); ++} ++ ++static void g5p_r8_op_relax(void *ctx, u32 us) ++{ ++ usleep_range(us, us + us / 4 + 1); ++} ++ ++/* G6: R1+0x0 bit 30 after a verified stop leg (the slot runs only then, before RETIRE). */ ++static int g5p_r8_op_stable_off(void *ctx, u32 *value) ++{ ++ struct g5p_r8_ctx *c = ctx; ++ ++ *value = readl(c->g->window[1]) >> 30; ++ return 0; ++} ++ ++static void g5p_r8_op_emit(void *ctx, const char *line) ++{ ++ dev_emerg(((struct g5p_r8_ctx *)ctx)->g->dev, "%s\n", line); ++} ++ ++static void g5p_r8_in_fill(struct g5p_diag *g, struct g5p_r8_in *in, ++ bool locked, enum g5p_td_mode mode) ++{ ++ void (*cb)(void *, struct g5p_r8_in *) = ++ smp_load_acquire(&g5p_r8_epoch_cb); ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ ++ memset(in, 0, sizeof(*in)); ++ in->terminal = READ_ONCE(g5p_td_state.terminal); ++ /* FINAL without the lock: a worker (or a sysfs writer) holds it, maybe inside a wait. */ ++ in->worker_busy = mode == G5P_TD_FINAL && !locked; ++ if (b) { ++ in->booted = READ_ONCE(b->maps_ready); ++ in->released = READ_ONCE(b->released); ++ in->alive = READ_ONCE(b->alive); ++ in->failed = READ_ONCE(b->failed); ++ in->fault_event = READ_ONCE(b->event_failed); ++ in->parked = READ_ONCE(b->parked); ++ in->stopped_unparked = READ_ONCE(b->stopped) && ++ !READ_ONCE(b->parked); ++ in->power_abort = b->r73b && ++ READ_ONCE(b->r73b->early_power_abort); ++ } ++ if (cb) ++ cb(g5p_r8_epoch_cookie, in); ++} ++ ++static int g5p_td_execute(struct g5p_diag *g, enum g5p_td_mode mode, ++ const char *why) ++{ ++ const struct g5p_td_ops ops = { ++ .ctx = g, ++ .now_ns = g5p_td_op_now, ++ .relax_us = g5p_td_op_relax, ++ .freeze = g5p_td_op_freeze, ++ .ring_read = g5p_td_op_ring_read, ++ .event_pass = g5p_td_op_event_pass, ++ .terminal_seen = g5p_td_op_terminal_seen, ++ .detach = g5p_td_op_detach, ++ .aic_ack = g5p_td_op_aic_ack, ++ .irq_mask = g5p_td_op_irq_mask, ++ .irq_total = g5p_td_op_irq_total, ++ .fender_stop = g5p_td_op_fender_stop, ++ .pmgr_actual = g5p_td_op_pmgr, ++ .emit = g5p_td_op_emit, ++ }; ++ struct g5p_td_pre pre; ++ struct g5p_r8_in in; ++ struct g5p_r8_ctx c = { .g = g, .ops = &ops, .pre = &pre, .why = why }; ++ /* fw-sleep (G2) and pmgr-gate (G1) have no mechanism: their ops stay NULL. */ ++ const struct g5p_r8_ops r8ops = { ++ .ctx = &c, ++ .now_ns = g5p_r8_op_now, ++ .relax_us = g5p_r8_op_relax, ++ .td_run = g5p_r8_op_td_run, ++ .td_flags = g5p_r8_op_td_flags, ++ .td_verdict = g5p_r8_op_td_verdict, ++ .slot = { [G5P_R8S_STABLE_OFF] = g5p_r8_op_stable_off }, ++ .emit = g5p_r8_op_emit, ++ }; ++ bool locked = false; ++ int ret; ++ ++ /* Never queue behind a worker or sysfs writer that may be inside MMIO. CAPTURE ++ * reads RAM only and needs no lock. ++ */ ++ if (mode == G5P_TD_FINAL) ++ locked = mutex_trylock(&g->lock); ++ c.locked = locked; ++ g5p_td_pre_fill(g, &pre, locked); ++ g5p_r8_in_fill(g, &in, locked, mode); ++ g5p_r8_state.slot_on[G5P_R8S_STABLE_OFF] = (0); ++ ret = g5p_r8_run(&g5p_r8_state, &r8ops, mode, &in, why); ++ if (g5p_td_state.flags & G5P_TDF_RT1_OK && g->boot) ++ WRITE_ONCE(g->boot->parked, true); ++ if (locked) ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Observation only: arms the quiet predicate, changes nothing else. */ ++static int __maybe_unused g5p_td_capture(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_CAPTURE, why); ++} ++ ++static int g5p_td_final(struct g5p_diag *g, const char *why) ++{ ++ return g5p_td_execute(g, G5P_TD_FINAL, why); ++} ++ ++/* ++ * The quiesce before a kexec, reboot or power-off (render-user-quiesce.h). It runs from the ++ * reboot notifier whatever `teardown` is, once per boot; .shutdown and the `td-kexec` command ++ * reach the same run. It leaves the GPU powered with its firmware stopped and the boot state ++ * frozen: the next kernel boots it as after any warm kexec. ++ */ ++ ++#define G5P_KQ_LOCK_MS 2000U ++#define G5P_KQ_ADMIT_US 20000U ++#define G5P_KQ_OFF_MS 10U ++ ++/* The render frontend's share: its TYPED_V3 epoch inputs and the RAM-only unbind of slots. */ ++struct g5p_kq_frontend { ++ void (*state)(void *cookie, struct g5p_ru_kq_in *in); ++ int (*detach)(void *cookie, const u32 *slots, u32 n); ++}; ++ ++static const struct g5p_kq_frontend *g5p_kq_fe; ++static void *g5p_kq_fe_cookie; ++static atomic_t g5p_kq_ran = ATOMIC_INIT(0); ++ ++struct g5p_kq_ctx { ++ struct g5p_diag *g; ++ u32 slots[G5P_RU_USER_SLOTS]; ++ u32 held; ++}; ++ ++static void __maybe_unused g5p_kq_register(const struct g5p_kq_frontend *fe, ++ void *cookie) ++{ ++ g5p_kq_fe_cookie = cookie; ++ smp_store_release(&g5p_kq_fe, fe); ++} ++ ++static u32 g5p_kq_admit_close(void *ctx) ++{ ++ struct g5p_r8_admit *a = &g5p_r8_state.admit; ++ u64 deadline = ktime_get_ns() + (u64)G5P_KQ_ADMIT_US * NSEC_PER_USEC; ++ u32 inside = g5p_r8_admit_close(a); ++ ++ while (inside && ktime_get_ns() < deadline) { ++ usleep_range(G5P_R8_POLL_US, 2 * G5P_R8_POLL_US); ++ inside = g5p_r8_admit_inside(a); ++ } ++ return inside; ++} ++ ++/* Never queue behind a holder that may be inside MMIO: bounded, like g5p_render_run(). */ ++static bool g5p_kq_lock(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_KQ_LOCK_MS); ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ return true; ++} ++ ++static void g5p_kq_unlock(void *ctx) ++{ ++ mutex_unlock(&((struct g5p_kq_ctx *)ctx)->g->lock); ++} ++ ++/* RAM only: the boot state, the held user slots and the DC ring, then the frontend's inputs. */ ++static void g5p_kq_inputs(void *ctx, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ struct g5p_diag *g = c->g; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ u32 *ctl; ++ ++ if (!READ_ONCE(g->powered)) ++ return; /* Already idle: no detach, MMIO or second gate cycle. */ ++ in->pmgr = !!g->pmgr; ++ if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released) || ++ READ_ONCE(b->parked)) ++ return; ++ in->booted = true; ++ in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ dma_rmb(); ++ in->cons = READ_ONCE(ctl[0]); ++ in->prod = READ_ONCE(ctl[4]); ++ in->ring_idle = in->prod <= 255 && in->cons == in->prod; ++ c->held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), ++ g5p_boot_shared(b, 0x84), c->slots, ++ ARRAY_SIZE(c->slots)); ++ in->held = c->held; ++ if (fe) { ++ in->frontend = true; ++ fe->state(g5p_kq_fe_cookie, in); ++ } ++} ++ ++static int g5p_kq_detach(void *ctx) ++{ ++ struct g5p_kq_ctx *c = ctx; ++ const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); ++ ++ return fe ? fe->detach(g5p_kq_fe_cookie, c->slots, c->held) : -ENODEV; ++} ++ ++static void g5p_kq_freeze(void *ctx) ++{ ++ g5p_td_op_freeze(((struct g5p_kq_ctx *)ctx)->g); ++} ++ ++/* The IRQ-cap path's AIC ack and mask, then wait out a handler already running (process ++ * context; the threaded handler drains RAM and takes no lock the quiesce holds). ++ */ ++static void g5p_kq_irq_off(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ struct g5p_boot *b = g->boot; ++ ++ g5p_td_op_aic_ack(g); ++ g5p_td_op_irq_mask(g); ++ if (b && b->irq_ready && b->irq > 0) ++ synchronize_irq(b->irq); ++} ++ ++/* PMGR only: the bounded gate cycle of the r33/dirty-start path (refuses a gate that is not ++ * plainly on), which ends with the gate requested on. ++ */ ++static int g5p_kq_cycle(void *ctx) ++{ ++ struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ ++ g5p_mmio_gate(g); ++ return g5p_r33_gfx_off(g->dev, g->pmgr, G5P_KQ_OFF_MS); ++} ++ ++static int g5p_kq_run(struct g5p_diag *g, const char *why) ++{ ++ struct g5p_ru_kq_ops o = { ++ .admit_close = g5p_kq_admit_close, ++ .lock = g5p_kq_lock, ++ .unlock = g5p_kq_unlock, ++ .inputs = g5p_kq_inputs, ++ .detach = g5p_kq_detach, ++ .freeze = g5p_kq_freeze, ++ .irq_off = g5p_kq_irq_off, ++ .cycle = g5p_kq_cycle, ++ }; ++ struct g5p_ru_kq_out out; ++ struct g5p_kq_ctx *c; ++ u64 t0 = ktime_get_ns(); ++ ++ if (atomic_xchg(&g5p_kq_ran, 1)) { ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s already ran this boot; nothing done end=kq\n", ++ why); ++ return -EALREADY; ++ } ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ if (!c) { ++ dev_emerg(g->dev, "GPU-KQ why=%s ret=%d end=kq\n", why, ++ -ENOMEM); ++ return -ENOMEM; ++ } ++ c->g = g; ++ o.ctx = c; ++ g5p_ru_kq_run(&o, min((G5P_RU_KQ_FULL), G5P_RU_KQ_FULL), &out); ++ dev_emerg( ++ g->dev, ++ "GPU-KQ why=%s mode=%u inside=%u locked=%u booted=%u failed=%u wedged=%u " ++ "ram_only=%u exec=%u dc=%u/%u held=%u plan=%s%s skip=%#x detach=%d froze=%u " ++ "cycle=%d ms=%llu end=kq\n", ++ why, out.in.mode, out.inside, out.in.locked, out.in.booted, ++ out.in.failed, out.in.term_failed, out.in.ram_only, ++ out.in.executing, out.in.cons, out.in.prod, out.in.held, ++ out.plan & G5P_RU_KQ_DETACH ? "detach" : "-", ++ out.plan & G5P_RU_KQ_CYCLE ? "+cycle" : "", out.why, ++ out.detach_ret, out.froze, out.cycle_ret, ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC); ++ kfree(c); ++ return out.detach_ret ?: out.cycle_ret; ++} ++ ++/* sysfs `command`: td-capture / td-final / td-kexec. Runs before the command mutex, like ++ * r73b-export, so a wedged stage cannot block it. -EAGAIN (clean, GPU left as ++ * found) is reported as success; the verdict is in the TD98 lines. td-kexec is the ++ * reboot quiesce run by hand (the GPU is finished for this boot afterwards). ++ */ ++static int g5p_td_command(struct g5p_diag *g, const char *command) ++{ ++ int ret; ++ ++ if (sysfs_streq(command, "td-capture")) ++ ret = g5p_td_capture(g, "sysfs"); ++ else if (sysfs_streq(command, "td-final")) ++ ret = g5p_td_final(g, "sysfs"); ++ else if (sysfs_streq(command, "td-kexec")) ++ ret = g5p_kq_run(g, "sysfs"); ++ else ++ return -EINVAL; ++ return ret == -EAGAIN ? 0 : ret; ++} ++ ++/* Runs from kernel_restart_prepare() for reboot, halt, power-off and kexec, i.e. ++ * before device_shutdown() and its unbounded wait_for_device_probe(). Panic kexec ++ * skips both and is not covered. ++ */ ++static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, ++ void *data) ++{ ++ struct g5p_diag *g = READ_ONCE(g5p_td_dev); ++ ++ if (!g) ++ return NOTIFY_DONE; ++ if (action != SYS_RESTART && action != SYS_HALT && ++ action != SYS_POWER_OFF) ++ return NOTIFY_DONE; ++ /* The quiesce needs a live firmware for its detach, so it runs before the ladder freezes. */ ++ g5p_kq_run(g, "reboot-notifier"); ++ { ++ return NOTIFY_DONE; ++ } ++} ++ ++static struct notifier_block g5p_td_nb = { ++ .notifier_call = g5p_td_reboot_notify, ++ .priority = INT_MAX, /* before every other reboot notifier */ ++}; ++ ++/* g is devm-allocated and the DRM frontend's probe error path unwinds without ++ * g5p_remove(): the notifier must not outlive g. ++ */ ++static void g5p_td_devres_release(void *data) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ WRITE_ONCE(g5p_td_dev, NULL); ++} ++ ++static void g5p_td_attach(struct g5p_diag *g) ++{ ++ g5p_td_init(&g5p_td_state); ++ /* A probe after a retire (not reachable on the built-in driver) never reopens admission. */ ++ if (!g5p_r8_retired(&g5p_r8_state)) { ++ u32 gen = g5p_r8_state.gen; ++ ++ g5p_r8_init(&g5p_r8_state); ++ g5p_r8_state.gen = gen; /* monotonic across a re-probe */ ++ } ++ WRITE_ONCE(g5p_td_dev, g); ++ register_reboot_notifier(&g5p_td_nb); ++ /* On failure the action has already run: no notifier, teardown stays inert. */ ++ if (devm_add_action_or_reset(g->dev, g5p_td_devres_release, g)) ++ dev_warn(g->dev, "teardown not attached\n"); ++} ++ ++/* Returns true when the PM reference must be kept: a PMGR power-off was never ++ * validated on the outgoing side and the stop leg is not isStableOff. ++ */ ++static bool g5p_td_remove(struct g5p_diag *g) ++{ ++ unregister_reboot_notifier(&g5p_td_nb); ++ ; ++ ; ++ WRITE_ONCE(g5p_td_dev, NULL); ++ return (0) != 0; ++} ++ ++/* The .shutdown platform callback: second chance after the notifier. */ ++static void __maybe_unused g5p_td_shutdown(struct g5p_diag *g) ++{ ++ /* A second chance after the notifier (a no-op once it ran). */ ++ if (!atomic_read(&g5p_kq_ran)) ++ g5p_kq_run(g, "shutdown"); ++ { ++ ; ++ ; ++ }; ++} +diff --git a/drivers/gpu/apple-g5p/transport-budget.inc b/drivers/gpu/apple-g5p/transport-budget.inc +new file mode 100644 +index 000000000..d70e3c6e1 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/transport-budget.inc +@@ -0,0 +1,6 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#include "vm-mutation.h" ++static u32 g5p_transport_event_limit(void) ++{ ++ return 256 + 131072 * 4; ++} +diff --git a/drivers/gpu/apple-g5p/vm-mutation.h b/drivers/gpu/apple-g5p/vm-mutation.h +new file mode 100644 +index 000000000..4645240bd +--- /dev/null ++++ b/drivers/gpu/apple-g5p/vm-mutation.h +@@ -0,0 +1,36 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_VM_MUTATION_H ++#define G5P_VM_MUTATION_H ++struct g5p_vm_mutation { ++ bool generations; ++ u32 generation[3]; ++}; ++ ++/* Count actual changed descriptors, including clears, not dispatches. Repeated ++ * parent writes of identical bytes do not advance a generation. Packed fields ++ * follow iOS6cabf40. Publication is serialized by the existing command lock. ++ */ ++static void g5p_vm_entry(struct g5p_vm_mutation *c, void *at, u64 value, ++ u32 level) ++{ ++ u64 old = g5p_le32(at); ++ ++ if (level) ++ old |= (u64)g5p_le32(at + 4) << 32; ++ if (old == value) ++ return; ++ if (level) ++ g5p_put64(at, value); ++ else ++ g5p_put32(at, value); ++ c->generation[level]++; ++} ++ ++static u32 g5p_vm_generation(struct g5p_vm_mutation *c) ++{ ++ if (!c->generations) ++ return 0; ++ return ((c->generation[0] & 0x3ff) << 22) | ++ ((c->generation[1] & 0x3ff) << 12) | (c->generation[2] & 0xfff); ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/work-layout.h b/drivers/gpu/apple-g5p/work-layout.h +new file mode 100644 +index 000000000..b5dc73725 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/work-layout.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_WORK_LAYOUT_H ++#define G5P_WORK_LAYOUT_H ++ ++/* Experimental first CL queue probe: c89420/c88840/c8a808/c980a0. ++ * Tag 4 is emitted for dependencies by c8a808. Treating an already satisfied ++ * dependency as a NOP is inferred and must be validated after G2. ++ */ ++#define G5P_WORK_OFFSET 0x54000U ++#define G5P_WORK_BYTES 0x400U ++#define G5P_WORK_CTL 0x100U ++#define G5P_WORK_RING 0x200U ++#define G5P_WORK_CMD 0x300U ++#define G5P_WORK_STAMP 0x340U ++#define G5P_WORK_SLOTS 16U ++ ++/* Isolated visibility hypothesis, not the recovered normal host alias. */ ++ ++/* Firmware10004e94 copies notification word2 into ctl+30. */ ++ ++#endif diff --git a/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch b/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch new file mode 100644 index 00000000..8cdb117a --- /dev/null +++ b/kernel/patches/drivers/drm/0012-drm-apple-g5p-add-buffer-VM-and-cache-visibility-sup.patch @@ -0,0 +1,1027 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add buffer VM and cache visibility support + +Preserve the sealed mapping budgets, PRIME and cache policy. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/cache-policy.h | 13 + + drivers/gpu/drm/apple-g5p/cache-receipt.inc | 38 + + drivers/gpu/drm/apple-g5p/drm-vm.inc | 912 ++++++++++++++++++++ + drivers/gpu/drm/apple-g5p/vm-ranges.h | 21 + + 4 files changed, 984 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/cache-policy.h + create mode 100644 drivers/gpu/drm/apple-g5p/cache-receipt.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-vm.inc + create mode 100644 drivers/gpu/drm/apple-g5p/vm-ranges.h + +diff --git a/drivers/gpu/drm/apple-g5p/cache-policy.h b/drivers/gpu/drm/apple-g5p/cache-policy.h +new file mode 100644 +index 000000000..40f5cdfe1 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-policy.h +@@ -0,0 +1,13 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* No caller may infer CPU caching from BO size or GPU use alone. */ ++#ifndef G5P_CACHE_POLICY_H ++#define G5P_CACHE_POLICY_H ++static inline int g5p_cache_class(int imported, int exported, int wc) ++{ ++ return imported ? 2 : (!exported && wc ? 0 : 1); ++} ++static inline int g5p_cache_omit(int enabled, int imported, int exported, int wc) ++{ ++ return enabled && g5p_cache_class(imported, exported, wc) == 0; ++} ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/cache-receipt.inc b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +new file mode 100644 +index 000000000..1efeecd0c +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/cache-receipt.inc +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Bounded RAM-only records. No register accesses, default off. */ ++#include ++#include "cache-policy.h" ++/* Qualified by kernel-cost2b and combined-bench2c; conservative off switches remain. */ ++static const bool render_user_cache_import_wc = true; ++static const bool render_user_cache_read_va = true; ++static const bool render_user_cache_private_wc = true; ++static const bool render_user_cache_receipts; ++struct g5p_cache_receipt { ++ u64 job, start_cpu, start_wall, total_cpu, total_wall; ++ u64 requested[2][3], maintained[2][3], cpu[2][3], wall[2][3]; ++ u32 pid, serial; ++ bool enabled, policy, import_policy; ++}; ++#define G5P_CACHE_RECORDS 128 ++static struct g5p_cache_receipt g5p_cache_ring[G5P_CACHE_RECORDS]; ++static DEFINE_SPINLOCK(g5p_cache_ring_lock); ++static u32 g5p_cache_head; ++static int g5p_cache_show(struct seq_file *m, void *data) ++{ ++ struct g5p_cache_receipt r; ++ u32 head = READ_ONCE(g5p_cache_head), begin = head > G5P_CACHE_RECORDS ? head - G5P_CACHE_RECORDS : 0; ++ unsigned long flags; ++ seq_puts(m, "v=1 classes=private-wc,native-other,import time=task-runtime-ns wall=elapsed-ns\n"); ++ for (u32 n = begin; n < head; n++) { ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ r = g5p_cache_ring[n % G5P_CACHE_RECORDS]; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ if (r.serial != n + 1) continue; /* overwritten during this debugfs read */ ++ seq_printf(m, "job=%llu pid=%u policy=%u total_cpu_ns=%llu total_wall_ns=%llu", r.job, r.pid, r.policy, r.total_cpu, r.total_wall); ++ for (u32 phase = 0; phase < 2; phase++) ++ for (u32 c = 0; c < 3; c++) ++ seq_printf(m, " p%uc%u=%llu,%llu,%llu,%llu", phase, c, r.requested[phase][c], r.maintained[phase][c], r.cpu[phase][c], r.wall[phase][c]); ++ seq_putc(m, '\n'); ++ } ++ return 0; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-vm.inc b/drivers/gpu/drm/apple-g5p/drm-vm.inc +new file mode 100644 +index 000000000..644554156 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-vm.inc +@@ -0,0 +1,912 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* 16 KiB VM preparation; accepted user VMs freeze until reboot. */ ++ ++/* release8 memory accounting: every live G5P GEM object (native and imported) with the task ++ * that created it. One line per context release and per file close prints the totals; ++ * debugfs g5p_bos lists each object (size, references, handles, exported/imported, VM pins, ++ * creator) and each open file's handles, so a page held after its client exited is named. ++ */ ++static LIST_HEAD(g5p_bo_live); ++static DEFINE_SPINLOCK(g5p_bo_lock); ++static u32 g5p_bo_count; ++static u64 g5p_bo_bytes; ++ ++static void g5p_gem_free(struct drm_gem_object *obj); ++ ++static const struct drm_gem_object_funcs g5p_gem_funcs = { ++ .free = g5p_gem_free, ++ .print_info = drm_gem_shmem_object_print_info, ++ .pin = drm_gem_shmem_object_pin, ++ .unpin = drm_gem_shmem_object_unpin, ++ .get_sg_table = drm_gem_shmem_object_get_sg_table, ++ .vmap = drm_gem_shmem_object_vmap, ++ .vunmap = drm_gem_shmem_object_vunmap, ++ .mmap = drm_gem_shmem_object_mmap, ++ .export = g5p_gem_prime_export, ++ .vm_ops = &drm_gem_shmem_vm_ops, ++}; ++ ++/* perf60 fault address (render_user_fault_read, drm-render-v3.inc): bit 0 reads the MMU fault ++ * banks at a TA/3D fault event, bit 1 also at a silent or timed-out phase; any nonzero value also ++ * records every VM edit (MAP, UNMAP, render-target create/destroy) in a device-wide ring, so the ++ * fault address can be named. The ring holds an opaque VM pointer, never dereferenced. ++ */ ++ ++/* faultrec: the bank read and its decode alone at a TA/3D fault event (no per-job PM lanes, no ++ * PM snapshots, no VM edit log): one licensed read per fault, at most 32 per boot. ++ */ ++ ++/* perf60-fillrate.md §11.3 cut 5, kernel half (render-user-sync.h): cache maintenance by range. */ ++static const unsigned int render_user_sync_range = 3; /* release8 default */ ++MODULE_PARM_DESC( ++ render_user_sync_range, ++ "perf60 default0 (release8: whole-BO maintenance); bit0 BO_SYNC honours " ++ "offset/size; bit1 a job's ACCESS_CACHE clean/invalidate covers each resource's " ++ "BO range [offset, offset+size), page widened, instead of the whole BO (latched " ++ "per job at submit); BO_SYNC read live"); ++ ++/* CPU cache maintenance of [lo, hi) of a BO's DMA mapping, walked over its DMA segments. */ ++static void g5p_sync_sgt_range(struct device *dev, struct sg_table *sgt, u64 lo, ++ u64 hi, bool to_cpu) ++{ ++ struct scatterlist *sg; ++ u64 pos = 0, start, n; ++ unsigned int i; ++ ++ for_each_sgtable_dma_sg(sgt, sg, i) { ++ if (pos >= hi) ++ break; ++ if (g5p_sync_clip(pos, sg_dma_len(sg), lo, hi, &start, &n)) { ++ if (to_cpu) ++ dma_sync_single_for_cpu( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_single_for_device( ++ dev, sg_dma_address(sg) + start, n, ++ DMA_BIDIRECTIONAL); ++ } ++ pos += sg_dma_len(sg); ++ } ++} ++ ++#define G5P_VMLOG_N 1024U ++enum { G5P_VMLOG_MAP = 1, G5P_VMLOG_UNMAP, G5P_VMLOG_RT_NEW, G5P_VMLOG_RT_DEL }; ++static const char *const g5p_vmlog_ops[] = { "?", "map", "unmap", "rt-new", ++ "rt-del" }; ++ ++struct g5p_vmlog_ent { ++ u64 ns, va, size, offset; ++ const void *vm; ++ u32 op, handle, vm_id; ++ pid_t tgid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++static struct { ++ spinlock_t lock; /* head and the entries */ ++ u32 head; ++ struct g5p_vmlog_ent e[G5P_VMLOG_N]; ++} g5p_vmlog = { .lock = __SPIN_LOCK_UNLOCKED(g5p_vmlog.lock) }; ++ ++static void g5p_vmlog_add(u32 op, const void *vm, u32 vm_id, u64 va, u64 size, ++ u32 handle, u64 offset) ++{ ++ { ++ return; ++ } ++} ++ ++/* Latency123: leaf attributes of new user mappings (0 = Apple's CC | SLC bypass). */ ++ ++/* Stage 3: the device-wide pinned-memory budget. */ ++static const unsigned int render_pinned_mib = 1024; ++MODULE_PARM_DESC( ++ render_pinned_mib, ++ "MiB of BO memory all GPU VMs together may map; an object mapped in several VMs " ++ "or files (PRIME) counts once (default 1024)"); ++ ++static u32 g5p_pte_attrs(void) ++{ ++ static const u32 attrs[] = { 0xc, 0x4, 0x8, 0x0 }; ++ ++ return attrs[min_t(u32, (0), ARRAY_SIZE(attrs) - 1)]; ++} ++ ++static struct drm_gem_object *g5p_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct g5p_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); ++ ++ if (!bo) ++ return ERR_PTR(-ENOMEM); ++ bo->shmem.base.funcs = &g5p_gem_funcs; ++ INIT_LIST_HEAD(&bo->live); ++ return &bo->shmem.base; ++} ++ ++/* A fully created object enters the live list (an object the core frees on a failed init ++ * never does). ++ */ ++static void g5p_bo_track(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ bo->pid = current->tgid; ++ strscpy(bo->comm, current->comm, sizeof(bo->comm)); ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_add_tail(&bo->live, &g5p_bo_live); ++ g5p_bo_count++; ++ g5p_bo_bytes += obj->size; ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++} ++ ++static void g5p_gem_free(struct drm_gem_object *obj) ++{ ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ if (!list_empty(&bo->live)) { ++ list_del_init(&bo->live); ++ g5p_bo_count--; ++ g5p_bo_bytes -= obj->size; ++ } ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ drm_gem_shmem_object_free(obj); ++} ++ ++static int g5p_bos_show(struct seq_file *m, void *data) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct drm_device *drm = entry->dev; ++ u32 exported = 0, imported = 0, nohandle = 0; ++ u64 nohandle_bytes = 0; ++ struct drm_file *file; ++ struct g5p_bo *bo; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&g5p_bo_lock, flags); ++ list_for_each_entry(bo, &g5p_bo_live, live) { ++ struct drm_gem_object *obj = &bo->shmem.base; ++ bool imp = drm_gem_is_imported(obj); ++ ++ exported += !!READ_ONCE(obj->dma_buf); ++ imported += imp; ++ if (!READ_ONCE(obj->handle_count)) { ++ nohandle++; ++ nohandle_bytes += obj->size; ++ } ++ seq_printf( ++ m, ++ "bo size=%zuK refs=%u handles=%u exported=%c imported=%c vm_pins=%u by=%s[%d]\n", ++ obj->size >> 10, kref_read(&obj->refcount), ++ READ_ONCE(obj->handle_count), ++ READ_ONCE(obj->dma_buf) ? 'y' : 'n', imp ? 'y' : 'n', ++ bo->vm_pins, bo->comm, bo->pid); ++ } ++ seq_printf( ++ m, ++ "live=%u bytes=%lluK exported=%u imported=%u no_handle=%u/%lluK device_pinned=%lluK\n", ++ g5p_bo_count, g5p_bo_bytes >> 10, exported, imported, nohandle, ++ nohandle_bytes >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++ spin_unlock_irqrestore(&g5p_bo_lock, flags); ++ mutex_lock(&drm->filelist_mutex); ++ list_for_each_entry(file, &drm->filelist, lhead) { ++ struct drm_gem_object *obj; ++ struct task_struct *task; ++ u64 bytes = 0; ++ u32 handles = 0; ++ struct pid *pid; ++ int id; ++ ++ spin_lock(&file->table_lock); ++ idr_for_each_entry(&file->object_idr, obj, id) { ++ handles++; ++ bytes += obj->size; ++ } ++ spin_unlock(&file->table_lock); ++ rcu_read_lock(); ++ pid = rcu_dereference(file->pid); ++ task = pid_task(pid, PIDTYPE_TGID); ++ seq_printf(m, "file %s[%d] handles=%u bytes=%lluK\n", ++ task ? task->comm : "?", pid_vnr(pid), handles, ++ bytes >> 10); ++ rcu_read_unlock(); ++ } ++ mutex_unlock(&drm->filelist_mutex); ++ return 0; ++} ++ ++static int g5p_create_bo(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_bo *a = data; ++ struct drm_gem_shmem_object *shmem; ++ int ret; ++ ++ if (!a->size || a->size > G5P_MAX_BO_SIZE || ++ (a->size & (G5P_GPU_PAGE - 1)) || a->handle || !a->flags || ++ (a->flags & ++ ~(DRM_G5P_BO_READ | DRM_G5P_BO_WRITE | DRM_G5P_BO_EXEC))) ++ return -EINVAL; ++ shmem = drm_gem_shmem_create(drm, a->size); ++ if (IS_ERR(shmem)) ++ return PTR_ERR(shmem); ++ /* Match the DMA API's coherent/noncoherent CPU mapping choice. The GPU ++ * cache contract remains an explicit COMPUTE capability prerequisite. ++ */ ++ shmem->map_wc = !dev_is_dma_coherent(drm->dev); ++ to_g5p_bo(&shmem->base)->roles = a->flags; ++ g5p_bo_track(&shmem->base); ++ ret = drm_gem_handle_create(file, &shmem->base, &a->handle); ++ drm_gem_object_put(&shmem->base); ++ return ret; ++} ++ ++static int g5p_bo_mmap_offset(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_mmap_offset *a = data; ++ struct drm_gem_object *obj; ++ int ret; ++ ++ if (a->flags || a->offset) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ ret = drm_gem_create_mmap_offset(obj); ++ if (!ret) ++ a->offset = drm_vma_node_offset_addr(&obj->vma_node); ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_bo_sync(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_bo_sync *a = data; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *ctx; ++ unsigned long id; ++ int ret = 0; ++ ++ if (a->op != DRM_G5P_BO_SYNC_TO_DEVICE && ++ a->op != DRM_G5P_BO_SYNC_TO_CPU) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, a->handle); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ mutex_lock(&f->lock); ++ xa_for_each(&f->contexts, id, ctx) ++ if (ctx->serial && ++ (ctx->bad || smp_load_acquire(&ctx->pending))) { ++ mutex_unlock(&f->lock); ++ ret = -EBUSY; ++ goto out; ++ } ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ mutex_unlock(&f->lock); ++ goto out; ++ } ++ /* render_user_sync_range bit 0: the requested range; else the whole BO (stronger, allowed). */ ++ if (READ_ONCE(render_user_sync_range) & 1) { ++ u64 lo, hi; ++ ++ if (g5p_sync_window(a->offset, a->size, obj->size, &lo, &hi)) ++ g5p_sync_sgt_range(drm->dev, sgt, lo, hi, ++ a->op == DRM_G5P_BO_SYNC_TO_CPU); ++ } else if (a->op == DRM_G5P_BO_SYNC_TO_DEVICE) { ++ dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } else { ++ dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ mutex_unlock(&f->lock); ++out: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_alloc_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ p->cpu = dma_alloc_coherent(dev, G5P_GPU_PAGE, &p->dma, GFP_KERNEL); ++ if (!p->cpu) ++ return -ENOMEM; ++ if (!g5p_user_pa_valid(p->dma) || dma_to_phys(dev, p->dma) != p->dma) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++ return -ERANGE; ++ } ++ memset(p->cpu, 0, G5P_GPU_PAGE); ++ return 0; ++} ++ ++static struct g5p_pt *g5p_vm_table(struct g5p_vm *vm, struct device *dev, ++ u32 level, u32 key, bool create) ++{ ++ struct g5p_pt *pt; ++ int ret; ++ ++ list_for_each_entry(pt, &vm->tables, link) ++ if (pt->level == level && pt->key == key) ++ return pt; ++ if (!create) ++ return NULL; ++ if (vm->table_count >= G5P_MAX_PT_PAGES) ++ return ERR_PTR(-ENOSPC); ++ pt = kzalloc(sizeof(*pt), GFP_KERNEL); ++ if (!pt) ++ return ERR_PTR(-ENOMEM); ++ ret = g5p_vm_alloc_page(dev, &pt->page); ++ if (ret) { ++ kfree(pt); ++ return ERR_PTR(ret); ++ } ++ pt->level = level; ++ pt->key = key; ++ list_add_tail(&pt->link, &vm->tables); ++ vm->table_count++; ++ return pt; ++} ++ ++static u8 *g5p_vm_pte(struct g5p_vm *vm, struct device *dev, u64 va, ++ bool create) ++{ ++ u32 ci = (va >> 30) & 1023, di = (va >> 21) & 511; ++ struct g5p_pt *dir, *leaf; ++ ++ dir = g5p_vm_table(vm, dev, 1, ci, create); ++ if (IS_ERR_OR_NULL(dir)) ++ return (void *)dir; ++ leaf = g5p_vm_table(vm, dev, 2, (ci << 9) | di, create); ++ if (IS_ERR_OR_NULL(leaf)) ++ return (void *)leaf; ++ if (create) { ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + ci * 4, ++ g5p_catalogue_entry(dir->page.dma), 0); ++ g5p_vm_entry(&vm->mutation, dir->page.cpu + di * 8, ++ g5p_directory_entry(leaf->page.dma), 1); ++ } ++ return leaf->page.cpu + ((va >> 14) & 127) * 8; ++} ++ ++static void g5p_vm_clear(struct g5p_vm *vm, struct device *dev, u64 va, ++ u64 size) ++{ ++ for (u64 off = 0; off < size; off += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, dev, va + off, false); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, 0, 2); ++ } ++ dma_wmb(); ++} ++ ++static bool g5p_vm_table_empty(struct g5p_pt *pt) ++{ ++ u32 count = pt->level == 2 ? 128 : 512; ++ ++ for (u32 i = 0; i < count; i++) ++ if (g5p_le32(pt->page.cpu + i * 8) & 1) ++ return false; ++ return true; ++} ++ ++static void g5p_vm_prune(struct device *dev, struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt, *tmp, *dir; ++ ++ /* These tables are mutable only before firmware attachment. ++ * Remove empty leaves before directories, including failed MAP rollback. ++ */ ++ for (u32 level = 2; level; level--) ++ list_for_each_entry_safe(pt, tmp, &vm->tables, link) { ++ if (pt->level != level || !g5p_vm_table_empty(pt)) ++ continue; ++ if (level == 2) { ++ dir = g5p_vm_table(vm, dev, 1, pt->key >> 9, ++ false); ++ if (dir) ++ g5p_vm_entry(&vm->mutation, ++ dir->page.cpu + (pt->key & ++ 511) * 8, ++ 0, 1); ++ } else { ++ g5p_vm_entry(&vm->mutation, ++ vm->root.cpu + pt->key * 4, 0, 0); ++ } ++ dma_wmb(); ++ list_del(&pt->link); ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ vm->table_count--; ++ kfree(pt); ++ } ++} ++ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj); ++ ++static void g5p_vm_release(struct kref *ref) ++{ ++ struct g5p_vm *vm = container_of(ref, struct g5p_vm, ref); ++ struct device *dev = vm->dev; ++ struct g5p_mapping *m, *mtmp; ++ struct g5p_pt *pt, *ptmp; ++ u64 pinned = vm->pinned_bytes; ++ u32 maps = vm->map_count, tables = vm->table_count; ++ ++ list_for_each_entry_safe(m, mtmp, &vm->maps, link) { ++ drm_mm_remove_node(&m->node); ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ list_del(&m->link); ++ kfree(m); ++ } ++ list_for_each_entry_safe(pt, ptmp, &vm->tables, link) { ++ dma_free_coherent(dev, G5P_GPU_PAGE, pt->page.cpu, ++ pt->page.dma); ++ list_del(&pt->link); ++ kfree(pt); ++ } ++ dma_free_coherent(dev, G5P_GPU_PAGE, vm->root.cpu, vm->root.dma); ++ xa_destroy(&vm->objs); ++ drm_mm_remove_node(&vm->reserved); ++ drm_mm_takedown(&vm->mm); ++ /* One line per freed VM: the device's pinned total must fall back after a client exits. */ ++ dev_info( ++ dev, ++ "G5P VM release maps=%u tables=%u pinned=%lluK device_pinned=%lluK\n", ++ maps, tables, pinned >> 10, ++ READ_ONCE(vm->gpu->pinned_bytes) >> 10); ++ kfree(vm); ++} ++ ++static void g5p_vm_put(struct g5p_vm *vm) ++{ ++ kref_put(&vm->ref, g5p_vm_release); ++} ++ ++static void g5p_context_release(struct kref *ref) ++{ ++ struct g5p_context *c = container_of(ref, struct g5p_context, ref); ++ g5p_render_release(c); ++ g5p_vm_put(c->vm); ++ kfree(c); ++} ++ ++static void g5p_context_put(struct g5p_context *c) ++{ ++ kref_put(&c->ref, g5p_context_release); ++} ++ ++static int g5p_create_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ u32 id; ++ int ret; ++ ++ if (a->flags || a->handle) ++ return -EINVAL; ++ vm = kzalloc(sizeof(*vm), GFP_KERNEL); ++ if (!vm) ++ return -ENOMEM; ++ kref_init(&vm->ref); ++ vm->dev = drm->dev; ++ vm->gpu = to_g5p(drm); ++ vm->bounded = (0); ++ vm->mutation.generations = true; ++ INIT_LIST_HEAD(&vm->maps); ++ xa_init(&vm->objs); ++ INIT_LIST_HEAD(&vm->tables); ++ drm_mm_init(&vm->mm, G5P_USER_VA_START, ++ G5P_USER_VA_END - G5P_USER_VA_START); ++ vm->reserved.start = G5P_USER_RESERVED_START; ++ vm->reserved.size = G5P_USER_RESERVED_END - G5P_USER_RESERVED_START; ++ ret = drm_mm_reserve_node(&vm->mm, &vm->reserved); ++ if (ret) ++ goto err_mm; ++ ret = g5p_vm_alloc_page(drm->dev, &vm->root); ++ if (ret) ++ goto err_reserved; ++ vm->table_count = 1; ++ mutex_lock(&f->lock); ++ ret = xa_alloc(&f->vms, &id, vm, XA_LIMIT(1, G5P_MAX_VMS), GFP_KERNEL); ++ mutex_unlock(&f->lock); ++ if (ret) { ++ g5p_vm_put(vm); ++ return ret; ++ } ++ a->handle = id; ++ return 0; ++err_reserved: ++ drm_mm_remove_node(&vm->reserved); ++err_mm: ++ drm_mm_takedown(&vm->mm); ++ kfree(vm); ++ return ret; ++} ++ ++static int g5p_destroy_vm(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_vm *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_vm *vm; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->handle); ++ if (!vm) ++ ret = -ENOENT; ++ else if (vm->contexts || f->submitted || vm->frozen) ++ ret = -EBUSY; ++ else ++ xa_erase(&f->vms, a->handle); ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_vm_put(vm); ++ return ret; ++} ++ ++static unsigned long g5p_vm_obj_count(struct g5p_vm *vm, ++ struct drm_gem_object *obj) ++{ ++ void *v = xa_load(&vm->objs, (unsigned long)obj); ++ ++ return v ? xa_to_value(v) : 0; ++} ++ ++/* ++ * One more mapping of `obj` in `vm`. Its first one in this VM counts its size against the ++ * VM's pin limit, and its first one in any VM against the device budget (render_pinned_mib). ++ * Our own dma-bufs import back as the same GEM object, in any file, so memory shared between ++ * clients (a client's buffer in the compositor's VM) counts once. ++ */ ++static int g5p_vm_obj_get(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ u64 limit = (u64)READ_ONCE(render_pinned_mib) << 20; ++ unsigned long count; ++ int ret = 0; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (!count && obj->size > G5P_MAX_PINNED_BYTES - vm->pinned_bytes) ++ ret = -ENOSPC; ++ else if (!count && !bo->vm_pins && ++ obj->size > limit - min(limit, d->pinned_bytes)) ++ ret = -ENOSPC; ++ if (!ret) ++ ret = xa_err(xa_store(&vm->objs, (unsigned long)obj, ++ xa_mk_value(count + 1), GFP_KERNEL)); ++ if (!ret && !count) { ++ vm->pinned_bytes += obj->size; ++ if (!bo->vm_pins++) ++ d->pinned_bytes += obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++ return ret; ++} ++ ++/* One mapping of `obj` in `vm` less: g5p_vm_obj_get() undone. Also called by the render ++ * scheduler for an unmap deferred behind queued jobs, hence the lock around the counts. ++ */ ++static void g5p_vm_obj_put(struct g5p_vm *vm, struct drm_gem_object *obj) ++{ ++ struct g5p_drm *d = to_g5p(obj->dev); ++ struct g5p_bo *bo = to_g5p_bo(obj); ++ unsigned long count; ++ ++ mutex_lock(&d->pin_lock); ++ count = g5p_vm_obj_count(vm, obj); ++ if (count > 1) { ++ /* replaces an existing entry: no allocation */ ++ xa_store(&vm->objs, (unsigned long)obj, xa_mk_value(count - 1), ++ GFP_KERNEL); ++ } else if (!WARN_ON(!count)) { ++ xa_erase(&vm->objs, (unsigned long)obj); ++ vm->pinned_bytes -= obj->size; ++ if (!--bo->vm_pins) ++ d->pinned_bytes -= obj->size; ++ } ++ mutex_unlock(&d->pin_lock); ++} ++ ++static int g5p_vm_map_bo(struct drm_device *drm, struct g5p_vm *vm, ++ struct drm_file *file, struct drm_g5p_vm_bind *a) ++{ ++ struct drm_gem_object *obj; ++ struct drm_gem_shmem_object *shmem; ++ struct sg_table *sgt; ++ struct sg_dma_page_iter iter; ++ struct g5p_mapping *m; ++ u64 *addresses, done = 0; ++ u32 count = 0; ++ bool imported; ++ int ret; ++ ++ if (vm->map_count >= G5P_MAX_MAPS || ++ a->size > G5P_MAX_VM_BYTES - vm->mapped_bytes) ++ return -ENOSPC; ++ obj = drm_gem_object_lookup(file, a->bo); ++ if (!obj) ++ return -ENOENT; ++ if (!g5p_bo_range_valid(a->offset, a->size, obj->size)) { ++ ret = -EINVAL; ++ goto err_obj; ++ } ++ ret = g5p_vm_obj_get(vm, obj); ++ if (ret) ++ goto err_obj; ++ addresses = kvmalloc_array(obj->size / G5P_GPU_PAGE, sizeof(*addresses), ++ GFP_KERNEL); ++ m = kzalloc(sizeof(*m), GFP_KERNEL); ++ if (!addresses || !m) { ++ ret = -ENOMEM; ++ goto err_alloc; ++ } ++ m->node.start = a->va; ++ m->node.size = a->size; ++ ret = drm_mm_reserve_node(&vm->mm, &m->node); ++ if (ret) { ++ ret = -EINVAL; ++ goto err_alloc; ++ } ++ imported = drm_gem_is_imported(obj); ++ shmem = to_drm_gem_shmem_obj(obj); ++ /* PRIME import mapped this attachment for our device and owns it until ++ * object release. Re-locking the exporter's reservation here can return ++ * -EBUSY after the first V3 submission even though the render VM is idle. ++ */ ++ if (imported) { ++ sgt = READ_ONCE(shmem->sgt); ++ if (!sgt) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=prime-no-sgt vm=%u bo=%u bytes=%llu\n", ++ a->vm, a->bo, a->size); ++ ret = -ENXIO; ++ goto err_node; ++ } ++ } else { ++ sgt = drm_gem_shmem_get_pages_sgt(shmem); ++ } ++ if (IS_ERR(sgt)) { ++ ret = PTR_ERR(sgt); ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND map=sgt-error vm=%u bo=%u imported=%u err=%d\n", ++ a->vm, a->bo, imported, ret); ++ goto err_node; ++ } ++ for_each_sgtable_dma_page(sgt, &iter, 0) { ++ u64 dma = sg_page_iter_dma_address(&iter); ++ ++ if (count >= obj->size / G5P_GPU_PAGE || ++ !g5p_user_pa_valid(dma) || ++ dma_to_phys(drm->dev, dma) != dma || ++ (!imported && dma_to_phys(drm->dev, dma) != ++ page_to_phys(shmem->pages[count]))) { ++ ret = -ERANGE; ++ goto err_node; ++ } ++ addresses[count++] = dma; ++ } ++ if (count != obj->size / G5P_GPU_PAGE) { ++ ret = -EINVAL; ++ goto err_node; ++ } ++ for (; done < a->size; done += G5P_GPU_PAGE) { ++ u8 *pte = g5p_vm_pte(vm, drm->dev, a->va + done, true); ++ ++ if (IS_ERR_OR_NULL(pte)) { ++ ret = pte ? PTR_ERR(pte) : -ENOMEM; ++ goto err_clear; ++ } ++ /* Recovered bypass bits; user permission encodings are not claimed. ++ * A render VM latched to r25 memory variant 4 uses flags 0. ++ */ ++ g5p_vm_entry( ++ &vm->mutation, pte, ++ g5p_leaf_entry( ++ addresses[(a->offset + done) / G5P_GPU_PAGE], ++ vm->zero_leaf_flags ? 0 : g5p_pte_attrs()), ++ 2); ++ } ++ dma_wmb(); ++ m->obj = obj; ++ m->offset = a->offset; ++ list_add_tail(&m->link, &vm->maps); ++ vm->mapped_bytes += a->size; ++ vm->map_count++; ++ kvfree(addresses); ++ return 0; ++err_clear: ++ g5p_vm_clear(vm, drm->dev, a->va, done); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++err_node: ++ drm_mm_remove_node(&m->node); ++err_alloc: ++ kvfree(addresses); ++ kfree(m); ++ g5p_vm_obj_put(vm, obj); ++err_obj: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int g5p_vm_bind(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_vm_bind *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_mapping *m; ++ struct g5p_vm *vm; ++ int ret = -ENOENT; ++ bool resident = false, locked = false; ++ u32 generation[3]; ++ ++ if (a->flags || !g5p_user_range_valid(a->va, a->size) || ++ (a->op != DRM_G5P_VM_MAP && a->op != DRM_G5P_VM_UNMAP) || ++ (a->op == DRM_G5P_VM_UNMAP && (a->bo || a->offset))) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ vm = xa_load(&f->vms, a->vm); ++ if (!vm) ++ goto out; ++ /* Its render context is closing: the close item owns the tables now, and the context ++ * may be freed at any time, so it is not looked at. ++ */ ++ if (vm->resident_gone) { ++ ret = -EBUSY; ++ goto out; ++ } ++ if ((f->submitted || vm->frozen) && !vm->resident) { ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=no-resident submitted=%u frozen=%u vm=%u op=%u\n", ++ f->submitted, vm->frozen, a->vm, a->op); ++ ret = -EBUSY; ++ goto out; ++ } ++ if (vm->resident) { ++ ret = g5p_resident_idle(vm->resident, true); ++ if (ret) ++ goto out; ++ /* A render context's edit is RAM only and touches its own tables and ++ * slots, which the firmware reads only at that context's next kick; ++ * f->lock plus resident-idle orders it against its submit (a MAP or ++ * UNMAP may pass queued jobs, see g5p_render_vm_idle()). Other contexts' jobs ++ * may hold the core lock meanwhile, so it is not taken. Serialized ++ * compute still uses the global mutex. ++ */ ++ if (!vm->resident->render) { ++ mutex_lock(&to_g5p(drm)->core->lock); ++ locked = true; ++ } ++ resident = true; ++ memcpy(generation, vm->mutation.generation, sizeof(generation)); ++ } ++ if (a->op == DRM_G5P_VM_MAP) { ++ if (vm->bounded && !(vm->resident && vm->resident->render) && ++ g5p_user_private_overlap(a->va, a->size)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = g5p_vm_map_bo(drm, vm, file, a); ++ if (!ret) ++ g5p_vmlog_add(G5P_VMLOG_MAP, vm, a->vm, a->va, a->size, ++ a->bo, a->offset); ++ if (ret == -EBUSY) ++ dev_warn_ratelimited( ++ drm->dev, ++ "G5P VM_BIND busy=map vm=%u bo=%u submitted=%u frozen=%u resident=%u render=%u\n", ++ a->vm, a->bo, f->submitted, vm->frozen, ++ !!vm->resident, ++ !!(vm->resident && vm->resident->render)); ++ goto out; ++ } ++ list_for_each_entry(m, &vm->maps, link) { ++ if (m->node.start != a->va || m->node.size != a->size) { ++ if (m->node.start < a->va + a->size && ++ a->va < m->node.start + m->node.size) { ++ ret = -EINVAL; ++ break; ++ } ++ continue; ++ } ++ g5p_vmlog_add(G5P_VMLOG_UNMAP, vm, a->vm, a->va, a->size, 0, 0); ++ g5p_vm_clear(vm, drm->dev, a->va, a->size); ++ if (!vm->resident) ++ g5p_vm_prune(drm->dev, vm); ++ else { ++ ret = g5p_resident_invalidate(vm->resident); ++ if (ret) ++ goto out; ++ memcpy(generation, vm->mutation.generation, ++ sizeof(generation)); ++ } ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ vm->mapped_bytes -= a->size; ++ /* Beside queued jobs the PTEs are gone, but the pages stay pinned until ++ * those jobs have retired. ++ */ ++ if (!vm->resident || !g5p_resident_defer(vm->resident, m)) { ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ ret = 0; ++ break; ++ } ++out: ++ if (resident) { ++ int invalidate = 0; ++ ++ if (memcmp(generation, vm->mutation.generation, ++ sizeof(generation))) ++ invalidate = g5p_resident_invalidate(vm->resident); ++ ++ if (invalidate) ++ ret = invalidate; ++ if (locked) ++ mutex_unlock(&to_g5p(drm)->core->lock); ++ } ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_create_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_context *a = data; ++ if (a->type != DRM_G5P_CONTEXT_RENDER) ++ return -EOPNOTSUPP; ++ return g5p_render_create_context(drm, file, a); ++} ++ ++static int g5p_destroy_context(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_context *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ int ret = 0; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->handle); ++ if (!c) ++ ret = -ENOENT; ++ else ++ ret = g5p_render_destroy(c); ++ if (!ret && c) { ++ c->vm->contexts--; ++ xa_erase(&f->contexts, a->handle); ++ } ++ mutex_unlock(&f->lock); ++ if (!ret) ++ g5p_context_put(c); ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/vm-ranges.h b/drivers/gpu/drm/apple-g5p/vm-ranges.h +new file mode 100644 +index 000000000..25bd98ee9 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/vm-ranges.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_H ++#define G5P_COMPUTE_USER_H ++ ++/* Public geometry is fixed across VMs. The private ranges never alias a BO. */ ++ ++static const u64 g5p_private_va[] = { ++ 0x300040000ULL, 0x300014000ULL, 0x300028000ULL, 0x300044000ULL, 0, ++ 0x100080000ULL, 0x100084000ULL, 0x1000a0000ULL, 0x1000a4000ULL, ++}; ++ ++static inline bool g5p_user_private_overlap(u64 va, u64 size) ++{ ++ for (u32 i = 0; i < ARRAY_SIZE(g5p_private_va); i++) ++ if (g5p_private_va[i] && va < g5p_private_va[i] + 0x4000 && ++ g5p_private_va[i] < va + size) ++ return true; ++ return false; ++} ++ ++#endif diff --git a/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch b/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch new file mode 100644 index 00000000..93dca40f --- /dev/null +++ b/kernel/patches/drivers/drm/0013-drm-apple-g5p-add-typed-render-and-compute-submissio.patch @@ -0,0 +1,8973 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:45 +0000 +Subject: [PATCH] drm/apple-g5p: add typed render and compute submission + +[Orchard: from brkosebay/linux d67f133590a; normal probe registration +and fixed default policy.] Keep the A10 PowerVR driver and its build +hooks unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/Kconfig | 2 + + drivers/gpu/drm/Makefile | 2 + + drivers/gpu/drm/apple-g5p/Kconfig | 14 + + drivers/gpu/drm/apple-g5p/Makefile | 2 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 742 ++++ + .../gpu/drm/apple-g5p/compute-user-validate.h | 141 + + drivers/gpu/drm/apple-g5p/drm-compute-v1.h | 17 + + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 720 ++++ + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 293 ++ + drivers/gpu/drm/apple-g5p/drm-prime.inc | 114 + + .../gpu/drm/apple-g5p/drm-render-sched.inc | 203 ++ + .../gpu/drm/apple-g5p/drm-render-unbind.inc | 488 +++ + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 62 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 3038 +++++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-render.inc | 2549 ++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-submit.inc | 29 + + .../gpu/drm/apple-g5p/render-user-validate.h | 384 +++ + drivers/gpu/drm/apple-g5p/runtime-idle.inc | 14 + + 18 files changed, 8814 insertions(+) + create mode 100644 drivers/gpu/drm/apple-g5p/Kconfig + create mode 100644 drivers/gpu/drm/apple-g5p/Makefile + create mode 100644 drivers/gpu/drm/apple-g5p/a9-gpu-drm.c + create mode 100644 drivers/gpu/drm/apple-g5p/compute-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-compute-v1.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-compute-v1.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-latency123.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-prime.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-sched.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-unbind.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.h + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render-v3.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-render.inc + create mode 100644 drivers/gpu/drm/apple-g5p/drm-submit.inc + create mode 100644 drivers/gpu/drm/apple-g5p/render-user-validate.h + create mode 100644 drivers/gpu/drm/apple-g5p/runtime-idle.inc + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 4c74c1501..eaf71742f 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -371,3 +371,5 @@ endif + # Separate option because drm_panel_orientation_quirks.c is shared with fbdev + config DRM_PANEL_ORIENTATION_QUIRKS + tristate ++ ++source "drivers/gpu/drm/apple-g5p/Kconfig" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index d75eba399..1315aa378 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -257,3 +257,5 @@ quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + + $(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) ++ ++obj-$(CONFIG_DRM_APPLE_G5P) += apple-g5p/ +diff --git a/drivers/gpu/drm/apple-g5p/Kconfig b/drivers/gpu/drm/apple-g5p/Kconfig +new file mode 100644 +index 000000000..1838fc90f +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Kconfig +@@ -0,0 +1,14 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config DRM_APPLE_G5P ++ bool "Apple A9/G5P render driver" ++ depends on DRM=y && ARCH_APPLE && OF && PM && ARM64_16K_PAGES ++ select DRM_GEM_SHMEM_HELPER ++ select FW_LOADER ++ select CRYPTO_LIB_SHA256 ++ help ++ Render-node driver for the S8003 G5P GPU using the unmodified META ++ firmware from the device. Supports TYPED_V3 jobs, GEM buffers, PRIME, ++ virtual address spaces and syncobj synchronization for the merged ++ Mesa winsys. The GPU is brought up at probe once the user's ++ apple/agx-g5p-c0.ldr is readable. Built-in only while reset and ++ active-DMA teardown require the platform boot sequence. +diff --git a/drivers/gpu/drm/apple-g5p/Makefile b/drivers/gpu/drm/apple-g5p/Makefile +new file mode 100644 +index 000000000..843d56a04 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/Makefile +@@ -0,0 +1,2 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++obj-$(CONFIG_DRM_APPLE_G5P) += a9-gpu-drm.o +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +new file mode 100644 +index 000000000..f2fb4bf2d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -0,0 +1,742 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* Experimental render frontend; default disabled, no implicit firmware boot. */ ++#include "../../apple-g5p/core.inc" ++#include "../../apple-g5p/drm-validate.h" ++#include "../../apple-g5p/render-user-evwait.h" ++#include "../../apple-g5p/render-user-sync.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define G5P_MAX_VMS 8 ++#define G5P_MAX_CONTEXTS 32 ++#define G5P_MAX_MAPS 4096 ++#define G5P_MAX_PT_PAGES 512 ++#define G5P_MAX_SYNCS 17 ++ ++struct g5p_bo { ++ struct drm_gem_shmem_object shmem; ++ u32 roles; ++ u32 vm_pins; /* VMs that map it, any file; counted once in pinned_bytes (pin_lock) */ ++ /* release8 memory accounting (drm-vm.inc g5p_bo_live): the live list, its creator */ ++ struct list_head live; ++ pid_t pid; ++ char comm[TASK_COMM_LEN]; ++}; ++ ++struct g5p_pt { ++ struct list_head link; ++ struct g5p_dma_page page; ++ u32 key; ++ u32 level; ++}; ++ ++struct g5p_mapping { ++ struct list_head link; ++ struct drm_mm_node node; ++ struct drm_gem_object *obj; ++ u64 offset; ++ u64 retire; /* unmapped beside queued jobs: freed once this fence seqno has signalled */ ++}; ++ ++struct g5p_vm { ++ struct g5p_vm_mutation mutation; ++ bool bounded, frozen, serial_context; ++ bool zero_leaf_flags; /* render context latched r25 memory variant 4 */ ++ /* Its render context's handle is gone (g5p_render_close()): no VM_BIND, no new ++ * context; the context's close item and release own the tables. ++ */ ++ bool resident_gone; ++ struct g5p_context *resident; ++ struct kref ref; ++ struct device *dev; ++ struct g5p_drm *gpu; ++ struct drm_mm mm; ++ struct drm_mm_node reserved; ++ struct list_head maps; ++ struct xarray objs; /* object -> its mappings in this VM (xa_mk_value) */ ++ struct list_head tables; ++ struct g5p_dma_page root; ++ u64 mapped_bytes, pinned_bytes; ++ u32 table_count, map_count, contexts; ++}; ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rdev; ++ ++struct g5p_context { ++ struct g5p_drm *gpu; ++ ++ bool live, bad; ++ bool serial, pending; ++ ++ struct g5p_render *render; /* render contexts only, see drm-render.inc */ ++ struct kref ref; ++ struct g5p_vm *vm; ++}; ++ ++struct g5p_file { ++ /* Serializes VM/context handles, mappings and the one-shot submit state. */ ++ struct mutex lock; ++ struct xarray vms; ++ struct xarray contexts; ++ struct xarray results; ++ struct xarray targets; /* render targets; each holds its context */ ++ u32 render_results; ++ bool submitted; ++ u64 job_id; ++}; ++ ++struct g5p_drm { ++ struct drm_device drm; ++ struct g5p_diag *core; ++ /* Protects cached IRQ event/status fields; never held for hardware IO. */ ++ spinlock_t status_lock; ++ u32 last_event[11], notifications[4], first_fault[11]; ++ u64 event_serial, fault_serial; ++ struct delayed_work idle_work; ++ bool idle_suspended; ++ struct delayed_work autoboot; /* probe-time bring-up */ ++ unsigned long autoboot_deadline; ++ u32 autoboot_phase; ++ /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ ++ wait_queue_head_t ev_wq; ++ /* TYPED_V3 3D completion-event latch (render-user-retire.h): G5P_RV3_TD_WANT | endpoint ++ * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. ++ */ ++ u32 td_want; ++ u64 td_hit_ns; ++ /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ ++ bool ta_want; ++ u64 ta_hit_ns; ++ /* wake110: type-1 completion events decoded per engine (word 1 bit 0 TA, bit 1 3D) since ++ * load, and the decode time of the first one since the executor last cleared it. ++ */ ++ u32 t1_count[2]; ++ u64 t1_first_ns[2]; ++ /* perf60 bell gates (drm-render-v3.inc): the previous TYPED_V3 job's end and context. */ ++ u64 bell_prev_end_ns; ++ const void *bell_prev_ctx; ++ u64 last_job; ++ int last_error, job_error; ++ u32 job_state; ++ ++ atomic_t accepted; ++ atomic64_t job_sequence; ++ u64 fence_context; ++ /* Attached catalogues survive fd close until proven unbind/reset. */ ++ struct g5p_context *quarantined_context; ++ /* The first accepted render job claimed the device for render, for the rest of the boot. */ ++ bool render_claimed; ++ struct g5p_rdev *rdev; /* device-wide render state, see drm-render.inc */ ++ struct workqueue_struct ++ *render_wq; /* ordered: the render scheduler's worker */ ++ /* Orders render acceptance, dma_resv fence attach and queueing. */ ++ struct mutex render_submit_lock; ++ /* recovery108 epoch inputs, RAM only: open render-node files, render jobs not yet ++ * signalled. ++ */ ++ atomic_t open_files, r8_queued; ++ /* Memory mapped into any VM, each object once (render_pinned_mib). */ ++ struct mutex pin_lock; ++ u64 pinned_bytes; ++}; ++ ++static inline struct g5p_drm *to_g5p(struct drm_device *drm) ++{ ++ return container_of(drm, struct g5p_drm, drm); ++} ++ ++static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) ++{ ++ return container_of(obj, struct g5p_bo, shmem.base); ++} ++ ++#include "vm-ranges.h" ++ ++static int g5p_resident_idle(struct g5p_context *c, bool live); ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m); ++static int g5p_resident_invalidate(struct g5p_context *c); ++static void g5p_render_release(struct g5p_context *c); ++static int g5p_render_destroy(struct g5p_context *c); ++static void g5p_render_close(struct g5p_context *c); ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a); ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags); ++ ++#include "drm-vm.inc" ++#include "drm-submit.inc" ++#include "drm-render.inc" ++#include "drm-prime.inc" ++#include "drm-compute-v1.inc" ++ ++static int g5p_get_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_info *a = data; ++ u64 caps = DRM_G5P_CAP_BO_CPU | DRM_G5P_CAP_VM_PREPARE | ++ DRM_G5P_CAP_SYNCOBJ | DRM_G5P_CAP_JOB_STATUS | ++ DRM_G5P_CAP_CONTEXT_STATUS | DRM_G5P_CAP_FAULT_INFO; ++ ++ if (a->size != sizeof(*a) || a->flags || a->reserved[0] || ++ a->reserved[1]) ++ return -EINVAL; ++ ; ++ ; ++ ; ++ ; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ a->version = DRM_G5P_UAPI_VERSION; ++ a->gpu_page_shift = 14; ++ ; ++ ; ++ ; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER | DRM_G5P_CAP_BOUNDED_RENDER | ++ DRM_G5P_CAP_TYPED_RENDER | DRM_G5P_CAP_VM_LIVE | ++ DRM_G5P_CAP_CONTEXT_LIVE | ++ DRM_G5P_CAP_CODE_BASE_VERIFIED; ++ a->code_base = 0x300000000ULL; ++ }; ++ ; ++ { ++ caps |= DRM_G5P_CAP_RENDER_KERNEL_GEOMETRY; ++ }; ++ if ((1) && g5p_render_msaa_enabled()) ++ caps |= DRM_G5P_CAP_RENDER_MSAA; ++ ; ++ ; ++ { ++ caps |= DRM_G5P_CAP_TYPED_RENDER_V3 | ++ DRM_G5P_CAP_RENDER_MULTI_CONTEXT | ++ DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH | ++ DRM_G5P_CAP_RENDER_OCCLUSION_QUERY; ++ }; ++ { ++ caps |= DRM_G5P_CAP_RENDER_UNPRIVILEGED; ++ }; ++ ; ++ ; ++ caps |= DRM_G5P_CAP_PRIME; ++ caps |= DRM_G5P_CAP_TYPED_COMPUTE; ++ a->capabilities = caps; ++ a->va_start = G5P_USER_VA_START; ++ a->va_end = G5P_USER_VA_END; ++ a->reserved_start = G5P_USER_RESERVED_START; ++ a->reserved_end = G5P_USER_RESERVED_END; ++ a->max_bo_size = G5P_MAX_BO_SIZE; ++ a->bvnc[0] = 9; ++ a->bvnc[1] = 15; ++ a->bvnc[2] = 6; ++ a->bvnc[3] = 2; ++ return 0; ++} ++ ++/* The gated dialect uses the common B proven by the r19c numerical store. */ ++static int g5p_get_exec_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_exec_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || a->pad || ++ memchr_inv(a->reserved, 0, sizeof(a->reserved))) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ a->size = sizeof(*a); ++ { ++ a->base_scope = DRM_G5P_BASE_VM; ++ a->usc_base = 0x300000000ULL; ++ a->pds_base = a->usc_base; ++ a->window_size = 0x34000; ++ a->allocation_alignment = G5P_GPU_PAGE; ++ a->usc_offset_bits = 32; ++ a->pds_offset_bits = 32; ++ } ++ return 0; ++} ++ ++static int g5p_get_job_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_job_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ bool owned; ++ ++ if (a->flags || a->pad || !a->job_id || a->reset_generation) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ if (xa_load(&f->results, a->job_id)) { ++ struct g5p_fence *fence = xa_load(&f->results, a->job_id); ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = fence->state; ++ a->error = fence->error; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ mutex_unlock(&f->lock); ++ return 0; ++ } ++ owned = f->job_id == a->job_id; ++ mutex_unlock(&f->lock); ++ if (!owned) ++ return -ENOENT; ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->state = d->job_state; ++ a->error = d->job_error; ++ /* Owner acknowledges terminal status before another fixed submission. ++ * This avoids waking a fence waiter before the device admission reopens. ++ */ ++ ; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return 0; ++} ++ ++/* c99788 classifies event4/7/8. Preserve opaque fields, never invent a VA. */ ++static int g5p_get_fault_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_fault_info *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ unsigned long flags; ++ ++ if (a->flags || a->reserved) ++ return -EINVAL; ++ memset(a, 0, sizeof(*a)); ++ spin_lock_irqsave(&d->status_lock, flags); ++ a->event_serial = d->fault_serial; ++ memcpy(a->raw, d->first_fault, sizeof(a->raw)); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!a->event_serial) ++ return 0; ++ switch (a->raw[0]) { ++ case 4: ++ a->kind = DRM_G5P_FAULT_RECOVERY; ++ break; ++ case 7: ++ a->kind = DRM_G5P_FAULT_PAGE_REQUEST; ++ break; ++ case 8: ++ a->kind = DRM_G5P_FAULT_CHANNEL; ++ a->valid = DRM_G5P_FAULT_VALID_CHANNEL; ++ a->channel_class = a->raw[1]; ++ a->channel_token = a->raw[3]; ++ a->channel_stamp = a->raw[4]; ++ break; ++ default: ++ a->kind = DRM_G5P_FAULT_UNKNOWN; ++ break; ++ } ++ return 0; ++} ++ ++static int g5p_get_context_status(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_context_status *a = data; ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(to_g5p(drm)->core->boot); ++ struct g5p_context *c; ++ bool failed; ++ int ret = 0; ++ ++ if (a->flags || a->state || a->map_count || a->mapped_bytes || ++ a->reserved) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c) { ++ ret = -ENOENT; ++ goto out; ++ } ++ a->state = DRM_G5P_CONTEXT_HOST_ONLY; ++ if (READ_ONCE(c->live)) ++ a->state = DRM_G5P_CONTEXT_LIVE; ++ /* a failure noted mid-job is not this context's until the executor rules */ ++ failed = b && READ_ONCE(b->failed) && ++ !g5p_rdev_ruling(to_g5p(drm)->rdev); ++ if (READ_ONCE(c->bad) || failed || (b && READ_ONCE(b->stopped))) ++ a->state = DRM_G5P_CONTEXT_LOST; ++ a->map_count = c->vm->map_count; ++ a->mapped_bytes = c->vm->mapped_bytes; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static void g5p_drm_event(void *cookie, const u32 words[11]) ++{ ++ struct g5p_drm *d = cookie; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ memcpy(d->last_event, words, sizeof(d->last_event)); ++ d->event_serial++; ++ if (words[0] == 1) ++ for (u32 i = 0; i < 4; i++) ++ d->notifications[i] |= words[i + 1]; ++ if (d->td_want && !d->td_hit_ns && ++ g5p_ru_rt_event_match(words, d->td_want & 0xff)) ++ WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); ++ if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) ++ WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); ++ for (u32 e = 0; words[0] == 1 && e < 2; e++) { ++ if (!(words[1] & BIT(e))) ++ continue; ++ WRITE_ONCE(d->t1_count[e], d->t1_count[e] + 1); ++ if (!d->t1_first_ns[e]) ++ WRITE_ONCE(d->t1_first_ns[e], ktime_get_ns()); ++ } ++ if (words[0] != 0 && words[0] != 1) { ++ d->last_error = -EIO; ++ if (!d->fault_serial) { ++ d->fault_serial = d->event_serial; ++ memcpy(d->first_fault, words, sizeof(d->first_fault)); ++ } ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ wake_up(&d->ev_wq); ++} ++ ++static int g5p_idle_open(struct g5p_drm *d); ++static void g5p_idle_queue(struct g5p_drm *d); ++ ++static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = kzalloc(sizeof(*f), GFP_KERNEL); ++ ++ if (!f) ++ return -ENOMEM; ++ mutex_init(&f->lock); ++ xa_init_flags(&f->vms, XA_FLAGS_ALLOC1); ++ xa_init_flags(&f->contexts, XA_FLAGS_ALLOC1); ++ xa_init(&f->results); ++ xa_init_flags(&f->targets, XA_FLAGS_ALLOC1); ++ file->driver_priv = f; ++ { ++ int ret = g5p_idle_open(to_g5p(drm)); ++ ++ if (ret) { ++ mutex_destroy(&f->lock); ++ kfree(f); ++ file->driver_priv = NULL; ++ return ret; ++ } ++ } ++ return 0; ++} ++ ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_vm *vm; ++ unsigned long id; ++ struct g5p_rt *rt; ++ ++ ++ /* Render-target handles hold their context; drop them first. */ ++ xa_for_each(&f->targets, id, rt) ++ g5p_context_put(rt->ctx); ++ xa_destroy(&f->targets); ++ /* Jobs hold contexts, contexts hold VMs, mappings hold GEM objects. ++ * Closing the fd drops handle ownership, never a queued job's references. ++ * Host-only tables may retire at job completion: no live GPU attachment. ++ */ ++ xa_for_each(&f->contexts, id, c) { ++ /* A render context's firmware binding outlives the handle until its close ++ * item, queued behind its jobs, has unbound it. ++ */ ++ if (c->render) ++ g5p_render_close(c); ++ g5p_context_put(c); ++ } ++ xa_destroy(&f->contexts); ++ { ++ struct g5p_fence *fence; ++ ++ xa_for_each(&f->results, id, fence) ++ dma_fence_put(&fence->base); ++ xa_destroy(&f->results); ++ } ++ xa_for_each(&f->vms, id, vm) ++ g5p_vm_put(vm); ++ xa_destroy(&f->vms); ++ mutex_destroy(&f->lock); ++ kfree(f); ++ atomic_dec(&to_g5p(drm)->open_files); ++ g5p_idle_queue(to_g5p(drm)); ++ /* The file's handles are gone (drm_gem_release() ran first): what stays live is held ++ * by another file's handle, a dma-buf, a VM or a job (debugfs g5p_bos names it). ++ */ ++ dev_info(drm->dev, ++ "G5P file close live_bos=%u live_bo_bytes=%lluK device_pinned=%lluK\n", ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10, ++ READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); ++} ++ ++static const struct drm_ioctl_desc g5p_ioctls[] = { ++ DRM_IOCTL_DEF_DRV(G5P_GET_INFO, g5p_get_info, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_BO, g5p_create_bo, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_MMAP_OFFSET, g5p_bo_mmap_offset, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_BO_SYNC, g5p_bo_sync, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_VM, g5p_create_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_VM, g5p_destroy_vm, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_VM_BIND, g5p_vm_bind, DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_CONTEXT, g5p_create_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_CONTEXT, g5p_destroy_context, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_EXEC_INFO, g5p_get_exec_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_JOB_STATUS, g5p_get_job_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_CONTEXT_STATUS, g5p_get_context_status, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_FAULT_INFO, g5p_get_fault_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_CREATE_RENDER_TARGET, g5p_create_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_DESTROY_RENDER_TARGET, g5p_destroy_render_target, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_GET_RENDER_INFO, g5p_get_render_info, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_RENDER_TYPED, g5p_submit_render_typed, ++ DRM_RENDER_ALLOW), ++ DRM_IOCTL_DEF_DRV(G5P_SUBMIT_COMPUTE_TYPED, g5p_submit_compute_typed, ++ DRM_RENDER_ALLOW), ++}; ++ ++DEFINE_DRM_GEM_FOPS(g5p_drm_fops); ++ ++static const struct drm_driver g5p_drm_driver = { ++ .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | ++ DRIVER_SYNCOBJ_TIMELINE, ++ .open = g5p_drm_open, ++ .postclose = g5p_drm_postclose, ++ .fops = &g5p_drm_fops, ++ .ioctls = g5p_ioctls, ++ .num_ioctls = ARRAY_SIZE(g5p_ioctls), ++ .gem_create_object = g5p_gem_create_object, ++ .prime_handle_to_fd = g5p_prime_handle_to_fd, ++ .prime_fd_to_handle = g5p_prime_fd_to_handle, ++ .gem_prime_import = g5p_gem_prime_import, ++ .gem_prime_import_sg_table = g5p_prime_import_sg_table, ++ .name = "apple_g5p", ++ .desc = "Apple A9 G5P experimental render driver", ++ .major = 0, ++ .minor = DRM_G5P_UAPI_VERSION, ++}; ++ ++/* recovery108: the frontend's epoch inputs for the retire layer. RAM only and lock free: ++ * the claimed context is compared, never dereferenced (it may be dropped concurrently). ++ */ ++static void g5p_drm_r8_epoch(void *cookie, struct g5p_r8_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ in->claimed = READ_ONCE(d->render_claimed) || ++ READ_ONCE(d->quarantined_context); ++ in->held = in->claimed && atomic_read(&d->open_files) > 0; ++ in->queued = queued > 0 ? queued : 0; ++} ++ ++/* The teardown DETACH phase. No fence is signalled here: with admission closed and the ++ * device frozen, the render scheduler fails each queued job exactly once (-ESHUTDOWN, ++ * without waiting for its inputs), and an in-flight job ends on its own RAM-only ++ * deadline. The count left to the workers is reported as the explicit abandonment. ++ */ ++static void g5p_drm_td_detach(void *cookie) ++{ ++ struct g5p_drm *d = cookie; ++ int queued = atomic_read(&d->r8_queued); ++ ++ g5p_r8_note_detach(&g5p_r8_state, queued > 0 ? queued : 0); ++ g5p_rsched_kick(d); ++} ++ ++#include "runtime-idle.inc" ++ ++static const bool autoboot = true; ++static const unsigned int autoboot_wait_s = 120; ++ ++/* The bootstrap's order: the one-time firmware, wake and identity steps, then the stages ++ * recovery repeats (recover.inc). ++ */ ++static const char *const g5p_autoboot_phases[] = { ++ "validate", "prepare", "wake", "id", "sample", "map", "arm", "load", "release", ++ "events", "control-prepare", "control-submit", "control-wait", ++}; ++ ++static void g5p_autoboot_work(struct work_struct *work) ++{ ++ struct g5p_drm *d = container_of(to_delayed_work(work), struct g5p_drm, autoboot); ++ struct g5p_diag *g = d->core; ++ bool ran; ++ int ret; ++ ++ while (d->autoboot_phase < ARRAY_SIZE(g5p_autoboot_phases)) { ++ const char *phase = g5p_autoboot_phases[d->autoboot_phase]; ++ ++ ret = g5p_run_phase(g, phase, &ran); ++ /* Built in: the firmware is on the root file system, which may not be mounted yet. */ ++ if (ret == -ENOENT && !d->autoboot_phase && ++ time_before(jiffies, d->autoboot_deadline)) { ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 2 * HZ); ++ return; ++ } ++ dev_info(g->dev, "autoboot phase=%s result=%d\n", phase, ret); ++ if (ret) { ++ dev_warn(g->dev, "autoboot stopped at %s (%d); the command file can continue\n", ++ phase, ret); ++ return; ++ } ++ d->autoboot_phase++; ++ } ++ dev_info(g->dev, "autoboot complete: GPU up\n"); ++ g5p_idle_queue(d); ++} ++ ++static int g5p_drm_probe(struct platform_device *pdev) ++{ ++ struct g5p_drm *d; ++ struct g5p_diag *g; ++ int ret; ++ ++ ; ++ ; ++ ret = g5p_probe(pdev); ++ if (ret) ++ return ret; ++ g = platform_get_drvdata(pdev); ++ d = devm_drm_dev_alloc(&pdev->dev, &g5p_drm_driver, struct g5p_drm, ++ drm); ++ if (IS_ERR(d)) { ++ ret = PTR_ERR(d); ++ goto err_pm; ++ } ++ d->core = g; ++ d->fence_context = dma_fence_context_alloc(1); ++ spin_lock_init(&d->status_lock); ++ init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->accepted, 0); ++ atomic64_set(&d->job_sequence, 0); ++ d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", 0); ++ if (IS_ERR_OR_NULL(d->render_wq)) { ++ ret = d->render_wq ? PTR_ERR(d->render_wq) : -ENOMEM; ++ goto err_pm; ++ } ++ ret = drmm_mutex_init(&d->drm, &d->render_submit_lock); ++ if (!ret) ++ ret = drmm_mutex_init(&d->drm, &d->pin_lock); ++ if (ret) ++ goto err_pm; ++ ret = g5p_render_init(d); ++ if (ret) ++ goto err_pm; ++ g->event_cookie = d; ++ g->event_notify = g5p_drm_event; ++ atomic_set(&d->open_files, 0); ++ atomic_set(&d->r8_queued, 0); ++ drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); ++ drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); ++ INIT_DELAYED_WORK(&d->idle_work, g5p_idle_work); ++ ret = drm_dev_register(&d->drm, 0); ++ if (ret) ++ goto err_pm; ++ g5p_r8_register_epoch(g5p_drm_r8_epoch, d); ++ g5p_td_register_detach(g5p_drm_td_detach, d); ++ g5p_kq_register(&g5p_kq_drm, d); ++ dev_info(&pdev->dev, ++ "render node registered; no GPU access, COMPUTE/RENDER capabilities absent\n"); ++ INIT_DELAYED_WORK(&d->autoboot, g5p_autoboot_work); ++ if (READ_ONCE(autoboot)) { ++ d->autoboot_deadline = jiffies + secs_to_jiffies(autoboot_wait_s); ++ queue_delayed_work(system_unbound_wq, &d->autoboot, 0); ++ } ++ return 0; ++err_pm: ++ g->event_notify = NULL; ++ pm_runtime_disable(&pdev->dev); ++ return ret; ++} ++ ++static void g5p_drm_remove(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g->event_cookie; ++ ++ cancel_delayed_work_sync(&d->autoboot); ++ cancel_delayed_work_sync(&d->idle_work); ++ drm_dev_unplug(&d->drm); ++ g5p_remove(pdev); ++ /* recovery108: the remove-time FINAL above was the callbacks' last use; d is ++ * devres-freed next. ++ */ ++ g5p_td_register_detach(NULL, NULL); ++ g5p_r8_register_epoch(NULL, NULL); ++ g5p_kq_register(NULL, NULL); ++} ++ ++static void g5p_drm_shutdown(struct platform_device *pdev) ++{ ++ struct g5p_diag *g = platform_get_drvdata(pdev); ++ struct g5p_drm *d = g ? g->event_cookie : NULL; ++ ++ if (d) { ++ cancel_delayed_work_sync(&d->autoboot); ++ cancel_delayed_work_sync(&d->idle_work); ++ } ++ g5p_shutdown(pdev); ++} ++ ++static const struct of_device_id g5p_drm_match[] = { ++ { .compatible = "apple,s8003-gpu" }, ++ {}, ++}; ++MODULE_DEVICE_TABLE(of, g5p_drm_match); ++ ++static struct platform_driver g5p_drm_platform = { ++ .probe = g5p_drm_probe, ++ .remove = g5p_drm_remove, ++ .shutdown = g5p_drm_shutdown, ++ .driver = { ++ .name = "apple-g5p-drm", ++ .of_match_table = g5p_drm_match, ++ .dev_groups = g5p_groups, ++ .suppress_bind_attrs = true, ++ }, ++}; ++ ++module_platform_driver(g5p_drm_platform); ++ ++/* Built-in only until reset can prove all DMA references are quiescent. */ ++MODULE_DESCRIPTION("Apple G5P render/GEM/VM experimental driver"); ++MODULE_LICENSE("Dual MIT/GPL"); ++MODULE_FIRMWARE(G5P_FW_NAME); +diff --git a/drivers/gpu/drm/apple-g5p/compute-user-validate.h b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +new file mode 100644 +index 000000000..ac16d4d0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/compute-user-validate.h +@@ -0,0 +1,141 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_COMPUTE_USER_VALIDATE_H ++#define G5P_COMPUTE_USER_VALIDATE_H ++ ++/* ++ * TYPED_COMPUTE pass checks over already-resolved resource ++ * ranges, as render-user-validate.h does for TYPED_V3. Pure, host-tested ++ * (scripts/gpu/compute-host/test_compute.c). Nothing here is a pointer the kernel ++ * dereferences; the stream's last word is read by the caller through the BO's own pages ++ * (g5p_cu_content()). Like TYPED_V3 the kernel does not decode the CDM records, the PDS ++ * programs or the USC code they name; it bounds every address the firmware or the CDM ++ * starts from and leaves the rest to the per-context VM and fault isolation. ++ * -EINVAL flags, alignment, sizes, a non-zero terminate control word, no terminator ++ * -ERANGE a PDS reference outside [pds_base, pds_base + code_window), or a range past ++ * hw_va_limit ++ * -ENOENT no listed range wholly covers the stream, a state word or a PDS reference ++ * -EACCES the range lacks READ (stream, PDS data), WRITE (state) or EXEC (PDS code) ++ */ ++#define G5P_CU_MAX_STREAM 0x10000U ++#define G5P_CU_STREAM_MIN 4U ++#define G5P_CU_TERMINATE 0x80000000U ++ ++/* The pass as the UAPI carries it, kernel copy. */ ++struct g5p_cu_pass { ++ u64 cdm_va; ++ u32 cdm_size, cdm_end; ++ u64 state[4]; ++ u64 tail; ++ u32 config[2]; ++ u32 store[3], resume[3], term[3]; /* code, data, control */ ++}; ++ ++/* Where the content rule reads: the range holding the stream. */ ++struct g5p_cu_loc { ++ u32 stream_range; ++}; ++ ++/* One {code, data} PDS pair: code in a READ|EXEC range, data in a READ range (g5p_ru_v3_ref()). */ ++static inline int g5p_cu_pds(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ const u32 *pds) ++{ ++ int ret = g5p_ru_v3_ref(lim, r, n, pds[0], true); ++ ++ return ret ?: g5p_ru_v3_ref(lim, r, n, pds[1], false); ++} ++ ++static inline int g5p_cu_pass_check(const struct g5p_cu_pass *p, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_cu_loc *loc) ++{ ++ const u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ u32 si; ++ int ret; ++ ++ if (p->cdm_size < G5P_CU_STREAM_MIN || ++ p->cdm_size > G5P_CU_MAX_STREAM || ++ ((p->cdm_va | p->cdm_size | p->cdm_end) & 3) || ++ p->cdm_end < G5P_CU_STREAM_MIN || p->cdm_end > p->cdm_size) ++ return -EINVAL; ++ if ((p->tail & 0xf) || p->term[2]) ++ return -EINVAL; ++ for (u32 i = 0; i < 4; i++) ++ if (p->state[i] & 3) ++ return -EINVAL; ++ /* Every listed range, as TYPED_V3: the CDM and its tasks read and write past them. */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, p->cdm_va, p->cdm_size, &si)) ++ return -ENOENT; ++ if (!(r[si].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ for (u32 i = 0; i < 5; i++) { ++ u64 va = i < 4 ? p->state[i] : p->tail; ++ u32 k; ++ ++ if (!g5p_ru_cover(r, n, va, i < 4 ? 4 : 16, &k)) ++ return -ENOENT; ++ if ((r[k].access & rw) != rw) ++ return -EACCES; ++ } ++ ret = g5p_cu_pds(lim, r, n, p->store); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->resume); ++ if (!ret) ++ ret = g5p_cu_pds(lim, r, n, p->term); ++ if (ret) ++ return ret; ++ loc->stream_range = si; ++ return 0; ++} ++ ++/* The last stream dword, at cdm_va + cdm_end - 4, is the CDM terminate word. */ ++static inline int g5p_cu_content(const u8 *tail) ++{ ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ return word == G5P_CU_TERMINATE ? 0 : -EINVAL; ++} ++ ++/* The firmware's register images of a checked pass. */ ++static inline void g5p_cu_regs_of(struct g5p_cu_regs *k, ++ const struct g5p_cu_pass *p) ++{ ++ k->stream = p->cdm_va; ++ k->terminate = p->cdm_va + p->cdm_end - 4; ++ for (u32 i = 0; i < 4; i++) ++ k->state[i] = p->state[i]; ++ k->tail = p->tail; ++ k->config[0] = p->config[0]; ++ k->config[1] = p->config[1]; ++ k->store = p->store[0] | (u64)p->store[1] << 32; ++ k->resume = p->resume[0] | (u64)p->resume[1] << 32; ++ k->term = p->term[0] | (u64)p->term[1] << 32; ++ k->store1 = p->store[2]; ++ k->resume1 = p->resume[2]; ++} ++ ++/* ++ * Submit-time admission of a compute job, on top of g5p_ru_v3_admit(): the CDM rides an epoch ++ * a render job brought up (the attach rings the TA bell; a compute job never does), and its ++ * own sequence is bounded like the render one. ++ */ ++enum g5p_cu_refusal { ++ G5P_CU_OK, ++ G5P_CU_NO_EPOCH, /* no render job has brought the device up in this epoch */ ++ G5P_CU_OVERFLOW, /* the CDM sequence is exhausted */ ++}; ++ ++static inline int g5p_cu_admit(bool device_up, u32 cu_sequence) ++{ ++ if (!device_up) ++ return G5P_CU_NO_EPOCH; ++ if (cu_sequence >= G5P_CU_SEQ_MAX) ++ return G5P_CU_OVERFLOW; ++ return G5P_CU_OK; ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.h b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +new file mode 100644 +index 000000000..3531cefb7 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_COMPUTE (drm-compute-v1.inc): what the TYPED_V3 executor, the scheduler's drain and the ++ * render device's bring-up call before the file is included. ++ */ ++#ifndef G5P_DRM_COMPUTE_V1_H ++#define G5P_DRM_COMPUTE_V1_H ++ ++struct g5p_drm; ++struct g5p_render; ++struct g5p_rjob; ++ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job); ++static bool g5p_cu_drain(struct g5p_drm *d); ++static void g5p_cu_device_init(struct g5p_drm *d); ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +new file mode 100644 +index 000000000..dcbe34c0b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -0,0 +1,720 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * TYPED_COMPUTE: compute jobs on TYPED_V3 render contexts. ++ * ++ * A compute job is a TYPED_V3 job of its render context with one phase on the device's CDM ++ * channel instead of a TA and a 3D: it is queued, scheduled, fenced, quarantined and lost ++ * exactly like a render job (g5p_render_run(), drm-render-sched.inc, render-user-fault.h). ++ * It binds the context's slot pair the RAM-only way if the context has not run a job yet, ++ * builds the CDM job (compute-user-layout.h) from the pass the submit validated ++ * (compute-user-validate.h), publishes it on the CL ring with a guarded DM2 wake, and waits RAM only ++ * for coherent channel watermarks and both completion stamps. It never brings the device up (that rings the TA bell): ++ * an epoch without a render job refuses it before the hardware. ++ * ++ * Only the licensed publish-time wake touches registers; validation, waits, ++ * samples, retirement and recovery decisions are RAM-only. ++ */ ++ ++static u8 *g5p_cu_page(struct g5p_boot *b) ++{ ++ return b->shared[G5P_CU_OFFSET / G5P_GPU_PAGE].cpu + ++ G5P_CU_OFFSET % G5P_GPU_PAGE; ++} ++ ++static u32 g5p_cu_meta(void) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + G5P_CU_OFFSET); ++} ++ ++/* The CDM's global object (the boot layout's engine-2 object, R0 group, CPU 0x49 / GPU 0x53). */ ++static u32 g5p_cu_global_meta(struct g5p_boot *b) ++{ ++ return g5p_meta_address( ++ g5p_object_va(&b->layout, 0x51 + G5P_CU_ENGINE)); ++} ++ ++/* The render device's bring-up (g5p_render_device_prepare()), after the TA/3D channel init that ++ * clears the 3D page: the CDM channel at its epoch start. RAM only. ++ */ ++static void g5p_cu_device_init(struct g5p_drm *d) ++{ ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_R22_DEP + 0x280); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE < G5P_RU_SLOT1_END); ++ BUILD_BUG_ON(G5P_CU_OFFSET % G5P_GPU_PAGE + G5P_CU_BYTES > ++ G5P_GPU_PAGE); ++ BUILD_BUG_ON(G5P_CU_DEP + G5P_CU_DEP_BYTES > G5P_CU_BYTES); ++ BUILD_BUG_ON(G5P_CU_W + G5P_CU_W_BYTES > G5P_CU_SKU); ++ g5p_cu_channel_init(g5p_cu_page(d->core->boot), g5p_cu_meta()); ++ d->rdev->cu_sequence = 0; ++ d->rdev->cu_target = 0; ++} ++ ++/* One RAM sample of the CDM channel for job `seq` (the slots of struct g5p_rv3_ev: stamp[0] = ++ * W258 mirror, stamp[1] = W25c, ctl = the five watermarks, idle[0..2] = the CDM global's ++ * G724/728/72c). RAM only. ++ */ ++static void g5p_cu_sample(struct g5p_diag *g, u32 seq, u32 target, ++ struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_cu_page(b), *global = g5p_ru_global(b, G5P_CU_ENGINE); ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ s->stamp[0] = g5p_le32(p + G5P_CU_START); ++ s->stamp[1] = g5p_le32(p + G5P_CU_END); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < 3; i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != target || s->cons > 255 || ++ !g5p_cu_stamp_sane(p, seq); ++ s->consumed = s->cons == target; ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2]; ++ s->retired = g5p_cu_retired(p, g5p_cu_meta(), seq); ++ s->pm_ok = true; ++} ++ ++#define G5P_CU_SLOW_MS 1000U ++ ++/* The CDM phase's RAM-only wait, the TYPED_V3 wait (g5p_rv3_wait()) for engine 2: the silence ++ * clock, the fault/protocol rules and the closing sample are the same; complete is the ring ++ * consumed plus the retirement predicate (the idle words are recorded, not required: render path). ++ */ ++/* Event serial is sampled before RAM, so an IRQ between sample and sleep ++ * cannot be lost. Any firmware event wakes us; only the full RAM retirement ++ * predicate may complete the job. No render-only engine-indexed observers. ++ */ ++static void g5p_cu_pause(struct g5p_drm *d, u64 seen) ++{ ++ u32 wait_us; ++ ++ if (g5p_ew_plan(READ_ONCE(render_user_event_wake), false, &wait_us) == ++ G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ return; ++ } ++ wait_event_hrtimeout(d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us)); ++} ++ ++static u32 g5p_cu_wait(struct g5p_drm *d, struct g5p_diag *g, u32 seq, ++ u32 target, struct g5p_ru_wait *w, struct g5p_rv3_ev *s, ++ u64 *consumed_ns, int *err) ++{ ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE(d->event_serial); ++ bool fault, silent, done; ++ ++ g5p_cu_sample(g, seq, target, s); ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed || silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = s->consumed && s->retired; ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (w->result) ++ return w->result; ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_CU_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ dev_warn( ++ g->dev, ++ "compute slow seq=%u ms=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x consumed=%u retired=%u events=%u\n", ++ seq, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, ++ s->cons, s->stamp[0], s->stamp[1], s->ctl[0], ++ s->ctl[1], s->ctl[2], s->ctl[3], s->ctl[4], ++ s->idle[0], s->idle[1], s->idle[2], s->consumed, ++ s->retired, READ_ONCE(g->boot->events)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ g5p_cu_pause(d, seen); ++ } ++} ++ ++/* The guarded publish-time wake, identical ordering to TA/3D with DM2. ++ * Core lock held; only this helper may touch registers in the compute path. ++ * The event lock closes the reset/fault race. Never called from wait/drain. ++ */ ++static int g5p_cu_wake(struct g5p_diag *g, u32 seq, u32 start, u32 target) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ u32 why, fence = U32_MAX, gpc = U32_MAX; ++ ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = G5P_WK_ALWAYS, ++ .e = G5P_CU_ENGINE, ++ .seq = seq, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = target, ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_CU_START), ++ g5p_le32(p + G5P_CU_END) }, ++ }; ++ why = g5p_wk_guard(&in); ++ if (!why) ++ why = g5p_wk_ring(&io, G5P_CU_ENGINE, &fence, &gpc); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_dbg(g->dev, ++ "compute wake seq=%u did=%u power=%#x fence=%#x gpc=%#x\n", seq, ++ why, in.power, fence, gpc); ++ return why == G5P_WK_DID_RANG ? 0 : -EHOSTDOWN; ++} ++ ++/* The CL publish of job `seq`: the global ring entry {type 2, channel, endpoint}, the channel's ++ * admission watermark, then the producer, followed by the guarded DM2 wake. Refused, with nothing written, unless the ++ * ring is idle and the channel stands at the previous job's endpoint and stamps. RAM only. ++ */ ++static int g5p_cu_publish(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); ++ u8 *p = g5p_cu_page(b); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_cu_endpoint(seq - 1); ++ u32 endpoint = g5p_cu_endpoint(seq), why; ++ ++ why = (u32)(start > 255) | (u32)(READ_ONCE(ring[0]) != start) << 1 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 2 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 3 | ++ (u32)(g5p_le32(p + G5P_CU_START) != seq - 1 || ++ g5p_le32(p + G5P_CU_END) != seq - 1) ++ << 4; ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "compute refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x stamp=%08x/%08x: nothing published\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + G5P_CU_START), g5p_le32(p + G5P_CU_END)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x69] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 2 : ++ j == 1 ? g5p_cu_meta() : ++ j == 2 ? endpoint : ++ 0); ++ } ++ rd->cu_target = (start + 1) & 255; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], rd->cu_target); ++ dma_wmb(); ++ if (g5p_ru_v3_verbose(r->jobs, 0)) ++ dev_dbg(g->dev, ++ "compute publish slot=%u endpoint=%u ring=%u W=%08x sequence=%u kick=dm2\n", ++ r->slot[0], endpoint, start, g5p_cu_meta() + G5P_CU_W, ++ seq); ++ return g5p_cu_wake(g, seq, start, rd->cu_target); ++} ++ ++/* The CDM phase timed out with the firmware alive: lose the job and its context, block the ++ * channel behind it until the scheduler's drain (g5p_cu_drain()) sees it retire. RAM only. ++ */ ++static int g5p_cu_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = G5P_CU_ENGINE; ++ s->drain_ms = 45000; ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "compute lost seq=%u slot=%u phase_ms=%lld drain_ms=%u: context lost, device kept, channel blocked\n", ++ seq, r->slot[0], ++ g5p_rv3_us(v->ta.start_ns, v->ta.end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++static void g5p_cu_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ const struct g5p_rv3 *v = &job->rv3; ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ dev_dbg(g->dev, ++ "compute job=%llu seq=%u ret=%d slot=%u consumed_us=%lld done_us=%lld ring=%u/%u stamp=%08x/%08x ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x lost=%u bind_failed=%u\n", ++ job->id, v->seq, ret, r->slot[0], ++ g5p_rv3_us(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_us(v->ta_pub_ns, v->ta.end_ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ v->lost, v->bind_failed); ++} ++ ++/* One compute job, core lock held (g5p_render_run() via g5p_rv3_execute()). Returns 0, -EAGAIN ++ * (no epoch: nothing touched), -ETIMEDOUT, or a fault. ++ */ ++static int g5p_cu_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (r->stage_done) ++ return -ECANCELED; ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ why = g5p_cu_admit(READ_ONCE(rd->up), rd->cu_sequence); ++ if (why) ++ return why == G5P_CU_NO_EPOCH ? -EAGAIN : -EOVERFLOW; ++ seq = rd->cu_sequence + 1; ++ if (!r->bound) { ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ } ++ /* The firmware owns the CL ring and the channel from here on: any failure quarantines. */ ++ v->touched = true; ++ g5p_cu_job(g5p_cu_page(b), g5p_cu_meta(), g5p_cu_global_meta(b), ++ r->slot[0], seq, &job->cu); ++ g5p_render_clean_job(g); ++ rd->cu_sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ ret = g5p_cu_publish(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, v->job_deadline_ns); ++ res = g5p_cu_wait(d, g, seq, rd->cu_target, &v->ta, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ /* The TA/3D paths' MMU fault-bank read, for the CDM (engine 2). */ ++ g5p_rv3_fault_read(d, r, seq, G5P_CU_ENGINE, res, v->ta.silent); ++ loss = g5p_rv3_loss(d, r, false, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && !g5p_cu_lose(d, r, v, seq)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, false)) { ++ g5p_rv3_lose_recover(d, r, v, seq, G5P_CU_ENGINE, ++ v->ta.silent ? "cdm-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "cdm-timeout" : ++ "cdm-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal(g, ++ res == G5P_RU_WAIT_TIMEOUT ? ++ G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "cdm-timeout" : ++ "cdm-fault"); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ if (!v->last[0].idle_ok) ++ dev_warn( ++ g->dev, ++ "compute retired seq=%u with idle=%08x,%08x,%08x set\n", ++ seq, v->last[0].idle[0], v->last[0].idle[1], ++ v->last[0].idle[2]); ++ v->phase = G5P_RV3_DONE; ++ ret = 0; ++out: ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), "cdm"); ++ g5p_cu_receipt(g, r, job, ret); ++ return ret; ++} ++ ++/* The scheduler's drain for a lost compute job (g5p_rv3_drain() hands engine 2 here): wait, RAM ++ * only and for the drain bound, for the stuck job to retire. It did: the channel is clear and ++ * the held jobs run. It did not (silent, fault, bound): the device recovers if it may, else the ++ * epoch ends. Returns false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_cu_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ struct g5p_ru_wait w = {}; ++ u32 res = G5P_RU_WAIT_FAULT; ++ int err = -EIO; ++ u64 now; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ now = ktime_get_ns(); ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (!READ_ONCE(b->failed) && !READ_ONCE(b->stopped)) ++ res = g5p_cu_wait(d, g, s->seq, rd->cu_target, &w, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ if (res == G5P_RU_WAIT_COMPLETE) { ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "compute drained seq=%u slot=%u lost_ms=%lld: channel clear, held jobs resume\n", ++ s->seq, r->slot[0], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "compute drain dead seq=%u why=%s err=%d lost_ms=%lld: the device recovers if it may, else the epoch ends\n", ++ s->seq, ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal(g, ++ fault ? G5P_RU_TERM_FAULT : ++ G5P_RU_TERM_TIMEOUT, ++ fault ? "cdm-drain-fault" : ++ "cdm-drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++static void g5p_cu_pass_from_uapi(struct g5p_cu_pass *p, ++ const struct drm_g5p_compute_pass *u) ++{ ++ *p = (struct g5p_cu_pass){ ++ .cdm_va = u->cdm_va, ++ .cdm_size = u->cdm_size, ++ .cdm_end = u->cdm_end, ++ .tail = u->state_tail_va, ++ .config = { u->config[0], u->config[1] }, ++ .store = { u->store.code, u->store.data, u->store.control }, ++ .resume = { u->resume.code, u->resume.data, u->resume.control }, ++ .term = { u->terminate.code, u->terminate.data, ++ u->terminate.control }, ++ }; ++ for (u32 i = 0; i < 4; i++) ++ p->state[i] = u->state_va[i]; ++} ++ ++static bool g5p_cu_uapi_clean(const struct drm_g5p_compute_pass *u) ++{ ++ return u->size == sizeof(*u) && !u->flags && !u->store.pad && ++ !u->resume.pad && !u->terminate.pad && ++ !memchr_inv(u->reserved, 0, sizeof(u->reserved)); ++} ++ ++/* Acceptance, as g5p_render_submit_admitted() for TYPED_V3 minus the render target: every error ++ * precedes the fence replacement and the queue. ++ */ ++static int g5p_cu_submit_admitted(struct drm_device *drm, struct drm_file *file, ++ const struct drm_g5p_submit_compute_typed *a, ++ u64 *job_id) ++{ ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_compute_pass *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_cu_pass pass; ++ struct g5p_cu_loc loc = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ u8 tail[4]; ++ u64 point = 0; ++ int ret; ++ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(a->resources), ++ a->resource_count, sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(a->syncs), a->sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ upass = memdup_user(u64_to_user_ptr(a->pass), sizeof(*upass)); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (!g5p_cu_uapi_clean(upass)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_cu_pass_from_uapi(&pass, upass); ++ ranges = kvcalloc(a->resource_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(a->resource_count, sizeof(*job->objs), ++ GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, a->sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ /* Compute rides an epoch a render job brought up; a recovery in progress is held. */ ++ if (!READ_ONCE(d->rdev->recovering) && ++ g5p_cu_admit(READ_ONCE(d->rdev->up), ++ READ_ONCE(d->rdev->cu_sequence))) { ++ ret = READ_ONCE(d->rdev->up) ? -EOVERFLOW : -EAGAIN; ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, a->resource_count, ranges, ++ job->objs, &job->obj_count); ++ if (ret) ++ goto unlock; ++ /* Structure first; the stream's last dword is read only once its range is proven backed. */ ++ ret = g5p_cu_pass_check(&pass, &lim, ranges, a->resource_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass.cdm_va + pass.cdm_end - 4, tail, ++ sizeof(tail)); ++ if (!ret) ++ ret = g5p_cu_content(tail); ++ if (ret) ++ goto unlock; ++ g5p_cu_regs_of(&job->cu, &pass); ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = true; ++ job->compute = true; ++ r->v3_accepted = true; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++static_assert(sizeof(struct drm_g5p_compute_pds) == 16); ++static_assert(sizeof(struct drm_g5p_compute_pass) == 152); ++static_assert(offsetof(struct drm_g5p_compute_pass, state_va) == 24); ++static_assert(offsetof(struct drm_g5p_compute_pass, config) == 64); ++static_assert(offsetof(struct drm_g5p_compute_pass, store) == 72); ++static_assert(offsetof(struct drm_g5p_compute_pass, reserved) == 120); ++static_assert(sizeof(struct drm_g5p_submit_compute_typed) == 72); ++static_assert(DRM_G5P_CAP_TYPED_COMPUTE == BIT_ULL(30)); ++static_assert(DRM_G5P_COMPUTE_MAX_STREAM_BYTES == G5P_CU_MAX_STREAM); ++static_assert(DRM_G5P_COMPUTE_STREAM_TERMINATE == G5P_CU_TERMINATE); ++ ++static int g5p_submit_compute_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_compute_typed *a = data; ++ u64 id = 0; ++ int ret; ++ ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_compute_pass)) ++ return -EINVAL; ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_cu_submit_admitted(drm, file, a, &id); ++ g5p_r8_exit(); ++ a->job_id = id; ++ return ret; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +new file mode 100644 +index 000000000..018e83a5b +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -0,0 +1,293 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * Latency123: runtime A/B knobs for the V3 render path, all default off and all writable ++ * through /sys/module/a9_gpu_drm/parameters on a live boot. Included from drm-render.inc ++ * before drm-render-v3.inc. The executor is serial and holds the core lock, so the state ++ * below needs no lock of its own. ++ * ++ * fw_tick_mask (boot.inc) ctl+0x04, re-applied before every job (RAM) ++ * slc_evict_us while a job is in flight, one pass over slc_evict_mib MiB of ++ * cacheable kernel memory at most every N us (the waiter's ++ * 1-2 ms sample cadence bounds it from below); slc_evict_ops: ++ * 1 read pass, 2 write pass, 4 Normal-NC read of the firmware's ++ * root+shared pages, 8 TLBI VMALLE1IS, 16 DSB SY + SEV ++ * cpu_sync 1 dma_sync every job BO (device before, CPU after), 2 clean ++ * root+shared to PoC before the publish, 4 clean+invalidate the ++ * polled pages before every sample ++ * lat123_log one RENDER-L123 line per job (times in ms from the submit ioctl) ++ * pstate_probe 1 PMGR gfx word at job start/end (always-on block), 2 also one ++ * read of Fender +0x1000 at the first sample that shows a fresh ++ * engine busy rise (engine running, so the block is powered) ++ * ++ * No other register access. Temporary: none of this is the fix. ++ */ ++ ++#include ++#include ++ ++#define G5P_L123_OP_READ BIT(0) ++#define G5P_L123_OP_WRITE BIT(1) ++#define G5P_L123_OP_FW_READ BIT(2) ++#define G5P_L123_OP_TLBI BIT(3) ++#define G5P_L123_OP_SEV BIT(4) ++#define G5P_L123_SYNC_BO BIT(0) ++#define G5P_L123_SYNC_REGIONS BIT(1) ++#define G5P_L123_SYNC_POLL BIT(2) ++#define G5P_L123_LINE 64 ++ ++static struct { ++ void *buf; ++ size_t size; ++ u64 sink; ++ /* per job */ ++ u64 exec_ns, next_ns, evict_ns, ps_ns; ++ u32 us, mib, ops, sync, passes, tick, ps; ++ u32 pmgr[2]; ++ bool seen[2], idle[2], probed; ++ /* render_user_event_wake: pauses ended by an event, the timeout, a post-event poll, legacy */ ++ u32 ew[4]; ++ u64 ew_ns; ++} g5p_l123; ++ ++/* perf60-fillrate.md §11.3 cut 1 (render-user-evwait.h): the phase wait's pause between RAM samples. */ ++static const unsigned int render_user_event_wake = 2000; /* release8 default */ ++MODULE_PARM_DESC( ++ render_user_event_wake, ++ "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " ++ "until the next firmware event (the IRQ thread's decode wakes the waiter), at " ++ "most N us (100..20000) as the safety net, and poll every 100 us once the " ++ "phase's own completion event is latched; read live"); ++ ++/* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ ++static u8 *g5p_l123_word(struct g5p_boot *b, u32 slot, u32 field) ++{ ++ u32 off = b->layout.offset[slot] + field; ++ struct g5p_dma_page *page = b->layout.group[slot] ? b->shared : b->root; ++ ++ return page[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_l123_pages(struct g5p_dma_page *page, u32 count, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ for (u32 i = 0; i < count; i++) ++ if (page[i].cpu) ++ op((unsigned long)page[i].cpu, ++ (unsigned long)page[i].cpu + G5P_GPU_PAGE); ++} ++ ++static void g5p_l123_page_of(const void *p, ++ void (*op)(unsigned long start, unsigned long end)) ++{ ++ unsigned long start = (unsigned long)p & ++ ~(unsigned long)(G5P_GPU_PAGE - 1); ++ ++ op(start, start + G5P_GPU_PAGE); ++} ++ ++/* cpu_sync 4: the pages the waiter samples (rings, TA/3D/PM pages, the idle words). */ ++static void g5p_l123_poll_pages(struct g5p_boot *b, u32 e) ++{ ++ g5p_l123_page_of(g5p_boot_shared(b, 0x60), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_boot_shared(b, 0x64), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 0), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_page(b, 1), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_pm_cpu(b), dcache_clean_inval_poc); ++ g5p_l123_page_of(g5p_ru_global(b, e) + 0x724, dcache_clean_inval_poc); ++ dsb(sy); ++} ++ ++static void g5p_l123_bos(struct g5p_drm *d, struct g5p_rjob *job, bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct sg_table *sgt = ++ to_drm_gem_shmem_obj(job->objs[i].obj)->sgt; ++ ++ if (!sgt) ++ continue; ++ if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ } ++} ++ ++static u32 g5p_l123_pmgr(struct g5p_diag *g) ++{ ++ u32 raw = U32_MAX; ++ ++ if (!((0) & 1) || !g->pmgr || regmap_read(g->pmgr, 0x80340, &raw)) ++ return U32_MAX; ++ return raw; ++} ++ ++/* At the executor's entry: this job's knob values and counters. */ ++static void g5p_l123_job_reset(struct g5p_diag *g) ++{ ++ g5p_l123.exec_ns = ktime_get_ns(); ++ g5p_l123.next_ns = 0; ++ g5p_l123.evict_ns = 0; ++ g5p_l123.ps_ns = 0; ++ g5p_l123.passes = 0; ++ g5p_l123.ps = U32_MAX; ++ g5p_l123.probed = false; ++ memset(g5p_l123.ew, 0, sizeof(g5p_l123.ew)); ++ g5p_l123.ew_ns = 0; ++ g5p_l123.seen[0] = false; ++ g5p_l123.seen[1] = false; ++ g5p_l123.us = (0); ++ g5p_l123.mib = clamp((16), 1U, 64U); ++ g5p_l123.ops = (1); ++ g5p_l123.sync = (0); ++ g5p_l123.pmgr[0] = g5p_l123_pmgr(g); ++ g5p_l123.pmgr[1] = U32_MAX; ++} ++ ++/* After the job's records are written, before they are cleaned and published. May sleep. */ ++static void g5p_l123_job_prepare(struct g5p_drm *d, struct g5p_rjob *job) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u8 *tick = g5p_l123_word(b, G5P_L123_CTL, 4); ++ u32 mask = (0); ++ size_t size = (size_t)g5p_l123.mib << 20; ++ ++ if (mask <= G5P_L123_TICK_MASK && g5p_le32(tick) != mask) { ++ g5p_put32(tick, mask); ++ dma_wmb(); ++ } ++ g5p_l123.tick = g5p_le32(tick); ++ ++ if (!g5p_l123.us || ++ !(g5p_l123.ops & (G5P_L123_OP_READ | G5P_L123_OP_WRITE))) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = NULL; ++ g5p_l123.size = 0; ++ } else if (g5p_l123.size != size) { ++ vfree(g5p_l123.buf); ++ g5p_l123.buf = vmalloc(size); ++ g5p_l123.size = g5p_l123.buf ? size : 0; ++ if (g5p_l123.buf) ++ memset(g5p_l123.buf, 0x5a, size); ++ } ++ ++ if (g5p_l123.sync & G5P_L123_SYNC_BO) ++ g5p_l123_bos(d, job, false); ++ if (g5p_l123.sync & G5P_L123_SYNC_REGIONS) { ++ g5p_l123_pages(b->root, ARRAY_SIZE(b->root), dcache_clean_poc); ++ g5p_l123_pages(b->shared, ARRAY_SIZE(b->shared), ++ dcache_clean_poc); ++ dsb(sy); ++ } ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(b, 0); ++} ++ ++/* One streaming pass and the other per-pass operations. */ ++static void g5p_l123_evict(struct g5p_boot *b) ++{ ++ u64 t0 = ktime_get_ns(), sum = 0; ++ u32 ops = g5p_l123.ops; ++ ++ if ((ops & G5P_L123_OP_READ) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ sum += READ_ONCE(p[i]); ++ } ++ if ((ops & G5P_L123_OP_WRITE) && g5p_l123.buf) { ++ u64 *p = g5p_l123.buf; ++ ++ for (size_t i = 0; i < g5p_l123.size / 8; ++ i += G5P_L123_LINE / 8) ++ WRITE_ONCE(p[i], i); ++ } ++ if (ops & G5P_L123_OP_FW_READ) { ++ struct g5p_dma_page *regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; ++ regions[r][i].cpu && at < G5P_GPU_PAGE; ++ at += G5P_L123_LINE) ++ sum += READ_ONCE(*( ++ const u32 *)(regions[r][i].cpu + ++ at)); ++ } ++ if (ops & G5P_L123_OP_TLBI) ++ flush_tlb_all(); ++ if (ops & G5P_L123_OP_SEV) { ++ dsb(sy); ++ sev(); ++ } ++ WRITE_ONCE(g5p_l123.sink, g5p_l123.sink + sum); ++ g5p_l123.passes++; ++ g5p_l123.evict_ns += ktime_get_ns() - t0; ++} ++ ++/* Replaces the waiter's sleep between two RAM samples of engine e. seen: d->event_serial as ++ * read after the drain and before this sample (render-user-evwait.h). ++ */ ++static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, ++ const struct g5p_rv3_ev *s, u64 seen) ++{ ++ u64 now = s->ns, t0; ++ enum g5p_ew_act act; ++ u32 wait_us; ++ ++ if (((0) & 2) && !g5p_l123.probed && g5p_l123.seen[e] && ++ g5p_l123.idle[e] && !s->idle_ok && !s->failed && !s->proto && ++ !g5p_ru_term_failed(&g5p_rv3_term) && !READ_ONCE(g->boot->failed)) { ++ struct g5p_mmio_ticket t; ++ ++ /* an R1-only ticket of the register licence (mmio.inc), never a bare read */ ++ if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &t); ++ g5p_l123.ps = t.gpc; ++ g5p_l123.ps_ns = ktime_get_ns(); ++ g5p_l123.probed = true; ++ } ++ g5p_l123.seen[e] = true; ++ g5p_l123.idle[e] = s->idle_ok; ++ if (g5p_l123.us && now >= g5p_l123.next_ns) { ++ g5p_l123_evict(g->boot); ++ g5p_l123.next_ns = now + (u64)g5p_l123.us * NSEC_PER_USEC; ++ } ++ act = g5p_ew_plan(READ_ONCE(render_user_event_wake), s->event, ++ &wait_us); ++ t0 = ktime_get_ns(); ++ if (act == G5P_EW_LEGACY) { ++ usleep_range(1000, 2000); ++ g5p_l123.ew[3]++; ++ } else if (act == G5P_EW_SHORT) { ++ usleep_range(wait_us, wait_us + wait_us / 2); ++ g5p_l123.ew[2]++; ++ } else if (wait_event_hrtimeout( ++ d->ev_wq, ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ us_to_ktime(wait_us))) { ++ g5p_l123.ew[1]++; /* -ETIME: the safety net */ ++ } else { ++ g5p_l123.ew[0]++; ++ } ++ g5p_l123.ew_ns += ktime_get_ns() - t0; ++ if (g5p_l123.sync & G5P_L123_SYNC_POLL) ++ g5p_l123_poll_pages(g->boot, e); ++} ++ ++/* After the receipt. One line per job while lat123_log is set. */ ++static void g5p_l123_job_end(struct g5p_drm *d, struct g5p_rjob *job, u32 seq, ++ int ret) ++{ ++ struct g5p_diag *g = d->core; ++ ++ if (!ret && (g5p_l123.sync & G5P_L123_SYNC_BO)) ++ g5p_l123_bos(d, job, true); ++ g5p_l123.pmgr[1] = g5p_l123_pmgr(g); ++ { ++ return; ++ } ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-prime.inc b/drivers/gpu/drm/apple-g5p/drm-prime.inc +new file mode 100644 +index 000000000..bd513b969 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-prime.inc +@@ -0,0 +1,114 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* PRIME for shmem GEM BOs. ++ * ++ * Export: Mesa WSI/client buffers and kmsro bookkeeping. The dma-buf adds ++ * begin/end_cpu_access, so an importer that CPU-reads (simpledrm's blit, ++ * llvmpipe) gets cache maintenance on the G5P DMA mapping. vmap/mmap reuse ++ * the shmem helpers (write-combined when the device is not DMA-coherent). ++ * ++ * Import: foreign dma-bufs, e.g. simpledrm dumb buffers that the GPU renders ++ * into for scanout. The attachment stays mapped (pinned) for the life of the ++ * GEM object; VM_BIND maps its sg_table. Render jobs listing an imported BO ++ * get whole-buffer cache maintenance and dma_resv fences (drm-render.inc). ++ */ ++ ++static int g5p_dmabuf_begin_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ /* No device mapping yet means no device-owned cache lines either. */ ++ if (sgt) ++ dma_sync_sgtable_for_cpu(obj->dev->dev, sgt, DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static int g5p_dmabuf_end_cpu_access(struct dma_buf *buf, ++ enum dma_data_direction dir) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(obj)->sgt; ++ ++ if (sgt) ++ dma_sync_sgtable_for_device(obj->dev->dev, sgt, ++ DMA_BIDIRECTIONAL); ++ return 0; ++} ++ ++static const struct dma_buf_ops g5p_dmabuf_ops = { ++ .attach = drm_gem_map_attach, ++ .detach = drm_gem_map_detach, ++ .map_dma_buf = drm_gem_map_dma_buf, ++ .unmap_dma_buf = drm_gem_unmap_dma_buf, ++ .release = drm_gem_dmabuf_release, ++ .mmap = drm_gem_dmabuf_mmap, ++ .vmap = drm_gem_dmabuf_vmap, ++ .vunmap = drm_gem_dmabuf_vunmap, ++ .begin_cpu_access = g5p_dmabuf_begin_cpu_access, ++ .end_cpu_access = g5p_dmabuf_end_cpu_access, ++}; ++ ++static struct dma_buf *g5p_gem_prime_export(struct drm_gem_object *obj, ++ int flags) ++{ ++ struct dma_buf_export_info info = { ++ .exp_name = KBUILD_MODNAME, ++ .owner = obj->dev->driver->fops->owner, ++ .ops = &g5p_dmabuf_ops, ++ .size = obj->size, ++ .flags = flags, ++ .priv = obj, ++ .resv = obj->resv, ++ }; ++ ++ return drm_gem_dmabuf_export(obj->dev, &info); ++} ++ ++static int g5p_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file, ++ u32 handle, u32 flags, int *fd) ++{ ++ return drm_gem_prime_handle_to_fd(dev, file, handle, flags, fd); ++} ++ ++static int g5p_prime_fd_to_handle(struct drm_device *dev, struct drm_file *file, ++ int fd, u32 *handle) ++{ ++ return drm_gem_prime_fd_to_handle(dev, file, fd, handle); ++} ++ ++/* Our own export comes back as the same GEM object (the core's identity check ++ * only knows its default ops); anything else is attached and mapped. ++ */ ++static struct drm_gem_object *g5p_gem_prime_import(struct drm_device *dev, ++ struct dma_buf *buf) ++{ ++ struct drm_gem_object *obj = buf->priv; ++ ++ if (buf->ops == &g5p_dmabuf_ops && obj->dev == dev) { ++ drm_gem_object_get(obj); ++ return obj; ++ } ++ return drm_gem_prime_import(dev, buf); ++} ++ ++/* Foreign buffers are GPU READ|WRITE data, never executable, and must be ++ * whole 16KiB GPU pages within the BO size limit. ++ */ ++static struct drm_gem_object * ++g5p_prime_import_sg_table(struct drm_device *dev, ++ struct dma_buf_attachment *attach, ++ struct sg_table *sgt) ++{ ++ struct drm_gem_object *obj; ++ ++ if (attach->dmabuf->size & (G5P_GPU_PAGE - 1) || ++ attach->dmabuf->size > G5P_MAX_BO_SIZE) ++ return ERR_PTR(-EINVAL); ++ obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); ++ if (!IS_ERR(obj)) { ++ to_g5p_bo(obj)->roles = DRM_G5P_BO_READ | DRM_G5P_BO_WRITE; ++ g5p_bo_track(obj); ++ } ++ return obj; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +new file mode 100644 +index 000000000..21dc64d89 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -0,0 +1,203 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * The render scheduler (docs/gpu/multi-client-design.md "Stage 2/3 as built"): one worker on ++ * the ordered render_wq serves the contexts round-robin (core/render-user-sched.h). Jobs wait ++ * for their inputs on dma-fence callbacks, not in the worker, so one client's unsignalled ++ * fence never holds up another client. Each picked job runs to completion on the one shared ++ * channel (one credit), exactly as a work item did before; a context's close item runs after ++ * its last job. Included from drm-render.inc after drm-render-v3.inc. ++ * ++ * The shape is drm_sched's (entity FIFO, run-list RR, dependency callbacks, entity teardown ++ * after the last job), so stage 4 can replace it with drm_sched without a UAPI change. ++ */ ++ ++static struct dma_fence *g5p_rjob_dep(struct g5p_rjob *job, u32 i) ++{ ++ return i < job->input_count ? job->inputs[i] : ++ job->deps[i - job->input_count]; ++} ++ ++/* A signalled input: its error, or that of any fence it wraps (array, chain), else 0. */ ++static int g5p_render_fence_status(struct dma_fence *fence) ++{ ++ struct dma_fence_unwrap cursor; ++ struct dma_fence *part; ++ u32 unwrapped = 0; ++ int ret = dma_fence_get_status(fence); ++ ++ if (ret < 0) ++ return ret; ++ dma_fence_unwrap_for_each(part, &cursor, fence) { ++ ret = dma_fence_get_status(part); ++ if (ret <= 0 || ++unwrapped > 4096) { ++ dma_fence_put(cursor.chain); ++ return ret < 0 ? ret : -ETIMEDOUT; ++ } ++ } ++ return 0; ++} ++ ++/* Fence callback (any context, the fence lock held): the head job's input signalled. */ ++static void g5p_rjob_dep_cb(struct dma_fence *fence, struct dma_fence_cb *cb) ++{ ++ struct g5p_rjob *job = container_of(cb, struct g5p_rjob, cb); ++ ++ WRITE_ONCE(job->cb_fired, true); ++ mod_delayed_work(job->gpu->render_wq, &job->gpu->rdev->sched_work, 0); ++} ++ ++/* Scheduler lock held: no callback of this job stays armed. */ ++static void g5p_rjob_disarm(struct g5p_rjob *job) ++{ ++ if (job->cb_fence) ++ dma_fence_remove_callback(job->cb_fence, &job->cb); ++ job->cb_fence = NULL; ++} ++ ++/* ++ * Scheduler lock held: may the head job of its context run now? Walks its explicit inputs, ++ * then its implicit dependencies, in order; at the first unsignalled one it arms a callback ++ * and reports the job's input deadline. A failed input, the deadline (G5P_RU_INPUT_MS after ++ * the submit, as the old in-worker wait) and closed admission make it ready with an error, so ++ * that it fails before the hardware and its fence still signals in order. ++ */ ++static bool g5p_rsched_ready(struct g5p_rs_job *node, void *cookie, ++ unsigned long *deadline) ++{ ++ struct g5p_rjob *job = container_of(node, struct g5p_rjob, node); ++ ++ if (job->setup_error || job->dep_error) ++ return true; ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ESHUTDOWN; ++ return true; ++ } ++ if (job->cb_fence) { ++ if (!READ_ONCE(job->cb_fired)) ++ goto pending; ++ job->cb_fence = NULL; ++ } ++ while (job->dep_next < job->input_count + job->dep_count) { ++ struct dma_fence *fence = g5p_rjob_dep(job, job->dep_next); ++ ++ if (dma_fence_is_signaled(fence)) { ++ job->dep_error = g5p_render_fence_status(fence); ++ if (job->dep_error) ++ return true; ++ job->dep_next++; ++ continue; ++ } ++ WRITE_ONCE(job->cb_fired, false); ++ if (dma_fence_add_callback(fence, &job->cb, g5p_rjob_dep_cb)) ++ continue; /* -ENOENT: signalled meanwhile */ ++ job->cb_fence = fence; ++ goto pending; ++ } ++ return true; ++pending: ++ if (time_after_eq(jiffies, job->deadline)) { ++ g5p_rjob_disarm(job); ++ job->dep_error = -ETIMEDOUT; ++ return true; ++ } ++ *deadline = job->deadline; ++ return false; ++} ++ ++static void g5p_rsched_kick(struct g5p_drm *d) ++{ ++ mod_delayed_work(d->render_wq, &d->rdev->sched_work, 0); ++} ++ ++/* Submit, render_submit_lock held: an accepted job joins its context's FIFO. */ ++static void g5p_rsched_push(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ g5p_rs_push(&d->rdev->sched, &r->entity, &job->node); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ g5p_rsched_kick(d); ++} ++ ++/* The close item joins the context's entity behind its jobs; false if one is queued. */ ++static bool g5p_rsched_close(struct g5p_drm *d, struct g5p_render *r) ++{ ++ unsigned long flags; ++ bool queued; ++ ++ spin_lock_irqsave(&d->rdev->sched_lock, flags); ++ queued = g5p_rs_close(&d->rdev->sched, &r->entity); ++ spin_unlock_irqrestore(&d->rdev->sched_lock, flags); ++ if (queued) ++ g5p_rsched_kick(d); ++ return queued; ++} ++ ++static void g5p_render_run(struct g5p_rjob *job); ++static void g5p_render_close_run(struct g5p_render *r); ++static void g5p_render_close_deferred(struct g5p_drm *d); ++ ++/* ++ * The worker: pick, run to completion, repeat; then sleep until a callback, a submit or the ++ * earliest input deadline wakes it. render_wq is ordered, so this never runs twice at once, ++ * and it is the only writer of the device's job state, as each job's work item was. ++ */ ++static void g5p_rsched_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = container_of(to_delayed_work(work), ++ struct g5p_rdev, sched_work); ++ struct g5p_drm *d = rd->drm; ++ ++ /* Each item drops the references it held; the device must outlive this loop. */ ++ drm_dev_get(&d->drm); ++ for (;;) { ++ struct g5p_rs_entity *e; ++ struct g5p_rs_job *node; ++ struct g5p_render *r; ++ struct g5p_context *c; ++ unsigned long flags, wake = 0; ++ bool has_wake; ++ int what; ++ ++ /* A lost job blocks the channel: hold every job until the drain is decided, and ++ * until a recovery the drain or the executor asked for has run. ++ */ ++ if (READ_ONCE(rd->stuck) || READ_ONCE(rd->recovering)) { ++ if (rd->stuck ? g5p_rv3_drain(d) : g5p_rv3_recover(d)) ++ continue; ++ queue_delayed_work(d->render_wq, &rd->sched_work, ++ msecs_to_jiffies(100)); ++ break; ++ } ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ what = g5p_rs_pick(&rd->sched, g5p_rsched_ready, d, &e, &node, ++ &wake, &has_wake); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ if (what == G5P_RS_IDLE) { ++ if (has_wake) ++ queue_delayed_work( ++ d->render_wq, &rd->sched_work, ++ max_t(long, (long)(wake - jiffies), 1)); ++ break; ++ } ++ if (what == G5P_RS_JOB) { ++ g5p_render_run( ++ container_of(node, struct g5p_rjob, node)); ++ g5p_render_close_deferred(d); ++ continue; ++ } ++ r = container_of(e, struct g5p_render, entity); ++ c = r->ctx; ++ g5p_render_close_run(r); ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ g5p_rs_closed(&rd->sched, e); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ /* The close item's own reference (g5p_render_close()); r may go with it. */ ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ drm_dev_put(&d->drm); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +new file mode 100644 +index 000000000..36550c443 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -0,0 +1,488 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Unbinding firmware slots (docs/gpu/multi-client-design.md §4.5): a closed context's pair ++ * (stage 2, core/render-user-close.h) and the render frontend's share of the reboot quiesce ++ * (core/render-user-quiesce.h, docs/gpu/experiments/mcfold2-kexec.md). ++ * Included from drm-render.inc after drm-render-v3.inc. ++ */ ++ ++/* The RAM words of `slot` and of its unbind record at ring entry `index`. */ ++static void g5p_render_slot_mem(struct g5p_boot *b, u32 slot, u32 index, ++ struct g5p_ru_slot_mem *m) ++{ ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + (index & 255) * 20 + j * 4; ++ ++ m->rec[j] = ++ b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE; ++ } ++ m->pa = g5p_boot_shared(b, 0x86) + slot * 8; ++ m->host = g5p_boot_shared(b, 0x84) + slot * 4; ++ m->cached = g5p_boot_shared(b, 0x85) + slot * 4; ++} ++ ++/* ++ * Unbind `n` firmware slots in one RAM-only DC3 batch: PA and host tag of each to zero, one ++ * record {3, slot} per slot, one producer store, then the RAM-only wait of a bind and a check ++ * that the firmware's DC3 handler copied the zero tag of every slot. No doorbell and no ++ * register access, so it is safe after a job retired and while the GPU may be idle-gated. ++ * ++ * The caller holds the core lock (the DC ring has one writer) and has retired every use of ++ * these slots: no job of theirs is queued on the firmware or in flight. On success the ++ * firmware holds, per slot, PA 0, host and cached tag 0 and CBASE 0, with the slot's PC level ++ * invalidated; the slot may be bound again by a later DC3. Its translations stay allocated: ++ * freeing them is the caller's step, after this returns 0. A batch that is not consumed in ++ * time keeps later binds and unbinds refused until the firmware catches up (dc_pending). ++ */ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_slot_sample s = {}; ++ struct g5p_ru_slot_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, cons, target, bad = 0, verified = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!g5p_ru_unbind_fits(n)) ++ return -EINVAL; ++ for (u32 i = 0; i < n; i++) ++ if (slots[i] < G5P_RU_SLOT_MIN || slots[i] > G5P_RU_SLOT_MAX) ++ return -EINVAL; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ cons = READ_ONCE(ctl[0]); ++ if (start > 255 || cons != start || ++ (rd->dc_pending && cons != rd->dc_wait)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s refused slots=%u dc=%u/%u pending=%u: ring busy; no " ++ "hardware touched end=v3\n", ++ why, n, cons, start, rd->dc_pending); ++ return -EBUSY; ++ } ++ rd->dc_pending = false; ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_unbind_publish(&m, slots[i]); ++ } ++ target = (start + n) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, why); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ for (u32 i = 0; i < n; i++) { ++ u32 one; ++ ++ g5p_render_slot_mem(b, slots[i], start + i, &m); ++ g5p_ru_slot_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ one = g5p_ru_unbind_check(&s, slots[i], target); ++ bad |= one; ++ verified += !one; ++ } ++ if (!ret && bad) ++ ret = -EUCLEAN; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 unbind why=%s slots=%u verified=%u ret=%d consumed_us=%lld " ++ "dc=%u->%u/%u bad=%#x first=%u last=%u mode=ram-dc3 end=v3\n", ++ why, n, verified, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ s.cons, s.prod, bad, slots[0], slots[n - 1]); ++ return ret; ++} ++ ++/* ++ * render_user_close_park: bind a closed context's just-unbound pair to the park catalogue, one ++ * DC3 pair as a context's bind (g5p_render_vm_prepare()), checked the same way. The catalogue ++ * holds the firmware window's entry and the borrowed sink entry, both stable within an epoch, so ++ * rewriting them for a later park never clears what an earlier parked pair resolves through. ++ * Core lock held, between jobs (the close item), RAM only and no register: as the unbind. ++ */ ++static int g5p_rv3_park(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, target, entry, sink, bad = 0; ++ int ret; ++ ++ lockdep_assert_held(&g->lock); ++ if (!rd->park.cpu) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->park); ++ if (ret) ++ return ret; ++ memset(rd->park.cpu, 0, G5P_GPU_PAGE); ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ dma_rmb(); ++ start = READ_ONCE(ctl[4]); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (start > 255 || rd->dc_pending || g5p_ru_bind_busy(&s)) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park refused slots=%u/%u dc=%u/%u pending=%u busy=%#x: the pair stays " ++ "unbound end=v3\n", ++ r->slot[0], r->slot[1], s.cons, start, rd->dc_pending, ++ g5p_ru_bind_busy(&s)); ++ return -EBUSY; ++ } ++ sink = b->sink_ready ? g5p_le32(b->catalogue.cpu) : 0; ++ if (g5p_le32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4) != entry) ++ g5p_put32(rd->park.cpu + G5P_RU_FIRMWARE_CI * 4, entry); ++ if (g5p_le32(rd->park.cpu) != sink) ++ g5p_put32(rd->park.cpu, sink); ++ g5p_table_clean_page(g, &rd->park); ++ dsb(sy); ++ g5p_ru_bind_publish(&m, r->slot, rd->park.dma, G5P_RU_PARK_TAG); ++ target = (start + 2) & 255; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "park"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, rd->park.dma, ++ G5P_RU_PARK_TAG, target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ rd->parks += !ret; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 park slots=%u/%u ret=%d consumed_us=%lld dc=%u->%u/%u bad=%#x " ++ "parks=%u pc=%pad end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ start, s.cons, s.prod, bad, rd->parks, &rd->park.dma); ++ return ret; ++} ++ ++/* Drop what the claim of the device took (g5p_render_device_claim()); the device stays ++ * claimed for render. Returns whether a context reference was dropped. ++ */ ++static bool g5p_render_unclaim(struct g5p_drm *d, struct g5p_render *r) ++{ ++ if (!r->claimant) ++ return false; ++ r->claimant = false; ++ if (d->quarantined_context == r->ctx) ++ d->quarantined_context = NULL; ++ return true; ++} ++ ++/* ++ * The close item of a TYPED_V3 context (multi-client-design.md "Stage 2/3 as built"): the ++ * scheduler runs it after every job of the context, so every use of the pair has retired or ++ * failed. RAM only: the unbind is g5p_rv3_unbind_slots() for this pair under the core lock. ++ * After a verified unbind the binding's references go, and the last one frees the VM tables, ++ * render targets and mappings (unpinning the BOs) and returns the slots ++ * (g5p_render_release()). Anything the firmware may still hold is retained until reboot. ++ */ ++/* perf60 (docs/gpu/experiments/perf60-clock.md), corrected by the release8 bench (bench-gaps3, ++ * render-user-pmset.h): the PM writes a previous PM context back (its free list F, through that ++ * context's TA slot) not once at the switch but on the later jobs of the next context, until a ++ * third context runs. A closed context therefore keeps its slots bound and its set and pages alive ++ * while it is the last job's context or the previous distinct one; its unbind and release run once ++ * it is neither. At most those two wait. (The first version held only the last job's context and ++ * released it after one other job; 2 held one job longer, by job records: both released it while ++ * the PM still wrote to it.) RAM only: bookkeeping and the close item's references. ++ */ ++/* release8 default */ ++ ++/* ++ * A close item follows every job of this context on the ordered render worker. ++ * resident_gone, set under the file lock before queueing it, rejects all later ++ * VM_BIND and context creation. Thus the close item owns vm->maps exclusively. ++ * ++ * Idle VM_BIND UNMAP already drops a user mapping's pages after retirement. ++ * Do that here without releasing the PM's write-back route: no unbind, no ++ * table pruning, no PM set/pool/aux/RT release, no pm_last/pm_prev change. The ++ * PM free list is in the firmware window, never in vm->maps. All user mappings ++ * must pass the same range check as VM_BIND before any PTE is touched. ++ * ++ * No new kick of this detached context can occur. Clean the cleared user PTEs ++ * to PoC, but do not publish a tag, invalidate a slot or access R0. The PM can ++ * keep using its unchanged translations and pages, including cached ones. ++ * Failed/queued/terminal/shutdown epochs keep every BO, as before. ++ */ ++static void g5p_render_close_trim(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_boot *b; ++ struct g5p_ru_close_in in; ++ struct g5p_mapping *m, *tmp; ++ struct g5p_pt *pt; ++ u64 pinned; ++ u32 maps, why; ++ ++ if (!vm->map_count || !vm->resident_gone || !r->closing || ++ !r->detached || !mutex_trylock(&g->lock)) ++ return; ++ b = g->boot; ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* Match close_run: a successfully drained lost job has no user-memory use left. */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .dc_pending = d->rdev->dc_pending, ++ .queued = atomic_read(&r->queued), ++ }; ++ if (g5p_ru_close_plan(&in, &why) != G5P_RU_CLOSE_UNBIND) ++ goto out; ++ list_for_each_entry(m, &vm->maps, link) ++ if (!g5p_user_range_valid(m->node.start, m->node.size) || ++ (m->node.start < G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE && ++ G5P_RU_PRIVATE_VA < m->node.start + m->node.size)) ++ goto out; ++ ++ pinned = vm->pinned_bytes; ++ maps = vm->map_count; ++ list_for_each_entry(m, &vm->maps, link) ++ g5p_vm_clear(vm, d->drm.dev, m->node.start, m->node.size); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ list_for_each_entry_safe(m, tmp, &vm->maps, link) { ++ vm->mapped_bytes -= m->node.size; ++ drm_mm_remove_node(&m->node); ++ list_del(&m->link); ++ vm->map_count--; ++ g5p_vm_obj_put(vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-trim slots=%u/%u maps=%u unpinned=%lluK remaining=%lluK " ++ "device_pinned=%lluK pm=retained end=v3\n", ++ r->slot[0], r->slot[1], maps, (pinned - vm->pinned_bytes) >> 10, ++ vm->pinned_bytes >> 10, READ_ONCE(d->pinned_bytes) >> 10); ++out: ++ mutex_unlock(&g->lock); ++} ++ ++/* render_user_close_defer: may the firmware's PM still write this context's state back? */ ++static bool g5p_render_pm_holds(struct g5p_rdev *rd, struct g5p_render *r, ++ u32 mode) ++{ ++ return mode && g5p_pmset_holds(rd->pm_last, rd->pm_prev, r); ++} ++ ++static void g5p_render_close_run(struct g5p_render *r) ++{ ++ struct g5p_context *c = r->ctx; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_close_in in; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 puts = 0, why; ++ bool locked = false; ++ int plan, ret = 0; ++ u32 mode, k; ++ ++ mode = (1); ++ k = d->rdev->close_deferred[0] ? 1 : 0; ++ if (mode && !r->close_defer_run && r->bound && ++ g5p_render_pm_holds(d->rdev, r, mode) && ++ k < ARRAY_SIZE(d->rdev->close_deferred)) { ++ g5p_render_close_trim(r); ++ /* the deferred close's own references, dropped by g5p_render_close_deferred() */ ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->rdev->close_deferred[k] = r; ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer slots=%u/%u jobs=%u: the PM may still write its state " ++ "back; unbind once it has moved past it end=v3\n", ++ r->slot[0], r->slot[1], r->jobs); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close-defer2 slots=%u/%u pm=%s pending=%u end=v3\n", ++ r->slot[0], r->slot[1], ++ d->rdev->pm_last == r ? "last" : "previous", k + 1); ++ return; ++ } ++ ++ for (;;) { ++ in = (struct g5p_ru_close_in){ ++ .retained = r->retained, ++ .bound = r->bound, ++ /* a lost job that drained left the firmware nothing of this context */ ++ .bad = READ_ONCE(c->bad) && !r->lost_drained, ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .dev_failed = !b || READ_ONCE(b->failed), ++ .stopped = !b || READ_ONCE(b->stopped), ++ .shutdown = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ /* as g5p_rv3_unbind_slots() refuses it */ ++ .dc_pending = b && d->rdev->dc_pending && ++ READ_ONCE(((u32 *)g5p_boot_shared( ++ b, 0x6c))[0]) != d->rdev->dc_wait, ++ .queued = atomic_read(&r->queued), ++ }; ++ plan = g5p_ru_close_plan(&in, &why); ++ if (plan != G5P_RU_CLOSE_UNBIND || locked) ++ break; ++ /* Bounded, as a job takes it: never wait forever behind a command in MMIO. */ ++ while (!(locked = mutex_trylock(&g->lock)) && ++ time_before(jiffies, deadline)) ++ msleep(5); ++ if (!locked) { ++ plan = G5P_RU_CLOSE_RETAIN; ++ why = G5P_RU_CLOSE_LOCK; ++ break; ++ } ++ /* Plan again under the lock: the reboot quiesce or a failure may have run. */ ++ } ++ if (plan == G5P_RU_CLOSE_UNBIND) { ++ ret = g5p_rv3_unbind_slots(d, r->slot, 2, "close"); ++ if (!ret) { ++ r->bound = false; ++ r->retained = false; ++ WRITE_ONCE(c->live, false); ++ puts++; ++ /* the PM may still write this context's free list back through its TA slot */ ++ { ++ g5p_rv3_park(d, r); ++ }; ++ } ++ } ++ if (plan == G5P_RU_CLOSE_RELEASE || ++ (plan == G5P_RU_CLOSE_UNBIND && !ret)) ++ puts += g5p_render_unclaim(d, r); ++ if (locked) ++ mutex_unlock(&g->lock); ++ dev_info( ++ g->dev, ++ "RENDER-V3 close slots=%u/%u plan=%s why=%#x ret=%d jobs=%u claimant=%u " ++ "retained=%u end=v3\n", ++ r->slot[0], r->slot[1], g5p_ru_close_name(plan), why, ret, ++ r->jobs, r->claimant, r->retained); ++ while (puts--) { ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++} ++ ++/* After a job: a deferred close runs once the last job was another context's. */ ++static void g5p_render_close_deferred(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ u32 mode = (1); ++ ++ for (u32 k = 0; k < ARRAY_SIZE(rd->close_deferred); k++) { ++ struct g5p_render *r = rd->close_deferred[k]; ++ struct g5p_context *c; ++ ++ /* mode 0 now: a pending close still waits until the PM has moved past it */ ++ if (!r || g5p_render_pm_holds(rd, r, mode ?: 1)) ++ continue; ++ rd->close_deferred[k] = NULL; ++ c = r->ctx; ++ r->close_defer_run = true; ++ dev_info( ++ d->core->dev, ++ "RENDER-V3 close-deferred slots=%u/%u: neither the PM's last nor its previous " ++ "context end=v3\n", ++ r->slot[0], r->slot[1]); ++ g5p_render_close_run(r); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ /* keep a pending close in [0]: close_run fills [1] only behind it */ ++ if (!rd->close_deferred[0] && rd->close_deferred[1]) { ++ rd->close_deferred[0] = rd->close_deferred[1]; ++ rd->close_deferred[1] = NULL; ++ } ++} ++ ++/* ++ * The handle of a TYPED_V3 context is gone (DESTROY_CONTEXT, or its file closed): stop VM and ++ * render-target edits and queue the close item behind the context's jobs (g5p_rsched_close()). ++ * File lock held, or the file is closing. A context that never submitted is released by its ++ * last reference. ++ */ ++static void g5p_render_close(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ ++ if (!r->v3_accepted || r->closing) ++ return; ++ r->closing = true; ++ r->detached = true; ++ c->vm->resident_gone = true; ++ /* The close item's references, dropped by the scheduler once it ran. */ ++ kref_get(&c->ref); ++ drm_dev_get(&c->gpu->drm); ++ if (!g5p_rsched_close(c->gpu, r)) { ++ g5p_context_put(c); ++ drm_dev_put(&c->gpu->drm); ++ } ++} ++ ++/* Reboot quiesce, RAM only and lock free: the TYPED_V3 latch and whether its executor runs. ++ * The executor is RAM-only once the latch is quiet, unless the wake110 experiment (doorbells ++ * after retirement, off in every desktop tuple) is configured. Called after admission closed; ++ * the scheduler is kicked so that jobs waiting on an input fail now, not at their deadline. ++ */ ++static void g5p_kq_drm_state(void *cookie, struct g5p_ru_kq_in *in) ++{ ++ struct g5p_drm *d = cookie; ++ ++ /* Admission is closed by now: queued jobs blocked on an input fail at once. */ ++ g5p_rsched_kick(d); ++ ++ /* A channel blocked behind a lost job is a wedged epoch for the quiesce. */ ++ in->term_failed = g5p_ru_term_failed(&g5p_rv3_term) || ++ READ_ONCE(d->rdev->stuck) || ++ READ_ONCE(d->rdev->recovering); ++ in->ram_only = READ_ONCE(d->rdev->v3) && ++ !g5p_ru_mmio_ok(&g5p_rv3_term) && !(2); ++ in->executing = READ_ONCE(d->rdev->executing); ++} ++ ++/* Reboot quiesce, core lock held: every held user slot, the first claimant's pair included. ++ * Nothing is freed and no context changes state: admission is closed and the boot state is ++ * frozen right after, so no job runs again on this boot. ++ */ ++static int g5p_kq_drm_detach(void *cookie, const u32 *slots, u32 n) ++{ ++ return g5p_rv3_unbind_slots(cookie, slots, n, "kexec"); ++} ++ ++static const struct g5p_kq_frontend g5p_kq_drm = { ++ .state = g5p_kq_drm_state, ++ .detach = g5p_kq_drm_detach, ++}; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +new file mode 100644 +index 000000000..ac4dd841e +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* TYPED_V3 serial executor: per-job state (receipt data). Types only; code: drm-render-v3.inc. */ ++#ifndef G5P_DRM_RENDER_V3_H ++#define G5P_DRM_RENDER_V3_H ++ ++enum g5p_rv3_phase { ++ G5P_RV3_IDLE, ++ G5P_RV3_ATTACH, ++ G5P_RV3_BIND, ++ G5P_RV3_TA_PUBLISH, ++ G5P_RV3_TA_WAIT, ++ G5P_RV3_TD_PUBLISH, ++ G5P_RV3_TD_WAIT, ++ G5P_RV3_DONE, ++ G5P_RV3_PHASES, ++}; ++ ++/* g5p_drm.td_want: the 3D completion-event latch is armed for the endpoint in the low byte. */ ++#define G5P_RV3_TD_WANT 0x100U ++ ++/* One RAM sample of one engine: ring, stamps, channel watermarks, idle words, PM state, and ++ * (3D) whether the matching completion event has been latched. ++ */ ++struct g5p_rv3_ev { ++ u64 ns; ++ u32 prod, cons, stamp[2], ctl[5], idle[6], pm80; ++ bool proto, failed, consumed, retired, idle_ok, pm_ok, event; ++}; ++ ++struct g5p_rv3 { ++ struct g5p_ru_wait ta, td; ++ struct g5p_ru_touch ++ touch[2]; /* RC4a A/B (render-user-touch.h); zero when off */ ++ struct g5p_ru_retire rt; /* the 3D predicate (render-user-retire.h) */ ++ struct g5p_ru_ta tr; /* the TA predicate (render-user-retire.h) */ ++ struct g5p_rv3_ev last[2]; ++ u64 td_event_ns; /* decode time of the matching 3D event; 0 = none */ ++ u64 ta_event_ns; /* decode time of the first TA event since the publish */ ++ u64 attach_ns, job_deadline_ns, ta_pub_ns, td_pub_ns, ta_consumed_ns, ++ td_consumed_ns; ++ u32 phase, grace_end, bell; ++ u32 seq; /* the device sequence this job got; 0 = none */ ++ /* wake110 (render-user-wake.h), per engine: the decision, the power evidence word it saw ++ * (wake111), the two register words it read (U32_MAX = not read), the consumer it saw, the ++ * type-1 event counts at the phase's arm and at the job's end, the lock window and the ++ * first type-1 decode time. Zero when off. ++ */ ++ u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], ++ wk_t1_pub[2], wk_t1_end[2]; ++ u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ ++ /* The bind of this job's context published its DC3 pair and then failed: that context is ++ * lost, the device and the contexts already bound are not. ++ */ ++ bool bind_failed; ++ /* The job was lost alone (render-user-loss.h): its context is dead, the device is not; the ++ * channel stays blocked behind its stuck phase until the scheduler's drain clears it. ++ */ ++ bool lost; ++}; ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +new file mode 100644 +index 000000000..1d0a0a76d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -0,0 +1,3038 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* ++ * TYPED_V3 serial executor: the control flow that completed the natural TA and the ++ * natural 3D in r96 (docs/gpu/experiments/production-typed-render-kernel.md), on typed ++ * state. Included from drm-render.inc after g5p_render_execute(). ++ * ++ * What it does, in order, for each job a V3 context runs (strictly serial across every context, ++ * render-user-serial.h; only the epoch's job 1 attaches and has hardware evidence): ++ * 0. bind, the first job of every later context (multi-client-design.md §4.2): its own DC3 ++ * pair, published in RAM only, no doorbell, a RAM-only wait for the firmware to consume ++ * it. With the TEMPORARY render_user_spare_pairs instead: a pair job 1 pre-bound, taken ++ * over through the tag path. ++ * 1. attach (job 1): the typed preparation (PM, channels, catalogue entries), then the ++ * DC3 pair is published and its doorbell is owned by the r74f machinery (r87b ++ * self-drain: a closed gate is never repaired by the host). The firmware installs ++ * the page-table root and the generation tags itself (r74g); the host does NO MMU pass ++ * and checks the result in RAM only. ++ * 2. TA: record, WORK_CTL and producer, no doorbell; the r74g grace (four ALIVEs or ++ * 20 ms) lets the firmware drain it, and only if it is still pending afterwards the ++ * first ALIVE decoded owns one fenced fallback doorbell. ++ * 3. TA wait: RAM only (drain, sample, sleep), 20 s per phase by default, 30 s for the ++ * whole job (render-user-wait.h decides the deadline and closes the race with a ++ * completion). tahang1: complete on a coherent target with the PM saved and either ++ * the idle words clear, the TA completion event, or a 500 ms coherent hold ++ * (render-user-retire.h); a phase pending 1 s prints one RAM-only `slow` line. From ++ * the TA complete on, and after any timeout or fault, the host touches no register. ++ * 4. 3D: record, WORK_CTL and producer, NO doorbell (r96 stage 1: the firmware consumed ++ * it 2 ms later), RAM-only wait as above. ++ * release (render_user_touch_ms, default 0 = off): at publish + T of each phase, odd jobs read ++ * the receipt pages once and even jobs only record the instant (render-user-touch.h); RAM only. ++ * repeat109: the 3D phase is also complete when its target state is coherent and its completion ++ * event arrived, even if the engine idle words never clear (render-user-retire.h; the latched ++ * render_user_idle_settle_ms, default 0, adds a minimum coherent run); one `RENDER-V3 retire` ++ * receipt line splits the firmware's retirement from the host's decision. RAM only. ++ * wake110 (render_user_wake, default 0 = off): right after the producer store of every phase but ++ * job 1's TA, the A/B's factor phases run Apple's submit kick (R1+0x14 = 7, GPC active, R0+0xb00 = ++ * 0x10 / 0x11) and its control phases the same guards without a register (render-user-wake.h); ++ * one `RENDER-V3 wake` line per job. The only register access after the TA complete, licensed ++ * while no timeout or fault was noted. ++ * Jobs >= 2 (a compositor's next frame, or another context's): no attach, the PM pool is ++ * rebuilt, and both rings are published with no doorbell at all. ++ * Not in this path on purpose: the r74h / r80 / r85-r96 root-cause instrumentation (stream ++ * markers, PM canary, plateau and end banks, region/target dumps), host MMU passes, cache ++ * maintenance MMIO, selector reads, snapshots (they read R0), and any recovery. A timeout ++ * or fault retains everything; the context is frozen and kept until reboot. ++ * ++ * Terminal state (render-user-terminal.h): the TA complete, a timeout and a fault go through one ++ * first-writer-wins latch, g5p_rv3_terminal(). From then on g5p_ru_mmio_ok() is false, the admission ++ * refuses every job that would start behind a failure (-EIO), and the legacy typed paths that read or ++ * write registers (snapshot, invalidate, run, destroy) refuse. g5p_rv3_td_hook() is the optional ++ * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. ++ */ ++ ++static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { ++ "idle", "attach", "bind", "ta-publish", ++ "ta-wait", "3d-publish", "3d-wait", "done", ++}; ++static const char *const g5p_rv3_results[] = { "pending", "complete", "timeout", ++ "fault" }; ++static const char *const g5p_rv3_how[] = { "none", "idle-clear", "idle-stuck", ++ "hold" }; ++ ++#include "drm-latency123.inc" ++ ++/* perf60 stage 4 (render-user-pmset.h): per-context PM records, the iOS model (one ParamBuffer ++ * per work queue, 6cb9afc/6cb2378). Latched at the epoch's first attach, with boot_pm_sets: ++ * 1 every context bound after the epoch's first gets its own PM set (control + S, list, F in ++ * the firmware GiB), its own pool pages and its own aux pages at G5P_RU_PRIVATE_VA in its ++ * VM; the epoch's first context keeps the device PM. PM ID 0 everywhere. ++ * 2 as 1, and set s carries PM ID s + 1 (TA W+0x10, TA start, 3D start, tag6). ++ * RAM only, by the executor between jobs. ++ */ ++/* release8 default */ ++ ++/* The PM a context's jobs use: its set, or the device PM. */ ++static u8 *g5p_rv3_pm_cpu(struct g5p_boot *b, const struct g5p_render *r) ++{ ++ if (r->pm_set >= 0) ++ return b->pm_set[r->pm_set * G5P_PMSET_PAGES + G5P_PMSET_CONTROL] ++ .cpu; ++ return g5p_ru_pm_cpu(b); ++} ++ ++static u32 g5p_rv3_pm_meta(const struct g5p_render *r) ++{ ++ return g5p_meta_address( ++ r->pm_set >= 0 ? g5p_pmset_va(r->pm_set, G5P_PMSET_CONTROL) : ++ G5P_SHARED_VA + G5P_R22_PM); ++} ++ ++static u32 g5p_rv3_pm_id(const struct g5p_rdev *rd, const struct g5p_render *r) ++{ ++ return rd->pm_ctx >= 2 && r->pm_set >= 0 ? g5p_pmset_id(r->pm_set) : 0; ++} ++ ++static struct g5p_dma_page *g5p_rv3_pm_aux(struct g5p_rdev *rd, ++ struct g5p_render *r) ++{ ++ return r->pm_aux_mapped ? r->pm_aux : rd->aux; ++} ++ ++/* The boot tuple the V3 path is proven under (r96 params.txt), minus the instrumentation: ++ * r73b/r73c/r73e/r73f, r74 with the raised IRQ cap, r74f, r74g, r87b self-drain; no r74h, ++ * no r80..r96 (their IRQ banks read registers), no r75/r79 firmware control, and no fixed ++ * render engine (it is exclusive with the DRM one). ++ */ ++static bool g5p_rv3_tuple(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g ? g->boot : NULL; ++ struct g5p_r73b *x = b ? b->r73b : NULL; ++ ++ return x && g5p_r74g_on(b) && g5p_r74b_ram_only(g) && ++ x->r87b_attach == G5P_R87B_ATTACH_SELF_DRAIN; ++} ++ ++/* The admission state of a context and the device, as plain values for g5p_ru_v3_admit(). */ ++static struct g5p_ru_v3_state g5p_rv3_state(struct g5p_diag *g, ++ struct g5p_render *r) ++{ ++ const struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ ++ return (struct g5p_ru_v3_state){ ++ .tuple = g5p_rv3_tuple(g), ++ .v3_accepted = r->v3_accepted, ++ .bound = r->bound, ++ .jobs = r->jobs, ++ .device_up = READ_ONCE(rd->up), ++ .device_v3 = READ_ONCE(rd->v3), ++ .sequence = READ_ONCE(rd->sequence), ++ .slot_ta = r->slot[0], ++ .slot_3d = r->slot[1], ++ .variant = 0 || r->after_ta || 0 || r->v.pm || r->v.memory || ++ r->v.rt || r->v.slots, ++ .wait_ms = r->wait_ms, ++ .quiet = !g5p_ru_mmio_ok(&g5p_rv3_term), ++ .terminal_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ }; ++} ++ ++/* A terminal epoch is a device state (-EIO), the others are about this request. */ ++static int g5p_rv3_refusal_errno(int why) ++{ ++ if (why == G5P_RU_V3_TERMINAL) ++ return -EIO; ++ return why == G5P_RU_V3_OVERFLOW ? -EOVERFLOW : -EOPNOTSUPP; ++} ++ ++/* Submit-time admission: every V3 job refused here has touched nothing. */ ++static int g5p_rv3_admit_submit(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_ru_v3_state st; ++ int why; ++ ++ /* The epoch is being replaced: the executor admits the job again under the lock. */ ++ if (READ_ONCE(d->rdev->recovering)) ++ return 0; ++ st = g5p_rv3_state(g, r); ++ why = g5p_ru_v3_admit(&st, false); ++ ++ if (!why) ++ return 0; ++ /* per submit while admission is closed and a client retries: ratelimited */ ++ dev_info_ratelimited( ++ g->dev, "RENDER-V3 refused: %s; no hardware touched end=v3\n", ++ g5p_ru_v3_why[why]); ++ return g5p_rv3_refusal_errno(why); ++} ++ ++/* Recovery hooks (patches/gpu-recovery), compiled only when that series is applied: core.inc then includes ++ * teardown.inc before this file, and teardown.h defines G5P_TEARDOWN_H. The CAPTURE entry is observation only ++ * (RAM reads, no register access) and runs only with the recovery module parameter set, once per epoch, on a ++ * timeout or fault. Without the series both are nothing, and the prod98 kernel does not depend on it. ++ */ ++#ifdef G5P_TEARDOWN_H ++static void g5p_rv3_td_hook(struct g5p_diag *g, int kind, const char *why) ++{ ++ g5p_td_note_terminal(&g5p_td_state, g5p_ru_td_kind(kind)); ++ if ((0) && g5p_ru_term_capture_due(&g5p_rv3_term, kind)) ++ g5p_td_capture(g, why); ++} ++#else ++static inline void g5p_rv3_td_hook(struct g5p_diag *g, int kind, ++ const char *why) ++{ ++} ++#endif ++ ++/* The terminal state of the epoch: the TA complete, and any timeout or fault. First writer wins; from the first ++ * note on g5p_ru_mmio_ok() is false and the host touches no register (R0, R1, AIC, doorbell) and no bank. The r80 ++ * flag is set when an r80 block exists, so that a bank that might be latched cannot run behind this path (the V3 ++ * tuple forbids r80, so there is none today); the latch above does not depend on it. ++ */ ++static void g5p_rv3_terminal(struct g5p_diag *g, int kind, const char *why) ++{ ++ bool failed = g5p_ru_term_failed(&g5p_rv3_term); ++ ++ if (g5p_ru_term_note(&g5p_rv3_term, kind)) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s mmio=off r0=off export=ram-only end=v3\n", ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)], ++ why); ++ else if (!failed && g5p_ru_term_failed(&g5p_rv3_term)) ++ /* The epoch's TA complete took the latch long ago (job 1): a later timeout or fault ++ * only makes it sticky-failed, which closes admission for every client. Say so once. ++ */ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 terminal kind=%s why=%s latched=%s failed=1 admission=closed end=v3\n", ++ g5p_ru_term_names ++ [kind > G5P_RU_TERM_NONE && ++ kind < G5P_RU_TERM_KINDS ? ++ kind : ++ G5P_RU_TERM_FAULT], ++ why, ++ g5p_ru_term_names[g5p_ru_term_kind(&g5p_rv3_term)]); ++ g5p_rv3_td_hook(g, kind, why); ++} ++ ++/* "Retained boundary" of the r74f machinery -> the errno a job reports. -EINPROGRESS is ++ * never a job error (the UAPI uses it for "pending"). ++ */ ++static int g5p_rv3_errno(struct g5p_diag *g, int ret) ++{ ++ if (ret != -EINPROGRESS) ++ return ret; ++ switch (READ_ONCE(g->boot->r73b->r74f_stop)) { ++ case G5P_R74F_STOP_ATTACH_NOT_KICKED: ++ case G5P_R74F_STOP_ATTACH_NOT_CONSUMED: ++ case G5P_R74F_STOP_TA_NOT_KICKED: ++ case G5P_R74G_STOP_TA_NOT_CONSUMED: ++ case G5P_R74G_STOP_TA_WINDOW_MISSED: ++ return -ETIMEDOUT; ++ default: ++ return -EIO; ++ } ++} ++ ++/* ++ * Interim: on iOS the firmware writes back its own lines with the cache maintenance it runs on ++ * every tick (meta+0x6d4); that needs init words still being recovered (latency123.md), and ++ * this read stands in for it until then. ++ * ++ * While a phase is in flight the firmware does not see its own progress in the root and ++ * shared pages until a coherent read of those lines reaches the fabric: without one, the TA ++ * and 3D stages wait for the firmware tick, for seconds (latency123: key to frame 7.9 s ++ * median; with a Normal-NC read of both regions every 1-2 ms, 0.27 s). One 32-bit load per ++ * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU ++ * has no sampler, so it costs nothing then. ++ */ ++static void g5p_rv3_fw_touch(struct g5p_boot *b) ++{ ++ struct g5p_dma_page *const regions[] = { b->root, b->shared }; ++ u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; ++ ++ for (u32 r = 0; r < ARRAY_SIZE(regions); r++) ++ for (u32 i = 0; i < counts[r]; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE( ++ *(const u32 *)(regions[r][i].cpu + at)); ++ /* perf60 stage 4: the per-context PM sets are firmware pages like the shared ones */ ++ for (u32 i = 0; i < b->pm_sets * G5P_PMSET_PAGES; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(b->pm_set[i].cpu + at)); ++} ++ ++/* perf60 bell gates, A/B knobs for the multi-context 3D fault on the bell path (render_user_wake=2 only; ++ * 0600, read at every bell, all 0 = no gate). They run between a phase's producer store and its bell; ++ * in policy state 0 the firmware does not drain a ring on its own, so a gate holds the phase back. ++ * RAM reads, sleeps and barriers only; no register access. ++ * bell_gap_us (a) the TA bell waits until this long after the previous job (any context) ended ++ * bell_idle_ms (a) the TA bell first waits, up to this, for the 3D engine idle words to read zero ++ * bell_3d_delay_us (c) mb() and this delay between the 3D producer store and the 3D bell ++ * bell_ctx_gap_us (d) mb() and this delay before the TA bell of a job of another context than the last ++ * bell_touch (c) the Normal-NC read of the firmware pages (g5p_rv3_fw_touch) right before each bell ++ */ ++ ++/* perf60 job-start gate: the same waits before the job's own host writes (the PM pool rebuild and ++ * the shared TA/3D work records, g5p_rv3_pm_rebuild() and g5p_ru_job()), which happen before any ++ * publish. The bell gates above run after them, so they cannot separate "the host rewrites shared ++ * firmware pages while the firmware still finishes the previous job" from a doorbell race. ++ * job_gap_us the job's first host write no sooner than this after the previous job ended ++ * job_idle_ms then wait up to this (RAM) for the TA and 3D engine idle words to read zero ++ */ ++/* perf60 layout (latched at the epoch's first attach, default 0 = the r22 layout): ++ * render_user_job_slots 2: job seq writes its command, W record and microsequence into slot ++ * (seq - 1) % 2 of the channel page, so a job never rewrites the record the firmware used for ++ * the job before it; the pointer ring carries each job's addresses (render-user-layout.h). ++ * render_user_pm_once 1: the PM pool, list and F are built by the epoch's first attach only and ++ * never rewritten by the host between jobs; the TA's tag6 record has the firmware rebuild F, ++ * as on iOS (PM created per queue once; docs/gpu/experiments/freelist-lifetime-production-fix.md). ++ */ ++/* release8 default */ ++/* release8 default */ ++ ++/* The record slot of job seq on this epoch's layout. */ ++static u32 g5p_rv3_js(const struct g5p_rdev *rd, u32 seq) ++{ ++ return rd->job_slots == 2 ? (seq - 1) & 1 : 0; ++} ++ ++ ++ ++ ++ ++ ++ ++ ++ ++/* RAM only. */ ++static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_rv3_ev *s) ++{ ++ struct g5p_boot *b = g->boot; ++ u8 *p = g5p_ru_page(b, e), *global = g5p_ru_global(b, e), ++ *pm = g5p_rv3_pm_cpu(b, r); ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ ++ g5p_rv3_fw_touch(b); ++ memset(s, 0, sizeof(*s)); ++ dma_rmb(); ++ s->ns = ktime_get_ns(); ++ s->prod = READ_ONCE(ring[4]); ++ s->cons = READ_ONCE(ring[0]); ++ for (u32 i = 0; i < ARRAY_SIZE(s->stamp); i++) ++ s->stamp[i] = g5p_le32(p + G5P_R22_STAMP + 4 * i); ++ for (u32 i = 0; i < ARRAY_SIZE(s->ctl); i++) ++ s->ctl[i] = g5p_le32(p + G5P_WORK_CTL + i * 0x10); ++ for (u32 i = 0; i < ARRAY_SIZE(s->idle); i++) ++ s->idle[i] = g5p_le32(global + 0x724 + i * 4); ++ s->pm80 = g5p_le32(pm + 0x80); ++ s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ s->proto = s->prod != r->target[e] || s->cons > 255 || ++ !g5p_ru_stamp_sane(p, seq); ++ s->consumed = s->cons == r->target[e]; ++ /* The predicates of the flown r74b (TA) and r96 (3D) waits; the 3D's PM serial is a receipt. */ ++ s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2] && ++ (!e || (!s->idle[4] && !s->idle[5])); ++ s->retired = g5p_ru_retired_at(p, g5p_ru_meta(e), e, seq, ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ s->pm_ok = e ? s->pm80 == r->ctx->gpu->rdev->pm_serial + 1 : ++ g5p_r22_pm_saved(pm + 0x80); ++ s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns) : ++ READ_ONCE(r->ctx->gpu->ta_hit_ns)); ++} ++ ++/* TA: one step of render-user-retire.h's TA rules when ta is given, else the flown r74b ++ * predicate. 3D: one step of its 3D rules when rt is given, else the r96 predicate. ++ */ ++static bool g5p_rv3_done(u32 e, const struct g5p_rv3_ev *s, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta) ++{ ++ if (e && rt) ++ return g5p_ru_rt_step(rt, s->ns, s->consumed, s->retired, ++ s->idle_ok, ++ s->event) != G5P_RU_RT_PENDING; ++ if (!e && ta) ++ return g5p_ru_ta_step(ta, s->ns, s->consumed, s->retired, ++ s->pm_ok, s->idle_ok, ++ s->event) != G5P_RU_TA_PENDING; ++ return s->consumed && s->idle_ok && s->retired && (e || s->pm_ok); ++} ++ ++/* The TA clauses in the bits of render-user-retire.h, for the receipt. */ ++static u32 g5p_rv3_ta_unsat(const struct g5p_rv3_ev *s) ++{ ++ return (s->consumed ? 0 : G5P_RU_RT_CONSUMED) | ++ (s->retired ? 0 : G5P_RU_RT_RETIRED) | ++ (s->idle_ok ? 0 : G5P_RU_RT_IDLE) | ++ (s->pm_ok ? 0 : G5P_RU_RT_PM); ++} ++ ++/* Arm engine e's completion-event latch of g5p_drm_event() (3D: for this job's endpoint), before ++ * the phase's producer store, so an event decoded right after the publish is not missed. The ++ * latch is set under status_lock by whichever CPU decodes the event, and the wait samples it ++ * level-triggered every 1-2 ms, so there is no wakeup to lose. RAM only. ++ */ ++static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) ++{ ++ unsigned long flags; ++ ++ BUILD_BUG_ON(G5P_RU_RT_GRID != G5P_WORK_SLOTS); ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = G5P_RV3_TD_WANT | endpoint; ++ WRITE_ONCE(d->td_hit_ns, 0); ++ } else { ++ d->ta_want = true; ++ WRITE_ONCE(d->ta_hit_ns, 0); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) ++{ ++ unsigned long flags; ++ u64 hit; ++ ++ spin_lock_irqsave(&d->status_lock, flags); ++ if (e) { ++ d->td_want = 0; ++ hit = d->td_hit_ns; ++ } else { ++ d->ta_want = false; ++ hit = d->ta_hit_ns; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return hit; ++} ++ ++/* release: the receipt's read set (g5p_rv3_receipt()'s blobs: PM-control, then W, channels and the ++ * stamp window of the TA and of the 3D page), read once with 32-bit loads, the access size of the ++ * receipt's hex dump. The words are folded, not copied: no allocation, no store to shared memory. ++ * RAM only: no register, no doorbell, no print, no latch. ++ */ ++static u32 g5p_rv3_touch_sweep(struct g5p_boot *b, u32 *bytes) ++{ ++ u8 *ta = g5p_ru_page(b, 0), *td = g5p_ru_page(b, 1); ++ const u8 *const src[] = { ++ g5p_ru_pm_cpu(b), ta + G5P_R22_W, ta, ++ ta + G5P_R22_TQ, td + G5P_R22_W, td, ++ td + G5P_R22_TQ, ++ }; ++ static const u32 len[] = { ++ 0x140, G5P_RU_TA_W_BYTES, 0x340, 0x80, G5P_RU_3D_W_BYTES, 0x340, ++ 0x80, ++ }; ++ u32 sum = 0, n = 0; ++ ++ dma_rmb(); ++ for (u32 i = 0; i < ARRAY_SIZE(src); i++) { ++ for (u32 at = 0; at < len[i]; at += 4) ++ sum = (sum << 5 | sum >> 27) ^ ++ READ_ONCE(*(const u32 *)(src[i] + at)); ++ n += len[i]; ++ } ++ *bytes = n; ++ return sum; ++} ++ ++/* release: the instant of one phase, taken at the sample at now_ns. The touch arm reads, the control ++ * arm only records; both record the same instant. ++ */ ++static void g5p_rv3_touch(struct g5p_diag *g, struct g5p_ru_touch *t, ++ u64 now_ns) ++{ ++ u32 bytes = 0, sum = 0; ++ ++ if (t->sweep) ++ sum = g5p_rv3_touch_sweep(g->boot, &bytes); ++ g5p_ru_touch_mark(t, now_ns, t->sweep ? ktime_get_ns() : now_ns, bytes, ++ sum); ++} ++ ++/* tahang1: a phase still pending this long after its publish prints its RAM state once. */ ++#define G5P_RV3_SLOW_MS 1000U ++ ++/* The one `slow` line of a phase: the deciding sample's RAM words and the waiter's own state, so ++ * a stall that later times out is described from the first second on. RAM only. ++ */ ++static void g5p_rv3_slow(struct g5p_diag *g, u32 e, u32 seq, ++ const struct g5p_ru_wait *w, ++ const struct g5p_rv3_ev *s, u32 unsat, u64 hit_ns) ++{ ++ dev_warn( ++ g->dev, ++ "RENDER-V3 slow seq=%u engine=%u ms=%lld ring=%u/%u stamp=%08x/%08x " ++ "ctl=%08x,%08x,%08x,%08x,%08x idle=%08x,%08x,%08x,%08x,%08x,%08x pm80=%08x " ++ "consumed=%u retired=%u pm=%u event_ms=%lld unsat=%#x busy_ms=%lld events=%u " ++ "end=v3\n", ++ seq, e, g5p_ru_rt_ms(w->start_ns, s->ns), s->prod, s->cons, ++ s->stamp[0], s->stamp[1], s->ctl[0], s->ctl[1], s->ctl[2], ++ s->ctl[3], s->ctl[4], s->idle[0], s->idle[1], s->idle[2], ++ s->idle[3], s->idle[4], s->idle[5], s->pm80, s->consumed, ++ s->retired, s->pm_ok, g5p_ru_rt_ms(w->start_ns, hit_ns), unsat, ++ g5p_ru_rt_ms(w->start_ns, w->busy_ns), ++ READ_ONCE(g->boot->events)); ++ g5p_np_policy(g, "slow", seq, e); ++} ++ ++/* nopatch: the policy record at a few phase ends (log-spaced) and at every timeout. RAM only. */ ++static void g5p_rv3_np_end(struct g5p_diag *g, u32 e, u32 seq, u32 result) ++{ ++ if (seq == 1 || seq == 10 || seq == 100 || seq == 1000 || ++ seq == 10000 || result == G5P_RU_WAIT_TIMEOUT) ++ g5p_np_policy(g, ++ result == G5P_RU_WAIT_TIMEOUT ? "phase-timeout" : ++ "phase-end", ++ seq, e); ++} ++ ++/* One phase's RAM-only wait. g5p_ru_wait_step()/close() decide; this loop only samples. release: the ++ * touch instant is taken after a step that left the phase undecided, before the sleep (off: never). ++ * The predicate is level-triggered: every sample, including the one right after each sleep and the ++ * closing one at the deadline, re-reads the whole RAM state (rt: the 3D, ta: the TA). ++ */ ++static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, struct g5p_ru_touch *t, ++ struct g5p_ru_retire *rt, struct g5p_ru_ta *ta, ++ struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) ++{ ++ struct g5p_drm *d = r->ctx->gpu; ++ bool closing = false, slow = false; ++ ++ for (;;) { ++ int drain = g5p_boot_drain(g); ++ u64 seen = READ_ONCE( ++ d->event_serial); /* after the drain, before the sample */ ++ bool fault, silent, done; ++ ++ g5p_rv3_sample(g, r, e, seq, s); ++ /* latency101: a silent event stream during the phase is a dead firmware (never off). */ ++ silent = ++ g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); ++ g5p_ru_wait_busy(w, s->ns, !s->idle_ok); ++ fault = drain || s->failed || s->proto || silent; ++ if (drain) ++ *err = drain; ++ else if (s->proto) ++ *err = -EPROTO; ++ else if (s->failed) ++ *err = -EIO; ++ else if (silent) ++ *err = -EIO; ++ if (s->consumed && !*consumed_ns) ++ *consumed_ns = s->ns; ++ done = g5p_rv3_done(e, s, rt, ta); ++ if (closing) ++ g5p_ru_wait_close(w, s->ns, done, fault); ++ else ++ g5p_ru_wait_step(w, s->ns, done, fault); ++ if (rt) ++ g5p_ru_rt_note(rt, w->result); ++ if (ta) ++ g5p_ru_ta_note(ta, w->result); ++ if (w->result) { ++ g5p_rv3_np_end(g, e, seq, w->result); ++ return w->result; ++ } ++ if (!slow && s->ns - w->start_ns >= ++ (u64)G5P_RV3_SLOW_MS * NSEC_PER_MSEC) { ++ slow = true; ++ g5p_rv3_slow(g, e, seq, w, s, ++ rt ? rt->unsat : ++ ta ? ta->unsat : ++ 0, ++ e ? READ_ONCE(d->td_hit_ns) : ++ READ_ONCE(d->ta_hit_ns)); ++ } ++ if (w->expired) { ++ closing = true; /* the closing sample, at once */ ++ continue; ++ } ++ if (g5p_ru_touch_due(t, s->ns, w->result, w->expired)) ++ g5p_rv3_touch(g, t, s->ns); ++ g5p_l123_pause(d, g, e, s, seen); ++ } ++} ++ ++/* One phase's deadline: the legacy single bound, or (latency101) the soft/hard pair. */ ++static void g5p_rv3_wait_start(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_ru_wait *w, u64 pub_ns, ++ u64 job_deadline_ns) ++{ ++ if (r->hard_ms) ++ g5p_ru_wait_start_lat(w, pub_ns, r->soft_ms, r->hard_ms, ++ job_deadline_ns, ++ READ_ONCE(g->boot->events)); ++ else ++ g5p_ru_wait_start(w, pub_ns, r->wait_ms, job_deadline_ns); ++} ++ ++/* After a timeout (or fault) was decided: the one RAM look of the legacy path, or (latency101, ++ * timeout only) a RAM-only observation window that keeps draining and sampling at the wait cadence ++ * and prints nothing, so a completion that lands milliseconds after the decision is recorded ++ * (late_after/late_ns) before the receipt line and the terminal latch. It never changes the ++ * result and never touches a register. The 3D look steps a copy of the predicate's record, so the ++ * receipt keeps the deciding sample's. ++ */ ++static void g5p_rv3_observe(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_ru_wait *w, ++ const struct g5p_ru_retire *rt, ++ const struct g5p_ru_ta *ta, struct g5p_rv3_ev *late) ++{ ++ struct g5p_ru_retire copy, *c = NULL; ++ struct g5p_ru_ta tcopy, *tc = NULL; ++ ++ if (rt) { ++ copy = *rt; ++ c = © ++ } ++ if (ta) { ++ tcopy = *ta; ++ tc = &tcopy; ++ } ++ if (!r->obs_ms || w->result != G5P_RU_WAIT_TIMEOUT) { ++ g5p_rv3_sample(g, r, e, seq, late); ++ g5p_ru_wait_after(w, g5p_rv3_done(e, late, c, tc)); ++ return; ++ } ++ do { ++ g5p_boot_drain( ++ g); /* a fault latches b->failed, the result stays */ ++ g5p_rv3_sample(g, r, e, seq, late); ++ if (!g5p_ru_wait_observe(w, late->ns, ++ g5p_rv3_done(e, late, c, tc), ++ r->obs_ms)) ++ break; ++ usleep_range(1000, 2000); ++ } while (1); ++} ++ ++/* wake110: render-user-wake.h's register access, on the Fender and RGX windows. */ ++static void g5p_rv3_wk_w1(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->window[1] + reg); ++} ++ ++/* The GPC state read of g5p_wk_ring() comes right after its R1+0x14 = 7 wake: in bell mode ++ * (render_user_wake=2) it waits up to G5P_RV3_WK_SETTLE_US, re-reading every 10 us, for a settled ++ * powered pstate (bit 4 clear, nibble nonzero), as Apple's setGPCPerformanceState waits for bit 4. ++ * Reads only; the ring's own check of the final value is unchanged. ++ */ ++#define G5P_RV3_WK_SETTLE_US 2000U ++ ++static u32 g5p_rv3_wk_r1(void *ctx, u32 reg) ++{ ++ void __iomem *r1 = ((struct g5p_diag *)ctx)->window[1]; ++ u32 raw = readl(r1 + reg); ++ u64 t0; ++ ++ if (reg != G5P_WK_R1_GPC || (2) != G5P_WK_ALWAYS || ++ (!(raw & 0x10) && (raw & 0xf))) ++ return raw; ++ t0 = ktime_get_mono_fast_ns(); ++ while (((raw & 0x10) || !(raw & 0xf)) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_WK_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ raw = readl(r1 + reg); ++ } ++ return raw; ++} ++ ++static void g5p_rv3_wk_wmb(void *ctx) ++{ ++ dma_wmb(); ++} ++ ++static void g5p_rv3_wk_w0(void *ctx, u32 reg, u32 value) ++{ ++ writel(value, ((struct g5p_diag *)ctx)->rgx + reg); ++} ++ ++/* wake110: an r74f bell that still owns its ring (the ALIVE decoder may fire it). */ ++static bool g5p_rv3_wk_r74f_busy(struct g5p_r73b *x) ++{ ++ u32 k = READ_ONCE(x->r74f_active); ++ int s; ++ ++ if (!k || k > G5P_R74F_KINDS) ++ return false; ++ s = atomic_read_acquire(&x->r74f_bell[k - 1].state); ++ return s == G5P_R74F_PUBLISHED || s == G5P_R74F_OPEN || ++ s == G5P_R74F_FIRING; ++} ++ ++/* wake111: DS108's claim proof re-read at the publish: ALIVE proven in the current reset ++ * generation, the claim admitted in it, nothing latched failed since, no R74E power abort. ++ * Called under event_lock (interrupts off); g5p_ds_lock is a leaf (g5p_ds_event() is pure). ++ */ ++static bool g5p_rv3_wk_ds108(struct g5p_boot *b) ++{ ++ bool ok; ++ ++ spin_lock(&g5p_ds_lock); ++ ok = g5p_ds_proven(&g5p_ds_state) && ++ g5p_ds_ok(&g5p_ds_state, G5P_DSE_CLAIM); ++ spin_unlock(&g5p_ds_lock); ++ return ok && !(b->r73b && READ_ONCE(b->r73b->early_power_abort)); ++} ++ ++/* wake110: before the phase's producer store, the engine's type-1 count and a cleared first-decode ++ * time, so an event decoded right after the publish is counted. RAM only. ++ */ ++static void g5p_rv3_wake_mark(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ v->wk_t1_pub[e] = d->t1_count[e]; ++ WRITE_ONCE(d->t1_first_ns[e], 0); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++} ++ ++/* wake110: at the job's end, the type-1 counts and the first decode of each published phase. */ ++static void g5p_rv3_wake_end(struct g5p_drm *d, const struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&d->status_lock, flags); ++ for (u32 e = 0; e < 2; e++) { ++ v->wk_t1_end[e] = d->t1_count[e]; ++ v->wk_first_ns[e] = d->t1_first_ns[e]; ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (v->phase < G5P_RV3_TA_PUBLISH) ++ v->wk_first_ns[0] = 0; ++ if (v->phase < G5P_RV3_TD_PUBLISH) ++ v->wk_first_ns[1] = 0; ++} ++ ++/* wake110: microseconds / milliseconds from a phase's publish to a later instant; -1 = none. */ ++static s64 g5p_rv3_wk_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++static s64 g5p_rv3_wk_ms(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_MSEC : ++ -1; ++} ++ ++/* wake110: one phase's publish-time decision, right after its producer store and barrier. Both ++ * arms take the event lock (the r74f owner's serialization against the decoder) and sample the ++ * same RAM; only a factor that passed every guard runs g5p_wk_ring(). Nothing is printed under ++ * the lock. Off: returns at once. ++ */ ++static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 e, u32 start, u32 seq) ++{ ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; ++ struct g5p_boot *b = g->boot; ++ struct g5p_drm *d = r->ctx->gpu; ++ u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ u8 *p = g5p_ru_page(b, e); ++ struct g5p_wk_in in; ++ unsigned long flags; ++ ++ if (!r->wake) ++ return; ++ spin_lock_irqsave(&b->event_lock, flags); ++ v->wk_start_ns[e] = ktime_get_ns(); ++ dma_rmb(); ++ in = (struct g5p_wk_in){ ++ .mode = r->wake, ++ .seed = 0, ++ .seq = seq, ++ .e = e, ++ .attempted = g5p_wk_attempted(v->wk_did[e]), ++ .term_failed = g5p_ru_term_failed(&g5p_rv3_term), ++ .boot_failed = READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped), ++ .power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)), ++ .r74f_pending = g5p_rv3_wk_r74f_busy(b->r73b), ++ .start = start, ++ .target = r->target[e], ++ .prod = READ_ONCE(ring[4]), ++ .cons = READ_ONCE(ring[0]), ++ .stamp = { g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4) }, ++ .type1 = READ_ONCE(d->t1_count[e]) - v->wk_t1_pub[e], ++ }; ++ v->wk_cons[e] = in.cons; ++ v->wk_power[e] = in.power; ++ v->wk_did[e] = g5p_wk_decide(&in, &io, &v->wk_fence[e], &v->wk_gpc[e]); ++ v->wk_done_ns[e] = ktime_get_ns(); ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ if (v->wk_did[e] == G5P_WK_DID_RANG) ++ r->wake_rang++; ++} ++ ++/* The epoch's first job: the device (PM, channels) and this context's DC3 records, shared with ++ * g5p_render_attach(), then the r74f/r74g protocol of the fixed r74f_attach() on typed state. ++ */ ++static int g5p_rv3_attach(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline; ++ u32 start, tag, bad; ++ int ret; ++ ++ ret = g5p_r74f_admit(g, G5P_R74F_DC3); ++ if (ret) ++ return ret; ++ ret = g5p_render_attach_prepare(d, r, &start, &tag); ++ if (ret) ++ return ret; ++ r->dc_target = (start + 2) & 255; ++ r->bound = true; ++ rd->up = true; ++ rd->v3 = true; ++ rd->job_slots = (2) == 2 ? 2 : 1; ++ rd->pm_once = (1) == 1; ++ rd->pm_ctx = b->pm_sets ? min((3), 3U) : 0; ++ r->pm_first = r->pm_set < 0; ++ dev_info( ++ g->dev, ++ "RENDER-V3 layout job_slots=%u pm_once=%u slot1=%#x..%#x pm_ctx=%u pm_sets=%u " ++ "end=v3\n", ++ rd->job_slots, rd->pm_once, G5P_RU_SLOT1, G5P_RU_SLOT1_END, ++ rd->pm_ctx, b->pm_sets); ++ g5p_render_retain(r); ++ v->touched = true; ++ WRITE_ONCE(r->ctx->live, true); ++ dma_wmb(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ v->attach_ns = ktime_get_ns(); ++ g5p_r74_mark(g, G5P_R74_ATTACH_PUBLISHED, r->dc_target); ++ /* The producer is final from here; only the doorbell may be pending. */ ++ ret = g5p_r74f_bell(g, G5P_R74F_DC3, start, r->dc_target); ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || READ_ONCE(b->failed)) ++ return ret ?: -EIO; ++ dma_rmb(); ++ if (READ_ONCE(ctl[4]) != r->dc_target || ++ READ_ONCE(ctl[0]) > 255) ++ return -EPROTO; ++ if (READ_ONCE(ctl[0]) == r->dc_target) { ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ bad = g5p_r74g_check(g, 0, r->slot, vm->root.dma, tag, ++ start); ++ g5p_r74_mark(g, G5P_R74_ATTACH_CONSUMED, r->dc_target); ++ g5p_r74g_dc3_line(g, 0, false); ++ if (bad) ++ return g5p_rv3_errno(g, ++ g5p_r74g_refuse(g, bad)); ++ for (u32 e = 0; e < 2; e++) ++ g5p_mmu_record( ++ g, ++ "RENDER tags slot=%u host=%08x cached=%08x expected=%08x\n", ++ r->slot[e], ++ g5p_le32(g5p_boot_shared(b, 0x84) + ++ r->slot[e] * 4), ++ g5p_le32(g5p_boot_shared(b, 0x85) + ++ r->slot[e] * 4), ++ tag); ++ g5p_mmu_record( ++ g, ++ "RENDER-USER attached TA=%u 3D=%u PC=%pad tag=%08x blocks=%u; RAM-only, firmware " ++ "DC3 installed CBASE, host MMU pass skipped\n", ++ r->slot[0], r->slot[1], &vm->root.dma, tag, ++ rd->blocks); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } while (time_before(jiffies, deadline)); ++ g5p_r74f_consumed(g, G5P_R74F_DC3); ++ return g5p_rv3_errno(g, ++ g5p_r74f_stop(g, G5P_R74F_STOP_ATTACH_NOT_CONSUMED, ++ READ_ONCE(ctl[0]))); ++} ++ ++/* perf60 bell mode (render_user_wake=2): a DC publication (bind, spare pairs, unbind) gets Apple's ++ * submit kick too. With the policy held in state 0 the firmware no longer drains the DC ring on ++ * its own (perf60clk-9ec288b868: a second client's RAM-only DC3 bind was never consumed). The ++ * wake path's guards (event lock, power proof, no fault or terminal, no r74f bell owning a ring, ++ * the ring holding exactly this publication), then Apple's order: R1+0x14 = 7 and its read-back, ++ * R1+0x1000 re-read every 10 us until a settled powered pstate (g5p_gpc_ok(): bit 4 clear, ++ * nibble nonzero; Apple's setGPCPerformanceState waits for bit 4 the same way), at most ++ * G5P_RV3_DC_SETTLE_US, and R0+0xb00 = 0x10 only then. A GPC that never reads on is not written: ++ * no R0 access, the publication waits as before. Off (render_user_wake != 2): returns at once. ++ */ ++#define G5P_RV3_DC_SETTLE_US 2000U ++ ++static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, ++ const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 fence = U32_MAX, gpc = U32_MAX, power, cons, n = 0; ++ const char *did = "skip-guard"; ++ unsigned long flags; ++ u64 t0; ++ ++ ; ++ ; ++ spin_lock_irqsave(&b->event_lock, flags); ++ dma_rmb(); ++ cons = READ_ONCE(ctl[0]); ++ power = g5p_wk_power_mask(READ_ONCE(g->powered), ++ READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), ++ g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ !g5p_wk_power_ready(power) || g5p_rv3_wk_r74f_busy(b->r73b) || ++ cons != start || READ_ONCE(ctl[4]) != target) ++ goto out; ++ writel(7, g->window[1] + 0x14); ++ fence = readl(g->window[1] + 0x14); ++ if (fence & ~7U) { ++ did = "skip-fence"; ++ goto out; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ gpc = readl(g->window[1] + 0x1000); ++ while (!g5p_gpc_ok(gpc) && ++ ktime_get_mono_fast_ns() - t0 < ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) { ++ udelay(10); ++ gpc = readl(g->window[1] + 0x1000); ++ n++; ++ } ++ if (!g5p_gpc_ok(gpc)) { ++ did = "skip-gpc"; ++ goto out; ++ } ++ /* the publication's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ ++ wmb(); ++ writel(0x10, g->rgx + 0xb00); ++ did = "rang"; ++out: ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 dc-kick why=%s did=%s dc=%u->%u power=%02x fence=%08x gpc=%08x " ++ "settle=%u end=v3\n", ++ why, did, cons, target, power, fence, gpc, n); ++} ++ ++/* RAM only: drain the event stream, read the firmware pages (interim, see g5p_rv3_fw_touch) and ++ * sample the DC ring until its consumer reaches `target` ++ * or G5P_RU_BIND_MS pass. No doorbell and no register, whether or not a job has retired. ++ */ ++static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ unsigned long deadline = jiffies + msecs_to_jiffies(G5P_RU_BIND_MS); ++ ++ for (;;) { ++ bool expired = time_after_eq(jiffies, deadline); ++ int ret = g5p_boot_drain(g); ++ ++ if (ret) ++ return ret; ++ if (READ_ONCE(b->failed)) ++ return -EIO; ++ g5p_rv3_fw_touch(b); ++ dma_rmb(); ++ switch (g5p_ru_bind_step(READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), ++ target, expired)) { ++ case G5P_RU_BIND_CONSUMED: ++ *consumed_ns = ktime_get_ns(); ++ return 0; ++ } ++ usleep_range(100, 200); ++ } ++} ++ ++static s64 g5p_rv3_us(u64 from_ns, u64 to_ns) ++{ ++ return from_ns && to_ns >= from_ns ? ++ (s64)(to_ns - from_ns) / NSEC_PER_USEC : ++ -1; ++} ++ ++/* ++ * The first job of a context on a device that is already up binds the context's slot pair ++ * with a DC3 pair published in RAM only (multi-client-design.md §4.2). No doorbell and no ++ * register access of any kind, so it is safe after a job retired and while the GPU may be ++ * idle-gated: the firmware services the DC ring in its scheduler pass, before TA and 3D, and ++ * the host waits on RAM. Core lock held, between jobs. ++ * ++ * Refused before the producer store, it touched nothing. From the store on, the firmware may ++ * hold the pair: the context is retained, and on failure it is lost (v->bind_failed) but the ++ * device and the contexts already bound are not. A pair not consumed in time keeps every later ++ * bind refused until the firmware catches up. ++ */ ++/* ++ * render_user_close_park (release8 quiet2, docs/gpu/experiments/release8-quiet2.md). The PM writes ++ * a closed context's free list back through that context's TA slot after the context is gone, and ++ * not only while it is the last or the previous distinct context (render-user-pmset.h): two bench ++ * faults wrote the free list of a context two switches back, through its slot, on the 2nd job of a ++ * third context, after the pm-prev rule had unbound it (page-invalid level 2, its PC level empty). ++ * iOS never leaves the PM's target unresolvable: its PM is the device's (shared by the work queues, ++ * AGXWorkQueue_initAndAttachPM), in the kernel's GPU address space, and a GART keeps its slots until ++ * the address space is destroyed (AGXGart_detachSlotsAndDestroyAddressSpace). Here, a closed ++ * context's pair is bound again, right after its unbind (between jobs, core lock held), to the park ++ * catalogue: a kernel page that maps only the firmware window, where the PM sets live. A late ++ * write-back then lands in the closed context's own set page (released, reused last) instead of ++ * faulting. A later bind of the pair unbinds it first. Both steps are the proven DC3 ++ * transitions (PA to zero, zero to PA); a park that fails leaves the pair unbound, as before. ++ */ ++/* release8 default */ ++ ++static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, ++ const char *why); ++ ++/* Before a bind, core lock held: a pair a close parked (RAM: park PA and tag on both slots) is ++ * unbound first, by its own DC3. 0 when the pair was not parked or the unbind verified. ++ */ ++static int g5p_rv3_unpark(struct g5p_drm *d, const u32 *slot) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ ++ if (!d->rdev->park.cpu) ++ return 0; ++ dma_rmb(); ++ g5p_render_bind_mem(b, slot, READ_ONCE(ctl[4]), &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!g5p_ru_bind_parked(&s, d->rdev->park.dma)) ++ return 0; ++ return g5p_rv3_unbind_slots(d, slot, 2, "unpark"); ++} ++ ++static int g5p_rv3_bind(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s = {}; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ u32 start, tag, bad = 0; ++ int ret; ++ ++ dma_rmb(); ++ if (rd->dc_pending && READ_ONCE(ctl[0]) != rd->dc_wait) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u dc=%u/%u wait=%u: an earlier bind is " ++ "pending; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4]), rd->dc_wait); ++ return -EIO; ++ } ++ rd->dc_pending = false; ++ ret = g5p_rv3_unpark(d, r->slot); ++ if (ret) ++ return ret; ++ ret = g5p_render_vm_prepare(d, r, &start, &tag); ++ if (ret) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u ret=%d dc=%u/%u: ring busy or slot in use; " ++ "no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], ret, READ_ONCE(ctl[0]), ++ READ_ONCE(ctl[4])); ++ return ret; ++ } ++ r->dc_target = (start + 2) & 255; ++ g5p_render_retain(r); ++ v->bind_failed = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], r->dc_target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, r->dc_target, "bind"); ++ ret = g5p_rv3_dc_wait(g, r->dc_target, &consumed_ns); ++ if (ret == -ETIMEDOUT) { ++ rd->dc_pending = true; ++ rd->dc_wait = r->dc_target; ++ } ++ dma_rmb(); ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ if (!ret) { ++ bad = g5p_ru_bind_check(&s, r->slot, vm->root.dma, tag, ++ r->dc_target); ++ ret = bad ? -EUCLEAN : 0; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=%lld tag=%08x host=%08x/%08x " ++ "cached=%08x/%08x dc=%u->%u/%u bad=%#x pc=%pad mode=ram-dc3 end=v3\n", ++ r->slot[0], r->slot[1], ret, g5p_rv3_us(pub_ns, consumed_ns), ++ tag, s.host[0], s.host[1], s.cached[0], s.cached[1], start, ++ s.cons, s.prod, bad, &vm->root.dma); ++ if (ret) ++ return ret; ++ v->bind_failed = false; ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ return 0; ++} ++ ++/* ++ * TEMPORARY fallback (render_user_spare_pairs, read at the epoch's first job), for a firmware ++ * that never drains a DC3 published after a job retired. Before job 1's attach, bind that ++ * many extra slot pairs, each to a zeroed kernel-owned catalogue, in one RAM-only batch with no ++ * doorbell. The firmware drains it on its own like the self-drained attach of this boot tuple ++ * (r87b); job 1's own attach, doorbell and r74g checks then run unchanged behind it ++ * (g5p_r74f_ring() refuses a ring that holds more than that attach's pair). Later contexts ++ * take a pair over with g5p_rv3_adopt(). A pair that does not check out is never used. Core ++ * lock held, device not up, so register access is still allowed (none is made here). ++ */ ++static int g5p_rv3_spares(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 want = min((0), G5P_RU_MAX_SPARES); ++ u32 start = READ_ONCE(ctl[4]), target, n, ok = 0; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u64 pub_ns, consumed_ns = 0; ++ unsigned long flags; ++ int ret; ++ ++ if (!want || rd->spares) ++ return 0; ++ rd->spares = kcalloc(want, sizeof(*rd->spares), GFP_KERNEL); ++ if (!rd->spares) ++ return -ENOMEM; ++ for (n = 0; n < want; n++) { ++ struct g5p_rspare *sp = &rd->spares[n]; ++ bool pair; ++ ++ spin_lock_irqsave(&g->slot_lock, flags); ++ pair = g5p_ru_slot_pair(g->user_slots, false, sp->slot); ++ if (pair) { ++ __set_bit(sp->slot[0], g->user_slots); ++ __set_bit(sp->slot[1], g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ if (!pair) ++ break; ++ g5p_render_bind_mem(b, sp->slot, start + 2 * n, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ if (!g5p_ru_bind_busy(&s) && ++ !g5p_vm_alloc_page(d->drm.dev, &sp->root)) { ++ g5p_table_clean_page(g, &sp->root); ++ continue; ++ } ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(sp->slot[0], g->user_slots); ++ __clear_bit(sp->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ break; ++ } ++ if (!n) ++ return 0; ++ dsb(sy); ++ for (u32 i = 0; i < n; i++) { ++ g5p_render_bind_mem(b, rd->spares[i].slot, start + 2 * i, &m); ++ g5p_ru_bind_publish(&m, rd->spares[i].slot, ++ rd->spares[i].root.dma, G5P_RU_SPARE_TAG); ++ } ++ rd->spare_count = n; ++ target = (start + 2 * n) & 255; ++ v->touched = true; ++ dma_wmb(); ++ pub_ns = ktime_get_ns(); ++ smp_store_release(&ctl[4], target); ++ dma_wmb(); ++ g5p_rv3_dc_kick(g, start, target, "spares"); ++ ret = g5p_rv3_dc_wait(g, target, &consumed_ns); ++ for (u32 i = 0; !ret && i < n; i++) { ++ struct g5p_rspare *sp = &rd->spares[i]; ++ ++ g5p_render_bind_mem(b, sp->slot, start + 2 * i, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), &s); ++ sp->ok = !g5p_ru_bind_check(&s, sp->slot, sp->root.dma, ++ G5P_RU_SPARE_TAG, target); ++ ok += sp->ok; ++ } ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 spares pairs=%u ok=%u ret=%d consumed_us=%lld dc=%u->%u first=%u/%u " ++ "last=%u/%u end=v3\n", ++ n, ok, ret, g5p_rv3_us(pub_ns, consumed_ns), start, ++ READ_ONCE(ctl[0]), rd->spares[0].slot[0], rd->spares[0].slot[1], ++ rd->spares[n - 1].slot[0], rd->spares[n - 1].slot[1]); ++ return ret; ++} ++ ++/* ++ * TEMPORARY fallback: the first job of a context on a device whose job 1 bound spare pairs. ++ * Takes the next pair that checked out, makes its catalogue page this VM's root (a copy of the ++ * current root) and publishes the VM's tag through the proven RAM-only tag path: CBASE already ++ * names the page, and the changed tag makes the firmware invalidate the slots' PC level at this ++ * context's first job. No DC3, no register. Core lock held; the context has a job queued, so ++ * no VM edit of it runs concurrently. ++ */ ++static int g5p_rv3_adopt(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_rspare *sp = NULL; ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ struct g5p_dma_page old; ++ unsigned long flags; ++ struct g5p_rt *rt; ++ u32 entry; ++ int ret; ++ ++ for (u32 i = 0; i < rd->spare_count && !sp; i++) ++ if (rd->spares[i].ok && !rd->spares[i].used) ++ sp = &rd->spares[i]; ++ if (!sp) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare: none left of %u; no hardware " ++ "touched end=v3\n", ++ r->slot[0], r->slot[1], rd->spare_count); ++ return -ENOSPC; ++ } ++ /* RAM: the pair still names its page, with the tag job 1 checked. */ ++ dma_rmb(); ++ g5p_render_bind_mem(b, sp->slot, 0, &m); ++ g5p_ru_bind_read(&m, 0, 0, &s); ++ for (u32 e = 0; e < 2; e++) { ++ if (s.pa[e] == sp->root.dma && s.host[e] == G5P_RU_SPARE_TAG && ++ s.cached[e] == G5P_RU_SPARE_TAG) ++ continue; ++ sp->ok = false; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind refused slots=%u/%u mode=spare pair=%u/%u host=%08x cached=%08x: " ++ "pair changed; no hardware touched end=v3\n", ++ r->slot[0], r->slot[1], sp->slot[0], sp->slot[1], ++ s.host[e], s.cached[e]); ++ return -EIO; ++ } ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ memcpy(sp->root.cpu, vm->root.cpu, G5P_GPU_PAGE); ++ old = vm->root; ++ vm->root = sp->root; ++ sp->root = old; /* the VM's former root, never published */ ++ g5p_ru_free_page(d->drm.dev, &sp->root); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ spin_lock_irqsave(&g->slot_lock, flags); ++ __clear_bit(r->slot[0], g->user_slots); ++ __clear_bit(r->slot[1], g->user_slots); ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ r->slot[0] = sp->slot[0]; ++ r->slot[1] = sp->slot[1]; ++ sp->used = true; ++ g5p_render_retain(r); ++ r->bound = true; ++ WRITE_ONCE(r->ctx->live, true); ++ /* Root and tables to PoC, then a tag unlike the cached one in both slots. */ ++ ret = g5p_render_invalidate(r->ctx); ++ v->bind_failed = ret != 0; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 bind slots=%u/%u ret=%d consumed_us=0 tag=%08x cached=%08x/%08x " ++ "pc=%pad mode=spare end=v3\n", ++ r->slot[0], r->slot[1], ret, ++ g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[0] * 4), ++ s.cached[0], s.cached[1], &vm->root.dma); ++ return ret; ++} ++ ++/* r74f_publish_ta() on typed state: record, WORK_CTL, producer; then (job 1 only) the r74g grace ++ * and, only if the ring is still pending, the one ALIVE-owned fallback doorbell. Jobs >= 2 ++ * publish and return: no grace, no bell (the r74f/r74g machinery is one-shot). No canary, no marker. ++ */ ++static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ struct g5p_r73b *x = b->r73b; ++ struct g5p_vm *vm = r->ctx->vm; ++ struct g5p_r74g_dc3 *dc = x->r74g_dc3; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x60); ++ u8 *p = g5p_ru_page(b, 0); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ bool first = seq == 1; ++ u32 bad; ++ int ret; ++ ++ if (!r->bound || r->submitted[0] || start > 255 || ++ READ_ONCE(ring[0]) != start || ++ g5p_le32(p + G5P_WORK_CTL) != previous || ++ g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) ++ return -EBUSY; ++ if (first) { ++ ret = g5p_r74f_admit(g, G5P_R74F_TA); ++ if (ret) ++ return ret; ++ /* g5p_r74g_ta_preflight() on typed state: DC3 still complete with the same records, ++ * tag and backing, and no page-table mutation since the attach. ++ */ ++ bad = g5p_r74g_check(g, 1, r->slot, dc[0].expected_pa, ++ dc[0].tag, dc[0].start); ++ if (g5p_vm_generation(&vm->mutation) != dc[0].tag || ++ vm->root.dma != dc[0].expected_pa || ++ r->dc_target != dc[0].target) { ++ bad |= G5P_R74G_BAD_MUTATION; ++ dc[1].bad = bad; ++ } ++ g5p_r74g_dc3_line(g, 1, false); ++ if (bad) ++ return g5p_rv3_errno(g, g5p_r74g_refuse(g, bad)); ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x61] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 0 : ++ j == 1 ? g5p_ru_meta(0) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[0] = (start + 1) & 255; ++ r->submitted[0] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_PUBLISH, start); ++ v->ta_pub_ns = ktime_get_ns(); ++ v->job_deadline_ns = ++ v->ta_pub_ns + ++ (u64)(r->hard_ms ? 2 * r->hard_ms : G5P_RU_V3_CAP_MS) * ++ NSEC_PER_MSEC; ++ smp_store_release(&ring[4], r->target[0]); ++ dma_wmb(); ++ if (first) { ++ ret = g5p_r74g_ta(g, start, r->target[0]); ++ v->grace_end = READ_ONCE(x->r74g_grace_end); ++ v->bell = ret == 0; ++ if (ret < 0) ++ return g5p_rv3_errno(g, ret); ++ } ++ /* wake110: jobs >= 2 reach this at once after the producer store (job 1: excluded). */ ++ if (!first) ++ 0; ++ g5p_rv3_wake(g, r, v, 0, start, seq); ++ /* The line the userspace lane parses (a9c_parse_publish): same format as the fixed path's. */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 0, r->slot[0], endpoint, first ? 0x10 : 0, ++ g5p_ru_meta(0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 0); ++ return 0; ++} ++ ++/* The RAM preconditions of the 3D publish, one bit per clause (0 = admitted). RAM reads only; the bits are ++ * what the refusal line prints, so a second blocker is visible in the run that meets it: ++ * 0 TA not retired 1 TA not consumed 2 3D already submitted 3 producer > 255 4 consumer != producer ++ * 5 WORK_CTL 6 WORK_CTL+0x40 7 command tag != 4 8 dependency word 9 dependency value ++ * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) ++ */ ++static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, ++ const u32 *ring, u32 start, u32 previous, u32 seq) ++{ ++ const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); ++ ++ return (u32)!r->retired[0] | (u32)!r->consumed[0] << 1 | ++ (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | ++ (u32)(READ_ONCE(ring[0]) != start) << 4 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 5 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 6 | ++ (u32)(g5p_le32(cmd) != 4) << 7 | ++ (u32)(g5p_le32(cmd + 4) != g5p_ru_meta(0) + G5P_R22_STAMP + 4) ++ << 8 | ++ (u32)(g5p_le32(cmd + 8) != seq) << 9 | ++ (u32)(g5p_le32(p + G5P_R22_STAMP) != seq - 1 || ++ g5p_le32(p + G5P_R22_STAMP + 4) != seq - 1) ++ << 10; ++} ++ ++/* The 3D after the TA retired: record, WORK_CTL, producer, and NO doorbell. */ ++static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *ring = (u32 *)g5p_boot_shared(b, 0x64); ++ u8 *p = g5p_ru_page(b, 1); ++ u32 start = READ_ONCE(ring[4]), previous = g5p_ru_endpoint(seq - 1), ++ endpoint = g5p_ru_endpoint(seq); ++ u32 why; ++ ++ /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one ++ * fails, and the refusal names the clauses and the raw values it read. ++ */ ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq); ++ if (why) { ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 3d-refused seq=%u why=%#x ring=%u/%u ctl=%08x/%08x cmd=%08x/%08x/%08x " ++ "stamp=%08x/%08x end=v3\n", ++ seq, why, start, READ_ONCE(ring[0]), ++ g5p_le32(p + G5P_WORK_CTL), ++ g5p_le32(p + G5P_WORK_CTL + 0x40), ++ g5p_le32(p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ seq))), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 4), ++ g5p_le32(p + ++ g5p_ru_cmd_at( ++ g5p_rv3_js(r->ctx->gpu->rdev, seq)) + ++ 8), ++ g5p_le32(p + G5P_R22_STAMP), ++ g5p_le32(p + G5P_R22_STAMP + 4)); ++ return -EPROTO; ++ } ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x65] + start * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + at % G5P_GPU_PAGE, ++ j == 0 ? 1 : ++ j == 1 ? g5p_ru_meta(1) : ++ j == 2 ? endpoint : ++ 0); ++ } ++ r->target[1] = (start + 1) & 255; ++ r->submitted[1] = true; ++ dma_wmb(); ++ WRITE_ONCE(*(u32 *)(p + G5P_WORK_CTL + 0x40), endpoint); ++ dma_wmb(); ++ v->td_pub_ns = ktime_get_ns(); ++ smp_store_release(&ring[4], r->target[1]); ++ dma_wmb(); ++ 0; ++ g5p_rv3_wake(g, r, v, 1, start, seq); ++ /* The 3D twin of the TA's publish line: engine 1, kick 0 (no doorbell). */ ++ g5p_mmu_record_job( ++ g, ++ "RENDER publish engine=%u slot=%u endpoint=%u kick=%x W=%08x sequence=%u " ++ "identity=%u\n", ++ 1, r->slot[1], endpoint, 0, ++ g5p_ru_meta(1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)), ++ seq, 1); ++ return 0; ++} ++ ++static const char *const g5p_rv3_ta_how[G5P_RU_TA_HOWS] = { "none", ++ "idle-clear", ++ "event", "hold" }; ++ ++/* tahang1: the TA decisions since load, by rule (core lock held). A TA that completed with an ++ * idle word still set prints one line with the running totals: rate-limited for rule (b), which ++ * is routine, always for rule (c), which means the event was not seen. RAM only. ++ */ ++static u32 g5p_rv3_ta_rules[G5P_RU_TA_HOWS]; ++static DEFINE_RATELIMIT_STATE(g5p_rv3_ta_rs, 5 * HZ, 4); ++ ++static void g5p_rv3_ta_count(struct g5p_diag *g, struct g5p_drm *d, ++ const struct g5p_rv3 *v) ++{ ++ const struct g5p_rv3_ev *s = &v->last[0]; ++ u32 how = min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1); ++ u64 hit = READ_ONCE(d->ta_hit_ns); ++ ++ g5p_rv3_ta_rules[how]++; ++ if (how == G5P_RU_TA_IDLE_CLEAR) ++ return; ++ if (how == G5P_RU_TA_EVENT && !__ratelimit(&g5p_rv3_ta_rs)) ++ return; ++ dev_warn( ++ g->dev, ++ "RENDER-V3 ta-done seq=%u how=%s idle=%08x,%08x,%08x pm80=%08x coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld busy_ms=%lld rules=%u/%u/%u end=v3\n", ++ v->seq, g5p_rv3_ta_how[how], s->idle[0], s->idle[1], s->idle[2], ++ s->pm80, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, hit), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_ta_rules[G5P_RU_TA_IDLE_CLEAR], ++ g5p_rv3_ta_rules[G5P_RU_TA_EVENT], ++ g5p_rv3_ta_rules[G5P_RU_TA_HOLD]); ++} ++ ++/* A receipt line: a failed job's at KERN_EMERG as before (it survives a freeze), a completed job's at ++ * job_log (wake every job, end/lat/retire the first 8 of a context: routine at desktop rates). ++ */ ++#define g5p_rv3_receipt_printk(g, ret, fmt, ...) \ ++ do { \ ++ if (ret) \ ++ dev_emerg((g)->dev, fmt, ##__VA_ARGS__); \ ++ else \ ++ g5p_job_printk((g)->dev, fmt, ##__VA_ARGS__); \ ++ } while (0) ++ ++/* The end of a job, whatever it was: one receipt line on the console (it survives a freeze) and the ++ * RAM-only raw state in the blob. Reads RAM only. ++ */ ++static void g5p_rv3_receipt(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, int ret) ++{ ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_r73b *x = g->boot->r73b; ++ const struct g5p_rv3_ev *t = &v->last[0], *d = &v->last[1]; ++ const struct g5p_ru_touch *k = v->touch; ++ ++ /* release: every job while the A/B is on, so both arms are complete past the verbose jobs. */ ++ if (r->touch_ms) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 touch seq=%u ret=%d arm=%s touch_ms=%u ta=%s/%lld td=%s/%lld " ++ "seen_us=%lld/%lld read_us=%lld/%lld after_us=%lld/%lld bytes=%u/%u " ++ "sum=%08x/%08x end=v3\n", ++ v->seq, ret, ++ g5p_ru_touch_arm(v->seq) ? "touch" : "control", ++ r->touch_ms, g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? (s64)(v->ta.end_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? (s64)(v->td.end_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ g5p_ru_touch_us(v->ta_pub_ns, k[0].seen_ns), ++ g5p_ru_touch_us(v->td_pub_ns, k[1].seen_ns), ++ g5p_ru_touch_us(k[0].seen_ns, k[0].done_ns), ++ g5p_ru_touch_us(k[1].seen_ns, k[1].done_ns), ++ g5p_ru_touch_after_us(&k[0], v->ta.result, ++ v->ta.end_ns), ++ g5p_ru_touch_after_us(&k[1], v->td.result, ++ v->td.end_ns), ++ k[0].bytes, k[1].bytes, k[0].sum, k[1].sum); ++ /* wake110: every job while the wake is on (the end/lat/retire lines stop at seq 8). */ ++ if (r->wake) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 wake seq=%u ret=%d mode=%u seed=%08x arm=%s/%s did=%s/%s " ++ "power=%02x/%02x fence=%08x/%08x gpc=%08x/%08x cons_at=%u/%u bell_us=%lld/%lld " ++ "lock_us=%lld/%lld ta=%s/%lld td=%s/%lld ev_ms=%lld/%lld consumed_ms=%lld/%lld " ++ "busy_ms=%lld/%lld t1_pub=%u/%u t1_end=%u/%u endpoint=%u ta_stamp=%08x/%08x " ++ "td_stamp=%08x/%08x ta_ring=%u/%u td_ring=%u/%u rang=%u end=v3\n", ++ v->seq, ret, r->wake, r->wake_seed, ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 0)], ++ g5p_wk_arm_names[g5p_wk_arm(r->wake, r->wake_seed, ++ v->seq, 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[0], ++ G5P_WK_DIDS - 1)], ++ g5p_wk_did_names[min_t(u32, v->wk_did[1], ++ G5P_WK_DIDS - 1)], ++ v->wk_power[0], v->wk_power[1], v->wk_fence[0], ++ v->wk_fence[1], v->wk_gpc[0], v->wk_gpc[1], ++ v->wk_cons[0], v->wk_cons[1], ++ g5p_rv3_wk_us(v->ta_pub_ns, v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->td_pub_ns, v->wk_done_ns[1]), ++ g5p_rv3_wk_us(v->wk_start_ns[0], v->wk_done_ns[0]), ++ g5p_rv3_wk_us(v->wk_start_ns[1], v->wk_done_ns[1]), ++ g5p_rv3_results[v->ta.result], ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.end_ns), ++ g5p_rv3_results[v->td.result], ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->wk_first_ns[0]), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->wk_first_ns[1]), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta_consumed_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td_consumed_ns), ++ g5p_rv3_wk_ms(v->ta_pub_ns, v->ta.busy_ns), ++ g5p_rv3_wk_ms(v->td_pub_ns, v->td.busy_ns), ++ v->wk_t1_pub[0], v->wk_t1_pub[1], v->wk_t1_end[0], ++ v->wk_t1_end[1], g5p_ru_endpoint(v->seq), t->stamp[0], ++ t->stamp[1], d->stamp[0], d->stamp[1], t->prod, t->cons, ++ d->prod, d->cons, r->wake_rang); ++ if (!g5p_ru_v3_verbose(r->jobs, ret)) ++ return; ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 end seq=%u ret=%d phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " ++ "cap_ms=%u ta_consumed_ms=%lld td_consumed_ms=%lld late_gap=%u/%u " ++ "late_after=%u/%u grace=%u bells=%d ta_ring=%u/%u td_ring=%u/%u " ++ "ta_stamp=%08x/%08x td_stamp=%08x/%08x pm80=%08x r0_after_ta_publish=bell-only " ++ "end=v3\n", ++ v->seq, ret, ++ g5p_rv3_phase_names[min_t(u32, v->phase, G5P_RV3_PHASES - 1)], ++ g5p_rv3_results[v->ta.result], ++ v->ta.end_ns ? ++ (s64)(v->ta.end_ns - v->ta_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ g5p_rv3_results[v->td.result], ++ v->td.end_ns ? ++ (s64)(v->td.end_ns - v->td_pub_ns) / NSEC_PER_MSEC : ++ -1, ++ r->wait_ms, G5P_RU_V3_CAP_MS, ++ v->ta_consumed_ns ? (s64)(v->ta_consumed_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td_consumed_ns ? (s64)(v->td_consumed_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_gap, v->td.late_gap, v->ta.late_after, ++ v->td.late_after, v->grace_end, atomic_read(&x->r74f_bells), ++ t->prod, t->cons, d->prod, d->cons, t->stamp[0], t->stamp[1], ++ d->stamp[0], d->stamp[1], t->pm80 ? t->pm80 : d->pm80); ++ if (r->hard_ms) ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 lat seq=%u soft_ms=%u hard_ms=%u obs_ms=%u slow=%u/%u late_soft=%u/%u " ++ "silent=%u/%u soft_seen_ms=%lld/%lld busy_ms=%lld/%lld late_after_ms=%lld/%lld " ++ "events=%u end=v3\n", ++ v->seq, r->soft_ms, r->hard_ms, r->obs_ms, v->ta.slow, ++ v->td.slow, v->ta.late_soft, v->td.late_soft, ++ v->ta.silent, v->td.silent, ++ v->ta.soft_seen_ns ? ++ (s64)(v->ta.soft_seen_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.soft_seen_ns ? ++ (s64)(v->td.soft_seen_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.busy_ns ? (s64)(v->ta.busy_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.busy_ns ? (s64)(v->td.busy_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->ta.late_ns ? (s64)(v->ta.late_ns - v->ta_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ v->td.late_ns ? (s64)(v->td.late_ns - v->td_pub_ns) / ++ NSEC_PER_MSEC : ++ -1, ++ READ_ONCE(g->boot->events)); ++ /* repeat109: the 3D's firmware retirement (coherent_ms, event_ms) apart from the host's decision ++ * (decided_ms; held_ms = how long a coherent target waited for it), the clauses still open at the ++ * deciding or closing sample (unsat=TA/3D, render-user-retire.h bits) and that sample's 3D idle words. ++ */ ++ g5p_rv3_receipt_printk( ++ g, ret, ++ "RENDER-V3 retire seq=%u ret=%d td=%s how=%s settle_ms=%u coherent_ms=%lld " ++ "event_ms=%lld decided_ms=%lld held_ms=%lld unsat=%#x/%#x " ++ "idle=%08x,%08x,%08x,%08x,%08x,%08x run=%u breaks=%u busy_ms=%lld clear_ms=%lld " ++ "ta_how=%s ta_unsat=%#x ta_coherent_ms=%lld ta_event_ms=%lld " ++ "ta_idle=%08x,%08x,%08x end=v3\n", ++ v->seq, ret, g5p_rv3_results[v->td.result], ++ g5p_rv3_how[min_t(u32, v->rt.decided, 3)], r->settle_ms, ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td_event_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns), ++ g5p_ru_rt_ms(v->rt.coherent_ns, v->td.end_ns), ++ g5p_rv3_ta_unsat(t), v->rt.unsat, d->idle[0], d->idle[1], ++ d->idle[2], d->idle[3], d->idle[4], d->idle[5], v->rt.run, ++ v->rt.breaks, g5p_ru_rt_ms(v->td_pub_ns, v->rt.busy_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.clear_ns), ++ g5p_rv3_ta_how[min_t(u32, v->tr.decided, G5P_RU_TA_HOWS - 1)], ++ v->tr.unsat, g5p_ru_rt_ms(v->ta_pub_ns, v->tr.coherent_ns), ++ g5p_ru_rt_ms(v->ta_pub_ns, v->ta_event_ns), t->idle[0], ++ t->idle[1], t->idle[2]); ++ /* The raw rows once per context (its first job) and for every failure: the blob is not a ++ * frame log. ++ */ ++ if (!v->touched || (r->jobs > 1 && !ret)) ++ return; ++ g5p_blob_hex(g, "RENDER-V3 PM-control", g5p_rv3_pm_cpu(g->boot, r), ++ 0x140); ++ g5p_blob_hex(g, "RENDER-V3 TA-W", ++ g5p_ru_page(g->boot, 0) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, v->seq)), ++ G5P_RU_TA_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 TA-CH", g5p_ru_page(g->boot, 0), 0x340); ++ /* The stamp words (+0x1840/+0x1844) live in this 0x80-byte window; no earlier blob held one. */ ++ g5p_blob_hex(g, "RENDER-V3 TA-STAMP", ++ g5p_ru_page(g->boot, 0) + G5P_R22_TQ, 0x80); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) { ++ g5p_blob_hex(g, "RENDER-V3 3D-W", ++ g5p_ru_page(g->boot, 1) + ++ g5p_ru_w_at(g5p_rv3_js(r->ctx->gpu->rdev, ++ v->seq)), ++ G5P_RU_3D_W_BYTES); ++ g5p_blob_hex(g, "RENDER-V3 3D-CH", g5p_ru_page(g->boot, 1), ++ 0x340); ++ g5p_blob_hex(g, "RENDER-V3 3D-STAMP", ++ g5p_ru_page(g->boot, 1) + G5P_R22_TQ, 0x80); ++ } ++} ++ ++/* Jobs >= 2: the legacy job>=2 PM step with policy 0 (pool, list and F rebuilt from the device ++ * pool, state S kept), whichever context the job belongs to. ++ */ ++static int g5p_rv3_pm_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks); ++ mutex_unlock(&rd->lock); ++ kvfree(physical); ++ return ret; ++} ++ ++/* render-user-loss.h for a phase that did not complete. */ ++static int g5p_rv3_loss(struct g5p_drm *d, struct g5p_render *r, bool first, ++ u32 res, const struct g5p_ru_wait *w) ++{ ++ struct g5p_boot *b = d->core->boot; ++ u32 drain_ms = (45000); ++ struct g5p_ru_loss_in in = { ++ .result = res, ++ .first = first, ++ .isolate = drain_ms && ++ g5p_ru_drain_valid(drain_ms, r->hard_ms, r->obs_ms), ++ .silent = w->silent, ++ .fault = READ_ONCE(b->failed) || READ_ONCE(b->event_failed), ++ }; ++ ++ return g5p_ru_loss(&in); ++} ++ ++/* ++ * The phase on engine `e` of this job timed out with the firmware alive: lose the job and its ++ * context, not the device. No terminal note and no device failure: the job ends -ETIMEDOUT ++ * (g5p_render_run() marks the context bad, so its client gets -EIO from now on and its pair and ++ * memory stay retained), and the channel is blocked behind the stuck phase until the scheduler's ++ * drain (g5p_rv3_drain()) clears it. RAM only. Fails (-ENOMEM) only if the record cannot be ++ * allocated; the caller then ends the epoch as before. ++ */ ++static int g5p_rv3_lose(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e) ++{ ++ struct g5p_rstuck *s = kzalloc(sizeof(*s), GFP_KERNEL); ++ const struct g5p_ru_wait *w = e ? &v->td : &v->ta; ++ ++ if (!s) ++ return -ENOMEM; ++ v->lost = true; ++ s->r = r; ++ s->v = *v; ++ s->seq = seq; ++ s->engine = e; ++ s->drain_ms = (45000); ++ s->lost_ns = ktime_get_ns(); ++ kref_get(&r->ctx->ref); ++ d->rdev->stuck = s; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u: context " ++ "lost, device kept, channel blocked end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms); ++ return 0; ++} ++ ++/* A job that stopped the firmware may be recovered from in this boot (core recover.inc, ++ * render-user-recover.h). A refusal is one line: why the epoch ends. ++ */ ++static bool g5p_rv3_recoverable(struct g5p_drm *d, bool first) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_ru_recover_in in = { ++ .first = first, ++ .retired = g5p_r8_is_retired(), ++ .closed = g5p_r8_admit_closed(&g5p_r8_state.admit), ++ .recoveries = rd->recoveries, ++ .streak = rd->recover_streak, ++ .budget = (1), ++ .streak_max = (3), ++ }; ++ ++ if (g5p_ru_recoverable(&in)) ++ return true; ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 no-recover why=%s recoveries=%u streak=%u recover=%u streak_max=%u " ++ "boot_max=%u first=%u end=v3\n", ++ g5p_ru_recover_refusal(&in), in.recoveries, in.streak, ++ in.budget, in.streak_max, G5P_RU_RECOVER_BOOT_MAX, first); ++ return false; ++} ++ ++/* The job stopped the firmware (silent stream, a fault): lose it and its context, not the ++ * device; the worker recovers the GPU before it picks anything (g5p_rv3_recover()). RAM only. ++ */ ++static void g5p_rv3_lose_recover(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rv3 *v, u32 seq, u32 e, ++ const char *why) ++{ ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_fw_recovery rec = b->fw_recovery; ++ u32 recs = READ_ONCE(b->fw_recoveries); ++ ++ v->lost = true; ++ WRITE_ONCE(d->rdev->recovering, true); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u why=%s: context lost, device " ++ "recovers end=v3\n", ++ seq, e, r->slot[0], r->slot[1], why); ++ /* The guilty job is the one waited on (one job in flight, core lock); the packet's slot ++ * names the phase the firmware stopped in (render-user-recover.h). RAM only. ++ */ ++ if (recs) ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-attrib seq=%u engine=%u packet_slot=%u match=%u code=%08x " ++ "w20=%08x w24=%08x packets=%u end=v3\n", ++ seq, e, rec.slot, rec.slot == e, rec.code, rec.w20, ++ rec.w24, recs); ++} ++ ++/* render_user_test_fault_job: this job's TA reads its control stream (W+0x98, g5p_ru_fill_ta()) ++ * from the page the context reserved in its VM and never maps, so the GPU faults in that VM. A ++ * RAM edit of the job's own W copy before it is staged; the firmware sees an ordinary job. ++ */ ++static void g5p_rv3_test_fault(struct g5p_diag *g, struct g5p_render *r, ++ struct g5p_rjob *job, u32 seq) ++{ ++ g5p_put64(job->ta_w + 0x98, r->fault_node.start); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-fault seq=%u job=%u slots=%u/%u va=%#llx: TA control stream " ++ "unmapped end=v3\n", ++ seq, r->jobs + 1, r->slot[0], r->slot[1], r->fault_node.start); ++} ++ ++/* One V3 job: [attach, the epoch's job 1 only | bind, a later context's first job,] TA, 3D. ++ * Core lock held. Returns 0, -ECANCELED (a latched stage ended it), -ETIMEDOUT (a phase ran out ++ * its deadline, or the bind's DC3 was not consumed), or a fault. ++ */ ++/* perf60 MMU arm, bits 1/3 (render_user_mmu_inval): before a bound context's job publishes ++ * anything, move its slots' host tag to a fresh PC field, so the firmware's tag compare at this ++ * job's TA and 3D kicks takes its PC-level invalidate (e138 = (S << 3) | 4) for each slot. RAM ++ * only. vm->mutation is left alone: an edit on another thread (VM_BIND of a render VM takes no ++ * core lock) recomputes its own tag and checks it only against the cached tags, as before. ++ */ ++static void g5p_rv3_mmu_retag(struct g5p_diag *g, struct g5p_render *r, u32 seq, ++ bool switched) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 host[2], cached[2], tag; ++ ++ dma_rmb(); ++ for (u32 e = 0; e < 2; e++) { ++ host[e] = g5p_le32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4); ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ } ++ tag = g5p_ru_mmu_pc_tag(host[0] ?: cached[0], cached[0], cached[1]); ++ if (tag) { ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, ++ tag); ++ dma_wmb(); ++ } ++ g5p_mmu_record( ++ g, ++ "RENDER-V3 mmu-retag seq=%u slots=%u/%u why=%s host=%08x/%08x cached=%08x/%08x " ++ "tag=%08x end=v3\n", ++ seq, r->slot[0], r->slot[1], switched ? "switch" : "every", ++ host[0], host[1], cached[0], cached[1], tag); ++} ++ ++/* perf60 A/B: the park that the never-park envelope removed, back around a context switch only. ++ * Before a bound context's job that follows another context's job, ctrl+0x1040 = 50 makes the ++ * firmware's next pass compute x < 100 (policy state 2: it parks the GPC when idle, as release7); ++ * after the window the pin goes back and the job runs in state 0 with its bells as usual. RAM ++ * only: the firmware reloads policy+0x6f4 from ctrl+0x1040 on every pass (perf-envelope.h). ++ */ ++ ++ ++ ++/* perf60: the incoming context's render targets on a switch (lat123_log), RAM. */ ++static void g5p_rv3_switch_line(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ char buf[160]; ++ struct g5p_rt *rt; ++ u32 n = 0, len = 0; ++ ++ buf[0] = 0; ++ list_for_each_entry(rt, &r->targets, link) ++ if (n++ < 4) ++ len += scnprintf(buf + len, sizeof(buf) - len, ++ " %ux%u/s%u/p%u%s", rt->width, ++ rt->height, rt->samples ?: 1, ++ rt->page_count, ++ rt->destroyed ? "/d" : ""); ++ dev_info(g->dev, ++ "RENDER-V3 switch seq=%u to=%u/%u jobs=%u rts=%u%s end=v3\n", ++ seq, r->slot[0], r->slot[1], r->jobs, n, buf); ++} ++ ++/* perf60 per-context PM state (render_user_pm_switch, default 0). The PM record's selected state ++ * S (control+0x80, 0xc0 bytes: the backing pointers a0/a8/b0 the fresh TA loads the PM context ++ * from, the lanes TA finish saves and the 3D start restores, the bc accumulator) and the five ++ * aux pages it points into (the PM context store) are built once at attach and then shared by ++ * every context; the host never rewrites them. iOS builds one PM record per work queue ++ * (6cb2378: S explicit zeros plus the four backing pointers) and keeps it per queue. ++ * 1: a context's job after another context's job starts from attach's fresh state (S rebuilt, ++ * aux zeroed); ++ * 2: each context keeps its own S + aux: saved after each of its completed jobs, restored when ++ * it follows another context's job, fresh the first time. ++ * RAM only, between jobs (the previous job retired, this one has published nothing). ++ */ ++ ++/* perf60 stage 4: (re)build context r's PM set for this epoch: grow its pool to the device's ++ * block count, zero its aux pages, build its control + S, list and F (g5p_ru_pm(), as attach ++ * builds the device PM) and clean them to PoC. rd->lock (the pool's size) held. ++ */ ++static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ const char *how) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_dma_page *page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ u32 blocks = rd->blocks; ++ u64 *physical, list_va, free_va; ++ int ret = 0; ++ ++ lockdep_assert_held(&rd->lock); ++ if (blocks > r->pm_blocks) { ++ struct g5p_dma_page *pool = ++ kvcalloc(blocks * 8, sizeof(*pool), GFP_KERNEL); ++ ++ if (!pool) ++ return -ENOMEM; ++ if (r->pm_pool) ++ memcpy(pool, r->pm_pool, ++ r->pm_blocks * 8 * sizeof(*pool)); ++ for (u32 i = r->pm_blocks * 8; !ret && i < blocks * 8; i++) ++ ret = g5p_vm_alloc_page(d->drm.dev, &pool[i]); ++ if (ret) { ++ for (u32 i = r->pm_blocks * 8; i < blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &pool[i]); ++ kvfree(pool); ++ return ret; ++ } ++ kvfree(r->pm_pool); ++ r->pm_pool = pool; ++ r->pm_blocks = blocks; ++ } ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ for (u32 i = 0; i < r->pm_blocks * 8; i++) { ++ physical[i] = r->pm_pool[i].dma; ++ memset(r->pm_pool[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm_pool[i]); ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ struct g5p_dma_page *aux = &g5p_rv3_pm_aux(rd, r)[i]; ++ ++ memset(aux->cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, aux); ++ } ++ list_va = g5p_pmset_va(r->pm_set, G5P_PMSET_LIST); ++ free_va = g5p_pmset_va(r->pm_set, G5P_PMSET_FREE); ++ ret = g5p_ru_pm(page[G5P_PMSET_CONTROL].cpu, page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, list_va, free_va, physical, ++ r->pm_blocks, G5P_RU_PRIVATE_VA); ++ kvfree(physical); ++ for (u32 k = 0; k < G5P_PMSET_PAGES; k++) ++ g5p_table_clean_page(g, &page[k]); ++ dma_wmb(); ++ if (!ret) ++ r->pm_epoch = rd->recoveries + 1; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u meta=%#x " ++ "ret=%d end=v3\n", ++ seq, r->slot[0], r->slot[1], how, r->pm_set, ++ g5p_rv3_pm_id(rd, r), r->pm_blocks, g5p_rv3_pm_meta(r), ret); ++ return ret; ++} ++ ++/* Before a context's bind (its tables are published by the bind that follows): take a free set, ++ * allocate its aux pages, point G5P_RU_PRIVATE_VA in its VM at them, and build the set. On any ++ * failure the context keeps the device PM and the shared aux pages, as before stage 4. ++ */ ++static void g5p_rv3_pmctx_assign(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ int set, ret = 0; ++ ++ if (r->pm_set >= 0) ++ return; ++ set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next); ++ for (u32 i = 0; set >= 0 && !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!r->pm_aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &r->pm_aux[i]); ++ if (set < 0 || ret) { ++ dev_warn( ++ d->core->dev, ++ "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=none set=%d ret=%d: the device PM end=v3\n", ++ seq, r->slot[0], r->slot[1], set, ret); ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "new"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ return; ++ } ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) { ++ u64 va = G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE; ++ u8 *pte = g5p_vm_pte(vm, d->drm.dev, va, false); ++ ++ u64 leaf = g5p_leaf_entry(r->pm_aux[i].dma, ++ g5p_r25_attributes(&r->v, va)); ++ ++ if (!IS_ERR_OR_NULL(pte)) ++ g5p_vm_entry(&vm->mutation, pte, leaf, 2); ++ } ++ dma_wmb(); ++ r->pm_aux_mapped = true; ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* Before each job of a context with a set: a new epoch or a grown device pool rebuilds the set; ++ * without pm_once the pool is rebuilt per job, as g5p_rv3_pm_rebuild() does the device's. ++ */ ++static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ struct g5p_dma_page *page; ++ u64 *physical; ++ int ret = 0; ++ ++ if (r->pm_set < 0) ++ return 0; ++ page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; ++ mutex_lock(&rd->lock); ++ if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { ++ ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); ++ } else if (!rd->pm_once) { ++ physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), ++ GFP_KERNEL); ++ ret = physical ? 0 : -ENOMEM; ++ for (u32 i = 0; !ret && i < r->pm_blocks * 8; i++) ++ physical[i] = r->pm_pool[i].dma; ++ if (!ret) ++ ret = g5p_ru_pm_pool( ++ page[G5P_PMSET_CONTROL].cpu, ++ page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, ++ g5p_pmset_va(r->pm_set, G5P_PMSET_LIST), ++ g5p_pmset_va(r->pm_set, G5P_PMSET_FREE), ++ physical, r->pm_blocks); ++ kvfree(physical); ++ } ++ mutex_unlock(&rd->lock); ++ return ret; ++} ++ ++/* render_user_pm_ctx 3: the context that brought the epoch up (the device PM for its attach job) ++ * takes its own set before its next job; the device aux pages its VM already maps become the ++ * set's aux (no other context maps them once every later one has its own), so no PTE changes. ++ */ ++static void g5p_rv3_pmctx_take(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_boot *b = d->core->boot; ++ int set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, rd->pm_set_next), ++ ret; ++ ++ if (set < 0) { ++ r->pm_first = false; /* stays on the device PM; not retried */ ++ return; ++ } ++ r->pm_set = set; ++ mutex_lock(&rd->lock); ++ ret = g5p_rv3_pmctx_build(d, r, seq, "take"); ++ mutex_unlock(&rd->lock); ++ if (ret) { ++ r->pm_set = -1; ++ r->pm_first = false; ++ return; ++ } ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++} ++ ++/* perf60 stage 4 diagnostics (render_user_fault_read): whose page each PM lane in S points at ++ * after the TA saved them: the device pool or aux, a set page, a context's pool or aux. ++ */ ++ ++static void g5p_rv3_pm_lanes(struct g5p_drm *d, struct g5p_render *r, u32 seq) ++{ ++ { ++ return; ++ } ++} ++ ++/* The context is gone (g5p_render_release()): its set is free again; its pages go. */ ++static void g5p_rv3_pmctx_release(struct g5p_drm *d, struct g5p_render *r) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ ++ if (r->pm_set >= 0) ++ rd->pm_set_used[r->pm_set] = false; ++ r->pm_set = -1; ++ for (u32 i = 0; r->pm_pool && i < r->pm_blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_pool[i]); ++ kvfree(r->pm_pool); ++ r->pm_pool = NULL; ++ r->pm_blocks = 0; ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm_aux[i]); ++} ++ ++#define G5P_RV3_PM_S 0x80U ++#define G5P_RV3_PM_S_LEN 0xc0U ++#define G5P_RV3_PM_SAVE (G5P_RV3_PM_S_LEN + G5P_RU_AUX_PAGES * G5P_GPU_PAGE) ++ ++/* perf60 fault address: PM state snapshots (S, PM control, the aux windows S points at, each aux ++ * page's head) at three phases of every job, in a ring and per slot pair; printed at a fault read. ++ * Only the executor writes them. RAM only. ++ */ ++enum { G5P_PMSNAP_START, G5P_PMSNAP_TA_DONE, G5P_PMSNAP_END }; ++struct g5p_pmsnap { ++ u64 ns, auxva[3]; ++ u32 seq, phase, slot, valid; ++ u8 s[G5P_RV3_PM_S_LEN], ctl[0x80], win[3][64], ++ head[G5P_RU_AUX_PAGES][64]; ++}; ++#define G5P_PMSNAP_RING 9U ++static struct g5p_pmsnap g5p_pmsnap_ring[G5P_PMSNAP_RING]; ++static struct g5p_pmsnap g5p_pmsnap_slot[16][3]; ++static u32 g5p_pmsnap_head; ++ ++static void g5p_rv3_pmsnap(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 phase) ++{ ++ { ++ return; ++ } ++} ++ ++static void g5p_rv3_pmsnap_print(struct g5p_diag *g, u32 seq, const char *set, ++ const struct g5p_pmsnap *x) ++{ ++ u64 now = ktime_get_ns(); ++ ++ if (!x->valid) ++ return; ++ dev_info( ++ g->dev, ++ "RENDER-V3 pmsnap seq=%u set=%s job=%u slot=%u phase=%s ago_us=%llu " ++ "aux_va=%#llx/%#llx/%#llx end=v3\n", ++ seq, set, x->seq, x->slot, ++ x->phase == G5P_PMSNAP_START ? "start" : ++ x->phase == G5P_PMSNAP_TA_DONE ? "ta-done" : ++ "end", ++ (now - x->ns) / NSEC_PER_USEC, x->auxva[0], x->auxva[1], ++ x->auxva[2]); ++ for (u32 o = 0; o < sizeof(x->s); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap S+%02x %*ph\n", o, 64, ++ x->s + o); ++ for (u32 o = 0; o < sizeof(x->ctl); o += 64) ++ dev_info(g->dev, "RENDER-V3 pmsnap C+%02x %*ph\n", o, 64, ++ x->ctl + o); ++ for (u32 k = 0; k < 3; k++) ++ dev_info(g->dev, "RENDER-V3 pmsnap W%u %*ph\n", k, 64, ++ x->win[k]); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ dev_info(g->dev, "RENDER-V3 pmsnap A%u %*ph\n", i, 64, ++ x->head[i]); ++} ++ ++static void g5p_rv3_pmsnap_dump(struct g5p_diag *g, struct g5p_render *r, ++ u32 seq) ++{ ++ for (u32 p = 0; p < 3; p++) ++ g5p_rv3_pmsnap_print(g, seq, "slot-last", ++ &g5p_pmsnap_slot[r->slot[0] & 15][p]); ++ for (u32 i = G5P_PMSNAP_RING; i; i--) ++ g5p_rv3_pmsnap_print(g, seq, "ring", ++ &g5p_pmsnap_ring[(g5p_pmsnap_head - i) % ++ G5P_PMSNAP_RING]); ++} ++ ++/* perf60 fault address: the VM edit log entries that cover the fault VA (any VM, newest first), ++ * then the faulting job's VM's latest edits. Copied under the lock, printed outside it. ++ */ ++#define G5P_VMLOG_HITS 24U ++#define G5P_VMLOG_TAIL 16U ++static void g5p_vmlog_dump(struct g5p_diag *g, u32 seq, const void *vm, u64 va, ++ bool have_va) ++{ ++ struct g5p_vmlog_ent *out = kmalloc_array( ++ G5P_VMLOG_HITS + G5P_VMLOG_TAIL, sizeof(*out), GFP_KERNEL); ++ u32 hits = 0, tail = 0, n, head; ++ unsigned long flags; ++ u64 now = ktime_get_ns(); ++ ++ if (!out) ++ return; ++ spin_lock_irqsave(&g5p_vmlog.lock, flags); ++ head = g5p_vmlog.head; ++ n = min(head, G5P_VMLOG_N); ++ for (u32 i = 1; i <= n; i++) { ++ const struct g5p_vmlog_ent *x = ++ &g5p_vmlog.e[(head - i) % G5P_VMLOG_N]; ++ ++ if (have_va && hits < G5P_VMLOG_HITS && ++ g5p_ru_mmu_fault_in(va, x->va, x->size)) ++ out[hits++] = *x; ++ else if (x->vm == vm && tail < G5P_VMLOG_TAIL) ++ out[G5P_VMLOG_HITS + tail++] = *x; ++ } ++ spin_unlock_irqrestore(&g5p_vmlog.lock, flags); ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u va=%#llx edits=%u covering=%u vm_tail=%u end=v3\n", ++ seq, have_va ? va : 0, head, hits, tail); ++ for (u32 i = 0; i < hits + tail; i++) { ++ const struct g5p_vmlog_ent *x = ++ &out[i < hits ? i : G5P_VMLOG_HITS + i - hits]; ++ ++ dev_info( ++ g->dev, ++ "RENDER-V3 vmlog seq=%u %s ago_ms=%llu op=%s pid=%d comm=%s vm=%s/%u va=%#llx " ++ "size=%#llx handle=%#x off=%#llx end=v3\n", ++ seq, i < hits ? "cover" : "tail", ++ (now - x->ns) / NSEC_PER_MSEC, ++ g5p_vmlog_ops[min_t(u32, x->op, ++ ARRAY_SIZE(g5p_vmlog_ops) - 1)], ++ x->tgid, x->comm, x->vm == vm ? "this" : "other", ++ x->vm_id, x->va, x->size, x->handle, x->offset); ++ } ++ kfree(out); ++} ++ ++/* perf60 fault address (render_user_fault_read): the MMU fault banks, read once at a failed ++ * phase, the way iOS's fault handler (6ca5c14) reads them; the firmware keeps no copy. Bell mode ++ * only. Under the event lock with the wake path's power proof (the fault already latched ++ * b->failed, so that flag is not a guard here), then the GPU register licence (mmio.inc, ++ * mmio-access.md §4): no wake (Apple's reads never write R1+0x14, G6), a refusal retried outside ++ * the locks every 20-40 us for at most G5P_RV3_DC_SETTLE_US, and the four bank reads only under a ++ * held ticket. A parked GPC records skip-parked, as Apple's report has no bif fault entry then. ++ * At most G5P_RV3_FAULT_READS per boot. It runs before the loss decision and the terminal ++ * latch, while the faulted job is still the GPU's current work. ++ */ ++#define G5P_RV3_FAULT_READS G5P_RU_RECOVER_BOOT_MAX ++static atomic_t g5p_rv3_fault_reads = ATOMIC_INIT(0); ++ ++/* faultrec: a fault VA in the PM-set window (render-user-pmset.h) names a set and its part; say ++ * which context owns that set now and how it stands against the two job records and the deferred ++ * closes (render_user_close_defer). RAM only; rdev lock for the context list. ++ */ ++static void g5p_rv3_fault_pmset(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq, u32 bank, ++ const struct g5p_ru_mmu_fault *f) ++{ ++ static const char *const parts[G5P_PMSET_PAGES] = { "control", "list", ++ "free" }; ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_render *o, *owner = NULL; ++ u32 page, set, own[2] = { U32_MAX, U32_MAX }, df[2][2]; ++ bool last = false, prev = false, closing = false, bound = false; ++ ++ if (f->va < G5P_PMSET_VA || f->va >= G5P_PMSET_END) ++ return; ++ page = (f->va - G5P_PMSET_VA) / G5P_GPU_PAGE; ++ set = page / G5P_PMSET_PAGES; ++ mutex_lock(&rd->lock); ++ list_for_each_entry(o, &rd->contexts, rdev_link) ++ if (o->pm_set == (int)set) { ++ owner = o; ++ break; ++ } ++ if (owner) { ++ own[0] = owner->slot[0]; ++ own[1] = owner->slot[1]; ++ last = rd->pm_last == owner; ++ prev = rd->pm_prev == owner; ++ closing = owner->closing; ++ bound = owner->bound; ++ } ++ for (u32 k = 0; k < 2; k++) { ++ o = rd->close_deferred[k]; ++ df[k][0] = o ? o->slot[0] : U32_MAX; ++ df[k][1] = o ? o->slot[1] : U32_MAX; ++ } ++ mutex_unlock(&rd->lock); ++ dev_emerg( ++ d->core->dev, ++ "RENDER-V3 fault-pmset seq=%u bank=%u mmu_context=%u set=%u part=%s owner=%d/%d " ++ "bound=%u closing=%u last=%u prev=%u job=%u/%u deferred=%d/%d,%d/%d job_slots=%u " ++ "end=v3\n", ++ seq, bank, f->context, set, parts[page % G5P_PMSET_PAGES], ++ (int)own[0], (int)own[1], bound, closing, last, prev, ++ r->slot[0], r->slot[1], (int)df[0][0], (int)df[0][1], ++ (int)df[1][0], (int)df[1][1], rd->job_slots); ++} ++ ++static void g5p_rv3_fault_read(struct g5p_drm *d, struct g5p_render *r, u32 seq, ++ u32 e, int res, bool silent) ++{ ++ u32 full = (0), w[4] = { 0 }; ++ u32 mode = full | ((1) ? 1 : 0); ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_ru_mmu_fault f[2]; ++ struct g5p_mmio_ticket t = { .r1_0 = U32_MAX, ++ .gpc = U32_MAX, ++ .r1_0_end = U32_MAX, ++ .gpc_end = U32_MAX }; ++ const char *did = "skip-guard"; ++ u32 power = 0, tries = 0; ++ unsigned long flags; ++ bool fault = res == G5P_RU_WAIT_FAULT && !silent, ticket = false; ++ u64 t0; ++ int ret; ++ ++ if ((2) != G5P_WK_ALWAYS || !(mode & (fault ? 1 : 2))) ++ return; ++ if (atomic_inc_return(&g5p_rv3_fault_reads) > G5P_RV3_FAULT_READS) { ++ did = "skip-cap"; ++ goto print; ++ } ++ t0 = ktime_get_mono_fast_ns(); ++ for (;;) { ++ spin_lock_irqsave(&b->event_lock, flags); ++ power = g5p_wk_power_mask( ++ READ_ONCE(g->powered), READ_ONCE(g->fender_ready), ++ READ_ONCE(b->released), READ_ONCE(b->alive), ++ READ_ONCE(b->control_complete), g5p_rv3_wk_ds108(b), ++ READ_ONCE(g->reset_transition)); ++ if (READ_ONCE(b->stopped) || !g5p_wk_power_ready(power)) { ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ break; ++ } ++ tries++; ++ ticket = true; ++ ret = g5p_mmio_hold(g, &t, 0); ++ if (!ret) { ++ w[0] = g5p_mmio_r0_read(g, &t, 0xe150); ++ w[1] = g5p_mmio_r0_read(g, &t, 0xe154); ++ w[2] = g5p_mmio_r0_read(g, &t, 0xe158); ++ w[3] = g5p_mmio_r0_read(g, &t, 0xe15c); ++ g5p_mmio_release(g, &t); ++ } ++ spin_unlock_irqrestore(&b->event_lock, flags); ++ did = g5p_mmio_did(&t); ++ if (ret != -EAGAIN || ++ ktime_get_mono_fast_ns() - t0 >= ++ (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) ++ break; ++ usleep_range(20, 40); ++ } ++ if (ticket) ++ g5p_mmio_sample_line(g, G5P_MMS_FAULT, &t, w, t.ops ? 4 : 0, ++ true); ++print: ++ f[0] = g5p_ru_mmu_fault_decode(w[0], w[1]); ++ f[1] = g5p_ru_mmu_fault_decode(w[2], w[3]); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-read seq=%u engine=%u slots=%u/%u why=%s did=%s power=%02x " ++ "fence=none r1_0=%08x gpc=%08x tries=%u e150=%08x:%08x e158=%08x:%08x end=v3\n", ++ seq, e, r->slot[0], r->slot[1], ++ fault ? "fault" : ++ silent ? "silent" : ++ "timeout", ++ did, power, t.r1_0, t.gpc, tries, w[1], w[0], w[3], w[2]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 fault-decode seq=%u bank=%u va=%#llx %s type=%s level=%u " ++ "requester=%u/%s sideband=%#x context=%u job_slots=%u/%u end=v3\n", ++ seq, k, f[k].va, f[k].read ? "read" : "write", ++ g5p_ru_mmu_fault_types[f[k].type], f[k].level, ++ f[k].requester, ++ g5p_ru_mmu_requester(f[k].requester), ++ f[k].sideband, f[k].context, r->slot[0], ++ r->slot[1]); ++ for (u32 k = 0; k < 2; k++) ++ if (f[k].valid) ++ g5p_rv3_fault_pmset(d, r, seq, k, &f[k]); ++ g5p_rv3_switch_line(g, r, seq); ++ if (!full) ++ return; ++ g5p_vmlog_dump(g, seq, r->ctx->vm, f[0].valid ? f[0].va : f[1].va, ++ f[0].valid || f[1].valid); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ g5p_rv3_pmsnap_dump(g, r, seq); ++} ++ ++/* GPU overclock thermal stop (core boot.inc, boot_gpu_oc_mhz): every 500 ms while the epoch runs ++ * state 5, the three SoC zones; at or above boot_gpu_oc_temp_mc (or unreadable) the overclock is off ++ * for the rest of the boot and the render scheduler recovers the GPU (g5p_rv3_recover(): gate cycle, ++ * firmware boot) between jobs, so the new epoch powers on at state 4 with record 5 cleared: iOS's ++ * park -> reboot -> new pstate, never a pstate change under a live firmware. Every 10 s one status ++ * line: the temperatures, the real Fender state (an R1-only licence ticket) and the jobs run. ++ */ ++#define G5P_OC_POLL_MS 500 ++static void g5p_rsched_kick(struct g5p_drm *d); ++ ++static void g5p_oc_work(struct work_struct *work) ++{ ++ struct g5p_rdev *rd = ++ container_of(to_delayed_work(work), struct g5p_rdev, oc_work); ++ struct g5p_drm *d = rd->drm; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = READ_ONCE(g->boot); ++ u32 lim = (90000), zones; ++ const char *zone; ++ int t; ++ ++ if (!b || !0 || READ_ONCE(g5p_oc_tripped)) { ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ zones = g5p_oc_temp(&t, &zone); ++ if (zones != ARRAY_SIZE(g5p_oc_zone_names) || t >= (int)lim) { ++ WRITE_ONCE(g5p_oc_tripped, true); ++ dev_emerg( ++ g->dev, ++ "GPUPERF oc thermal-stop temp=%d zone=%s zones=%u limit=%u mhz=%u jobs=%u: the " ++ "GPU recovers at state 4 (723 MHz) end=gpuperf\n", ++ t, zone, zones, lim, b->gpu_oc_mhz, rd->oc_jobs); ++ WRITE_ONCE(rd->recovering, true); ++ g5p_rsched_kick(d); ++ clear_bit(0, &rd->oc_armed); ++ return; ++ } ++ if (!(++rd->oc_ticks % (10000 / G5P_OC_POLL_MS))) { ++ struct g5p_mmio_ticket tk; ++ ++ if (!g5p_mmio_hold(g, &tk, G5P_MMIO_CENSUS)) ++ g5p_mmio_release(g, &tk); ++ g5p_job_printk( ++ g->dev, ++ "GPUPERF oc status mhz=%u temp=%d zone=%s limit=%u r1=%s r1_1000=%08x jobs=%u " ++ "end=gpuperf\n", ++ b->gpu_oc_mhz, t, zone, lim, g5p_mmio_did(&tk), tk.gpc, ++ rd->oc_jobs); ++ } ++ queue_delayed_work(system_wq, &rd->oc_work, ++ msecs_to_jiffies(G5P_OC_POLL_MS)); ++} ++ ++ ++ ++static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ bool first = !rd->up, bound_before = r->bound; ++ int err = -EIO, ret, why, loss; ++ u32 seq, res; ++ ++ if (job->compute) ++ return g5p_cu_execute(d, r, job); ++ if (r->stage_done) ++ return -ECANCELED; ++ g5p_l123_job_reset(g); ++ /* The submit-time admission again, now under the core lock and with the bind state: still nothing touched. */ ++ why = g5p_ru_v3_admit(&st, true); ++ if (why) ++ return g5p_rv3_refusal_errno(why); ++ seq = rd->sequence + 1; ++ ; ++ 0; ++ if (bound_before && d->bell_prev_ctx && d->bell_prev_ctx != r->ctx) { ++ u32 ms = (0); ++ ++ ; ++ ; ++ if (ms && b->gpu_perf) ++ 0; ++ } ++ 0; ++ if (g5p_ru_v3_verbose(r->jobs + 1, 0)) ++ g5p_job_printk( ++ g->dev, ++ "RENDER-V3 start seq=%u bound_ms=%u cap_ms=%u slots=%u/%u attach=%s " ++ "ta_doorbell=%s 3d_doorbell=none wait=ram-only after_timeout=quiet end=v3\n", ++ seq, r->wait_ms, G5P_RU_V3_CAP_MS, r->slot[1], ++ r->slot[0], ++ first ? "firmware-dc3" : ++ r->bound ? "kept" : ++ rd->spare_count ? "spare" : ++ "ram-dc3", ++ first ? "grace-owned" : "none"); ++ if (first) { ++ v->phase = G5P_RV3_ATTACH; ++ ret = g5p_rv3_spares(d, v); ++ if (!ret) ++ ret = g5p_rv3_attach(d, r, v); ++ /* perf60 stage 4: a context with a set that brings a new epoch up */ ++ if (!ret) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ if (ret) ++ goto out; ++ ; ++ } else { ++ if (!r->bound) { ++ /* perf60 stage 4: its own PM set and aux pages before the bind ++ * publishes its tables ++ */ ++ if (rd->pm_ctx) ++ g5p_rv3_pmctx_assign(d, r, seq); ++ /* RAM only; a failure is this context's alone (v->bind_failed). */ ++ v->phase = G5P_RV3_BIND; ++ ret = rd->spare_count ? g5p_rv3_adopt(d, r, v) : ++ g5p_rv3_bind(d, r, v); ++ if (ret) ++ goto out; ++ ; ++ } ++ /* The firmware owns the rings and the channels from here on: any failure quarantines. */ ++ v->touched = true; ++ if (rd->pm_ctx == 3 && r->pm_first && r->pm_set < 0) ++ g5p_rv3_pmctx_take(d, r, seq); ++ if (r->pm_set >= 0) ++ ret = g5p_rv3_pmctx_job(d, r, seq); ++ else ++ ret = rd->pm_once ? 0 : g5p_rv3_pm_rebuild(d); ++ if (ret) ++ goto out; ++ ; ++ ; ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_START); ++ if (g5p_ru_mmu_retag_due((0), bound_before, ++ d->bell_prev_ctx != r->ctx)) ++ g5p_rv3_mmu_retag(g, r, seq, ++ d->bell_prev_ctx != r->ctx); ++ } ++ rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu(b, r) + 0x80); ++ if (g5p_ru_fault_due(r->fault_job, r->jobs, first)) ++ g5p_rv3_test_fault(g, r, job, seq); ++ for (u32 e = 0; e < 2; e++) { ++ g5p_ru_job_at( ++ g5p_ru_page(b, e), g5p_ru_meta(e), ++ g5p_meta_address(g5p_object_va(&b->layout, 0x51 + e)), ++ g5p_rv3_pm_meta(r), r->slot[e], e, seq, ++ e ? job->td_w : job->ta_w, ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r)); ++ r->submitted[e] = false; ++ r->consumed[e] = false; ++ r->retired[e] = false; ++ } ++ g5p_l123_job_prepare(d, job); ++ g5p_render_clean_job(g); ++ rd->sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_rv3_wake_mark(d, r, v, 0); ++ ret = g5p_rv3_publish_ta(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TA_WAIT; ++ if (r->stall_job && r->jobs == r->stall_job && !first) { ++ /* render_user_test_stall_job: the host gives up, the firmware runs on */ ++ g5p_ru_wait_start(&v->ta, v->ta_pub_ns, 1, v->job_deadline_ns); ++ dev_warn( ++ g->dev, ++ "RENDER-V3 test-stall seq=%u job=%u slots=%u/%u: TA bound 1 ms end=v3\n", ++ seq, r->jobs, r->slot[0], r->slot[1]); ++ } else { ++ g5p_rv3_wait_start(g, r, &v->ta, v->ta_pub_ns, ++ v->job_deadline_ns); ++ } ++ g5p_ru_touch_start(&v->touch[0], v->ta_pub_ns, r->touch_ms, seq); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, seq, &v->ta, &v->touch[0], NULL, &v->tr, ++ &v->last[0], &v->ta_consumed_ns, &err); ++ r->consumed[0] = v->ta_consumed_ns != 0; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ /* Timeout or fault: (latency101: the RAM-only observation window first,) terminal ++ * latch, one RAM sample for the record (a completion that arrives now is evidence, ++ * never the result), no export, no register access. ++ */ ++ g5p_rv3_fault_read(d, r, seq, 0, res, v->ta.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->ta); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 0)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 0, ++ v->ta.silent ? "ta-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "ta-timeout" : ++ "ta-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "ta-timeout" : "ta-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 0, seq, &v->ta, NULL, &v->tr, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_TIMEOUT, ret); ++ goto out; ++ } ++ g5p_rv3_ta_count(g, d, v); ++ r->retired[0] = true; ++ g5p_rv3_terminal(g, G5P_RU_TERM_COMPLETE, "ta-complete"); ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_TA_DONE); ++ g5p_rv3_pm_lanes(d, r, seq); ++ if (first) ++ g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, ++ g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); ++ ; ++ v->phase = G5P_RV3_TD_PUBLISH; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample( ++ g, r, 1, seq, ++ &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ ++ goto out; ++ } ++ v->phase = G5P_RV3_TD_WAIT; ++ g5p_rv3_wait_start(g, r, &v->td, v->td_pub_ns, v->job_deadline_ns); ++ g5p_ru_touch_start(&v->touch[1], v->td_pub_ns, r->touch_ms, seq); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, ++ &v->last[1], &v->td_consumed_ns, &err); ++ r->consumed[1] = v->td_consumed_ns != 0; ++ /* mmio_selftest bit 2 (rung 5, DFU-risk on a park tuple): one licensed ticket at the 3D ++ * completion decision; always: the idle clock of bit 3. ++ */ ++ ; ++ if (res != G5P_RU_WAIT_COMPLETE) { ++ struct g5p_rv3_ev late; ++ ++ g5p_rv3_fault_read(d, r, seq, 1, res, v->td.silent); ++ if (res == G5P_RU_WAIT_TIMEOUT && r->obs_ms) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ loss = g5p_rv3_loss(d, r, first, res, &v->td); ++ if (loss == G5P_RU_LOSS_CONTEXT && ++ !g5p_rv3_lose(d, r, v, seq, 1)) { ++ ret = -ETIMEDOUT; ++ goto out; ++ } ++ if (loss == G5P_RU_LOSS_DEVICE && ++ g5p_rv3_recoverable(d, first)) { ++ g5p_rv3_lose_recover(d, r, v, seq, 1, ++ v->td.silent ? "3d-silent" : ++ res == G5P_RU_WAIT_TIMEOUT ? ++ "3d-timeout" : ++ "3d-fault"); ++ ret = g5p_ru_lost_errno(res, err); ++ goto out; ++ } ++ g5p_rv3_terminal( ++ g, ++ res == G5P_RU_WAIT_TIMEOUT ? G5P_RU_TERM_TIMEOUT : ++ G5P_RU_TERM_FAULT, ++ res == G5P_RU_WAIT_TIMEOUT ? "3d-timeout" : "3d-fault"); ++ if (!(res == G5P_RU_WAIT_TIMEOUT && r->obs_ms)) ++ g5p_rv3_observe(g, r, 1, seq, &v->td, &v->rt, NULL, ++ &late); ++ ret = res == G5P_RU_WAIT_TIMEOUT ? -ETIMEDOUT : err; ++ goto out; ++ } ++ /* Rule (c) decided: the matching event was not seen. Always one line (RAM only). */ ++ if (v->rt.decided == G5P_RU_RT_HOLD) ++ dev_warn( ++ g->dev, ++ "RENDER-V3 3d-hold seq=%u idle=%08x,%08x,%08x,%08x,%08x,%08x coherent_ms=%lld " ++ "decided_ms=%lld end=v3\n", ++ seq, v->last[1].idle[0], v->last[1].idle[1], ++ v->last[1].idle[2], v->last[1].idle[3], ++ v->last[1].idle[4], v->last[1].idle[5], ++ g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), ++ g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns)); ++ r->retired[1] = true; ++ v->phase = G5P_RV3_DONE; ++ rd->recover_streak = ++ 0; /* a job finished: a later fault starts a new streak */ ++ ret = 0; ++out: ++ if (v->phase >= G5P_RV3_TA_PUBLISH) ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ if (v->phase >= G5P_RV3_TD_PUBLISH) ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ g5p_rv3_wake_end(d, r, v); ++ /* Any other failure after the hardware was touched (attach, publish, drain, protocol): the same ++ * criterion g5p_render_run() quarantines the context by. A stage stop (-ECANCELED) is not one. ++ */ ++ if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && ++ !g5p_ru_term_failed(&g5p_rv3_term)) ++ g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), ++ g5p_rv3_phase_names[min_t( ++ u32, v->phase, G5P_RV3_PHASES - 1)]); ++ g5p_rv3_receipt(g, r, job, ret); ++ g5p_l123_job_end(d, job, seq, ret); ++ if (!ret) { ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_END); ++ ; ++ ; ++ } ++ /* render-user-pmset.h: the PM's write-back targets */ ++ if (g5p_pmset_switched(rd->pm_last, r)) { ++ rd->pm_prev = rd->pm_last; ++ rd->pm_last = r; ++ } ++ /* perf60 bell gates: the next job's gap and context switch are measured from here. */ ++ d->bell_prev_end_ns = ktime_get_ns(); ++ d->bell_prev_ctx = r->ctx; ++ return ret; ++} ++ ++/* ++ * The channel is blocked behind a lost job (g5p_rv3_lose()): wait for its stuck phase, RAM only, ++ * with the firmware's event stream watched from the first sample (silent for 1 s = dead). A late ++ * TA gets the job's 3D published (its W is staged since the job's start; no doorbell for a job ++ * other than the epoch's first), so the 3D channel's sequence and the PM state stay as the ++ * firmware expects; a late 3D clears the channel. Then the held jobs run. A silent stream, a ++ * fault, a refused 3D or the drain bound end the epoch as a timeout or fault always did, and the ++ * held jobs then fail at admission. Called by the scheduler's worker before any pick; returns ++ * false, having done nothing, when the core lock is busy. ++ */ ++static bool g5p_rv3_drain(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rstuck *s = rd->stuck; ++ struct g5p_render *r = s->r; ++ struct g5p_rv3 *v = &s->v; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline = jiffies + 2 * HZ; ++ u32 st = s->engine ? G5P_RU_DRAIN_TD : G5P_RU_DRAIN_TA, ++ res = G5P_RU_WAIT_PENDING; ++ struct g5p_ru_wait w = {}; ++ bool armed = false; ++ int act, err = -EIO; ++ ++ /* TYPED_COMPUTE: a lost CDM job has its own one-phase drain. */ ++ if (s->engine == G5P_CU_ENGINE) ++ return g5p_cu_drain(d); ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ WRITE_ONCE(rd->executing, true); ++ for (;;) { ++ u32 e = st == G5P_RU_DRAIN_TA ? 0 : 1; ++ u64 now = ktime_get_ns(); ++ ++ g5p_ru_wait_start_lat(&w, now, 0, s->drain_ms, ++ now + (u64)s->drain_ms * NSEC_PER_MSEC, ++ READ_ONCE(b->events)); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->stopped)) { ++ res = G5P_RU_WAIT_FAULT; ++ } else if (!e) { ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_ru_ta_start(&v->tr); ++ res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], ++ NULL, &v->tr, &v->last[0], ++ &v->ta_consumed_ns, &err); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ } else { ++ if (!armed) ++ g5p_rv3_event_arm(d, 1, ++ g5p_ru_endpoint(s->seq)); ++ g5p_ru_rt_start(&v->rt, r->settle_ms); ++ res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], ++ &v->rt, NULL, &v->last[1], ++ &v->td_consumed_ns, &err); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ } ++ act = g5p_ru_drain_step(&st, res); ++ if (act != G5P_RU_DRAIN_PUBLISH_TD) ++ break; ++ r->retired[0] = true; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); ++ armed = true; ++ if (g5p_rv3_publish_3d(g, r, v, s->seq)) { ++ g5p_rv3_event_disarm(d, 1); ++ res = G5P_RU_WAIT_FAULT; ++ act = g5p_ru_drain_step(&st, res); ++ break; ++ } ++ } ++ if (act == G5P_RU_DRAIN_UNBLOCK) { ++ r->retired[1] = true; ++ r->lost_drained = true; ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drained seq=%u engine=%u slots=%u/%u lost_ms=%lld: channel clear, " ++ "held jobs resume end=v3\n", ++ s->seq, s->engine, r->slot[0], r->slot[1], ++ g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ } else { ++ bool fault = res == G5P_RU_WAIT_FAULT; ++ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drain dead seq=%u engine=%u phase=%s why=%s err=%d lost_ms=%lld: the " ++ "device recovers if it may, else the epoch ends end=v3\n", ++ s->seq, s->engine, ++ st == G5P_RU_DRAIN_DEAD && !v->td_pub_ns ? "ta" : "3d", ++ w.silent ? "silent" : ++ fault ? "fault" : ++ "timeout", ++ err, g5p_rv3_us(s->lost_ns, ktime_get_ns()) / 1000); ++ if (g5p_rv3_recoverable(d, false)) ++ WRITE_ONCE(rd->recovering, true); ++ else ++ g5p_rv3_terminal( ++ g, ++ fault ? G5P_RU_TERM_FAULT : G5P_RU_TERM_TIMEOUT, ++ fault ? "drain-fault" : "drain-timeout"); ++ } ++ /* after the ruling: `recovering` or the terminal latch is visible first (g5p_rdev_ruling()) */ ++ smp_store_release(&rd->executing, false); ++ rd->stuck = NULL; ++ mutex_unlock(&g->lock); ++ g5p_context_put(r->ctx); ++ kfree(s); ++ return true; ++} ++ ++/* The firmware window's leaves again from the new epoch's tables, in the same pages, so every ++ * render VM's root entry stays valid (g5p_render_fw_entry()). Core lock held. ++ */ ++static void g5p_render_fw_rebuild(struct g5p_drm *d) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_boot *b = d->core->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ ++ if (!w->built) ++ return; ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ w->pages = 0; ++ w->missing = 0; ++ for (u32 i = 0; i < G5P_RU_FW_PAGES; i++) { ++ u64 va = g5p_ru_fw_window_va[i]; ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (k == w->leaves) ++ continue; ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, ++ leaf & 1 ? leaf : 0); ++ w->pages += leaf & 1; ++ w->missing += !(leaf & 1); ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(d->core, &w->leaf[i]); ++} ++ ++/* ++ * A lost job stopped the firmware (g5p_rv3_lose_recover(), or a drain that died): cycle the GPU ++ * and boot the firmware again (core recover.inc), then start the render device over as at boot. ++ * Every context survives except the lost one: its bindings belonged to the old epoch, which the ++ * cycle ended, so each is unbound in RAM, its binding's reference dropped, and its next job binds ++ * again (the first job after the recovery runs the epoch's attach). The aux pages and the PM ++ * pool are zeroed as at their allocation. The held jobs run after this. Returns false, having ++ * done nothing, when the core lock is busy. A failed recovery leaves the device failed. ++ */ ++static bool g5p_rv3_recover(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ unsigned long deadline = jiffies + 2 * HZ, flags; ++ struct g5p_render **drop = NULL; ++ struct g5p_render *r; ++ u32 n = 0, count = 0, dropped = 0, stage; ++ int ret; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ count++; ++ } ++ mutex_unlock(&rd->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ rd->recoveries++; ++ rd->recover_streak++; ++ ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; ++ if (ret) { ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", ++ ret, rd->recoveries); ++ return true; ++ } ++ g5p_ru_term_reset(&g5p_rv3_term); ++ /* the new firmware's PM has written nothing back yet: no context is its last or previous */ ++ rd->pm_last = NULL; ++ rd->pm_prev = NULL; ++ rd->up = false; ++ rd->v3 = false; ++ rd->sequence = 0; ++ rd->pm_serial = 0; ++ rd->dc_pending = false; ++ rd->dc_wait = 0; ++ if (rd->spares) { ++ spin_lock_irqsave(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ if (!rd->spares[i].used) { ++ __clear_bit(rd->spares[i].slot[0], ++ g->user_slots); ++ __clear_bit(rd->spares[i].slot[1], ++ g->user_slots); ++ } ++ spin_unlock_irqrestore(&g->slot_lock, flags); ++ for (u32 i = 0; i < rd->spare_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rd->spares[i].root); ++ kfree(rd->spares); ++ rd->spares = NULL; ++ rd->spare_count = 0; ++ } ++ g5p_render_fw_rebuild(d); ++ mutex_lock(&rd->lock); ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ if (rd->aux[i].cpu) { ++ memset(rd->aux[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->aux[i]); ++ } ++ for (u32 i = 0; i < rd->blocks * 8; i++) { ++ memset(rd->physical[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &rd->physical[i]); ++ } ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ r->jobs = 0; ++ r->dc_target = 0; ++ WRITE_ONCE(r->live_maps, false); ++ memset(r->submitted, 0, sizeof(r->submitted)); ++ memset(r->consumed, 0, sizeof(r->consumed)); ++ memset(r->retired, 0, sizeof(r->retired)); ++ /* nothing of the old epoch holds this context's memory any more */ ++ r->lost_drained = true; ++ WRITE_ONCE(r->ctx->live, false); ++ if (r->retained && n < count) { ++ r->retained = false; ++ drop[n++] = r; ++ } ++ } ++ mutex_unlock(&rd->lock); ++ spin_lock_irqsave(&d->status_lock, flags); ++ d->last_error = 0; ++ d->job_error = 0; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ for (u32 i = 0; i < n; i++) { ++ struct g5p_context *c = drop[i]->ctx; ++ ++ dropped++; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recovered n=%u contexts=%u unbound=%u streak=%u: held jobs resume, " ++ "the next job attaches end=v3\n", ++ rd->recoveries, count, dropped, rd->recover_streak); ++ return true; ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +new file mode 100644 +index 000000000..bd86d7dff +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -0,0 +1,2549 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Userspace TA/3D render: render contexts, render targets and SUBMIT_RENDER. ++ * Builds on the r22 constructors through render-user-layout.h (byte-equal for ++ * the round16 inputs). Default off. The first accepted job claims the device ++ * for render; TYPED_V3 contexts then share it, each with its own VM and slot ++ * pair (docs/gpu/multi-client-design.md), the other profiles stay with that ++ * first context. Jobs of all contexts are serialized; no reset/recovery: any ++ * hardware failure retains everything. ++ */ ++#include "../../apple-g5p/render-user-layout.h" ++#include "../../apple-g5p/render-user-wait.h" ++#include "../../apple-g5p/render-user-touch.h" ++#include "../../apple-g5p/render-user-retire.h" ++#include "../../apple-g5p/render-user-wake.h" ++#include "../../apple-g5p/render-user-serial.h" ++#include "../../apple-g5p/render-user-terminal.h" ++#include "../../apple-g5p/render-user-bind.h" ++#include "../../apple-g5p/render-user-close.h" ++#include "../../apple-g5p/render-user-sched.h" ++#include "../../apple-g5p/render-user-window.h" ++#include "../../apple-g5p/render-user-msaa.h" ++#include "../../apple-g5p/render-user-loss.h" ++#include "../../apple-g5p/render-user-fault.h" ++#include "../../apple-g5p/render-user-mmu.h" ++#include "../../apple-g5p/render-user-faultreg.h" ++#include "render-user-validate.h" ++#include "drm-render-v3.h" ++#include "../../apple-g5p/compute-user-layout.h" ++#include "compute-user-validate.h" ++#include "drm-compute-v1.h" ++ ++/* The terminal state of the epoch (one GPU): from the first terminal note on, no register access. */ ++static struct g5p_ru_terminal g5p_rv3_term; ++ ++/* Defaults of the latched knobs. GET_RENDER_INFO reports VARIANT_ACTIVE when ++ * any knob differs, so the declarations below and that comparison share them. ++ */ ++#define G5P_RU_DEF_CACHE_VARIANT 3 ++#define G5P_RU_DEF_MMU_VARIANT 1 ++/* 20 s: the natural TA completion took 12.006 s in the r96 boots a 12 s cap turned into ++ * timeouts (render-user-wait.h); it is also the minimum of a TYPED_V3 job's per-phase bound. ++ */ ++#define G5P_RU_DEF_WAIT_MS G5P_RU_V3_WAIT_MS ++#define G5P_RU_DEF_PROFILE1_VDM_CONTROL 0x280 ++ ++static const unsigned int render_user_wait_ms = 20000; ++MODULE_PARM_DESC( ++ render_user_wait_ms, ++ "Latched per-phase TA/3D bound, 3000..30000 ms (default 20000; TYPED_V3 needs " ++ "20000..30000, absolute job cap 30000)"); ++/* latency101 (default off): host-side TYPED_V3 wait policy; nothing the firmware sees depends on it. */ ++static const unsigned int render_user_hard_ms = 60000; ++static const unsigned int render_user_soft_ms = 45000; ++static const unsigned int render_user_obs_ms = 2000; ++MODULE_PARM_DESC( ++ render_user_hard_ms, ++ "Latched TYPED_V3 hard per-phase bound soft..60000 ms, the job bound is twice " ++ "it; 0 = off, render_user_wait_ms and the 30000 job cap apply (default 0)"); ++MODULE_PARM_DESC( ++ render_user_soft_ms, ++ "Latched TYPED_V3 soft per-phase bound 20000..hard ms, 0 = 45000; needs " ++ "render_user_hard_ms"); ++MODULE_PARM_DESC( ++ render_user_obs_ms, ++ "Latched RAM-only observation window after a TYPED_V3 timeout, 0..5000 ms " ++ "(default 2000; needs render_user_hard_ms)"); ++/* release (default off): RAM-only touch A/B of the TYPED_V3 wait (render-user-touch.h). */ ++ ++/* repeat109: the TYPED_V3 3D completion without the idle words (render-user-retire.h). */ ++ ++/* wake110 (default off): Apple's submit kick at the TYPED_V3 publish (render-user-wake.h). */ ++/* release8 default */ ++ ++/* TEMPORARY (multi-client stage 1): the fallback for a firmware that never drains a DC3 ++ * published after a job retired. Remove once the device has settled that question. ++ */ ++ ++/* Stage 2 fault isolation (render-user-loss.h): how long the scheduler holds every other job ++ * while it waits, RAM only, for a lost job's stuck phase. Read at the loss. ++ */ ++ ++/* Stage 2 in-boot recovery (core recover.inc): a job that stopped the firmware loses only its ++ * context, and the GPU is power-cycled and its firmware booted again while the other contexts' ++ * jobs are held (each leaves the old epoch's firmware pages allocated); 0 = such a job ends the ++ * epoch, as before. How many: render_user_recover_streak. Off by default; the release tuple turns ++ * it on (faultiso F1-F4, 2026-11-02: two recoveries in one boot on hardware). ++ */ ++ ++/* render-user-recover.h: Apple restarts the GPU on every recovery packet. Nonzero: recover every ++ * time, but end the epoch after this many recoveries in a row with no job completing between ++ * them. 0 = render_user_recover counts recoveries per boot, as before (a boot's second fault then ++ * ends the epoch, however long the GPU ran in between). ++ */ ++ ++/* Fault-isolation test (host side only: the firmware runs the job as any other). A context ++ * created while this is N gives up on its N-th job's TA after 1 ms instead of its bound, so the ++ * host loses that job the way it loses a stuck one, while the firmware completes it late: the ++ * context is lost and the channel drained behind it. 0 = off (default). ++ */ ++ ++/* Fault-isolation test with a real GPU fault (faultiso). A context created by a CAP_SYS_RAWIO ++ * process while this is N reserves one page of its VM that is never mapped ++ * (G5P_RU_TEST_FAULT_VA), and its N-th job's TA fetches its control stream from that page ++ * instead of the validated stream: the TA reads an unmapped VA of the context's own VM. What ++ * the firmware does then (a fault event, a silent stream, or a stuck engine) is what the test ++ * records; each ends that job and loses its context. 0 = off (default). ++ */ ++ ++/* render_user_test_fault_job's page: the last GiB below the hardware VA limit, far from every ++ * kernel-private range (low sink, aux, PDS window, firmware window). ++ */ ++#define G5P_RU_TEST_FAULT_VA (G5P_RU_HW_VA_LIMIT - 0x40000000ULL) ++ ++/* Stage 3 (render-user-window.h): which firmware memory a render VM maps. Latched per context ++ * at creation. The kill switch 0 restores the stage 1 mapping and then needs CAP_SYS_RAWIO. ++ */ ++ ++/* The latency101 knobs are valid when all are 0 (off), or hard is in [soft, 60000], soft (0 = ++ * min(45000, hard)) in [20000, hard], obs <= 5000 and the dialect is TYPED_V3. The release touch ++ * is 0, or 1..20000 below the effective soft bound with latency101 on. The 3D idle settle is ++ * 0..5000. The wake110 mode is 0, or 1..2 with latency101 on and the touch off (one A/B per ++ * context). Anything else refuses the context. ++ */ ++static bool g5p_ru_lat_params_valid(void) ++{ ++ u32 hard = READ_ONCE(render_user_hard_ms), ++ soft = READ_ONCE(render_user_soft_ms); ++ ++ return g5p_ru_lat_valid(soft, hard, READ_ONCE(render_user_obs_ms)) && ++ g5p_ru_touch_valid((0), g5p_ru_lat_soft(soft, hard), hard) && ++ g5p_ru_rt_settle_valid((G5P_RU_RT_DEF_SETTLE_MS)) && ++ g5p_wk_mode_valid((2)) && (!(2) || (hard && !(0))) && ++ (!hard || (1)); ++} ++/* r25 frame-capable selections, same values as the fixed render_*_variant. */ ++ ++#define G5P_RU_MAX_QUEUED 16 ++#define G5P_RU_MAX_TARGETS 32 ++#define G5P_RU_RESULTS 1024 ++#define G5P_RU_INPUT_MS 10000 ++ ++struct g5p_rt { ++ struct list_head link; ++ struct g5p_context *ctx; ++ struct g5p_ru_rt geometry; ++ struct g5p_ru_rt_sizes sizes; ++ struct drm_mm_node node; ++ struct g5p_dma_page *pages; ++ u32 width, height, page_count; ++ u32 samples; /* 1 or 4 (CREATE_RENDER_TARGET); 0 = 1 */ ++ bool destroyed; ++}; ++ ++/* A pair job 1 bound to a kernel-owned catalogue (render_user_spare_pairs, TEMPORARY). */ ++struct g5p_rspare { ++ u32 slot[2]; /* TA, 3D */ ++ struct g5p_dma_page root; /* the catalogue CBASE points at */ ++ bool ok, used; ++}; ++ ++/* ++ * Device-wide render state (docs/gpu/multi-client-design.md §4.1): the one TA/3D channel pair ++ * and the one PM record with its pool and aux pages, shared by every render context. Jobs of ++ * all contexts run one at a time on render_wq with the core lock held, and only they write ++ * up, v3, sequence, pm_serial and the spares. `lock` guards the pool and the aux pages, which ++ * render-target creation grows while another context's job may run. The scheduler's worker ++ * is the only executor: it picks the jobs of all contexts round-robin. ++ */ ++/* The kernel-built firmware window shared by every render VM (render-user-window.h): built ++ * once from the firmware's tables, never edited, kept until reboot. ++ */ ++struct g5p_rwin { ++ bool built; ++ u32 pages, missing, leaves; ++ struct g5p_dma_page dir, leaf[G5P_RU_FW_PAGES]; ++ u32 leaf_di[G5P_RU_FW_PAGES]; ++}; ++ ++struct g5p_rdev { ++ struct mutex lock; ++ bool up, v3; /* PM and channels built; by the TYPED_V3 attach */ ++ bool dc_pending; /* a bind's DC3 pair was not consumed in time ... */ ++ u32 dc_wait; /* ... and this is the consumer it waits for */ ++ u32 sequence, pm_serial; ++ u32 cu_sequence, cu_target; /* TYPED_COMPUTE: CDM sequence, CL producer target */ ++ u32 blocks; ++ /* perf60, latched at the epoch's first TYPED_V3 attach: 2 = jobs alternate two record slots ++ * (render-user-layout.h), else 1; pm_once = the PM pool is built at that attach only. ++ */ ++ u32 job_slots; ++ bool pm_once; ++ /* perf60 stage 4, latched there too: render_user_pm_ctx (0 off, 1 per-context PM records, ++ * 2 also per-context PM IDs) and the boot-reserved sets in use (core lock). ++ */ ++ u32 pm_ctx, pm_set_next; ++ bool pm_set_used[G5P_PMSET_MAX]; ++ /* the context of the last job the executor ran and the previous distinct one, the PM's ++ * write-back targets (render-user-pmset.h; compared, never dereferenced), and the closed ++ * contexts whose unbind waits until they are neither (render_user_close_defer) ++ */ ++ struct g5p_render *pm_last, *pm_prev, *close_deferred[2]; ++ struct g5p_dma_page *physical; ++ struct g5p_dma_page aux[G5P_RU_AUX_PAGES]; ++ struct g5p_rspare *spares; ++ u32 spare_count; ++ /* render_user_close_park: the park catalogue a closed pair is bound to (allocated at the ++ * first park, kept for the device's life; rebuilt at every park) ++ */ ++ struct g5p_dma_page park; ++ u32 parks; ++ bool executing; /* a TYPED_V3 job is in g5p_rv3_execute() (reboot quiesce) */ ++ /* The scheduler (drm-render-sched.inc): its run list under sched_lock, and the one ++ * worker on render_wq that runs every job and close item. ++ */ ++ struct g5p_drm *drm; ++ spinlock_t sched_lock; ++ struct g5p_rs sched; ++ struct delayed_work sched_work; ++ /* boot_gpu_oc_mhz: the thermal stop's poll (system_wq, so a busy render_wq never starves it) */ ++ struct delayed_work oc_work; ++ unsigned long oc_armed; ++ u32 oc_ticks, oc_jobs; ++ struct g5p_rwin ++ window; /* written once, by the executor, core lock held */ ++ struct g5p_rstuck ++ *stuck; /* a lost job blocks the channel (drain), worker only */ ++ /* A lost job stopped the firmware: the worker recovers before any pick. Set by the ++ * executor, cleared by the worker; submissions and the claim pass meanwhile. ++ */ ++ bool recovering; ++ u32 recoveries; ++ u32 recover_streak; /* recoveries since a job completed (core lock) */ ++ struct list_head ++ contexts; /* every render context (lock), for the recovery */ ++}; ++ ++/* render-user-fault.h's ruling window on the device state: a failure noted while a TYPED_V3 job or ++ * a drain runs, or a recovery is pending, waits for the executor. RAM only. ++ */ ++static bool g5p_rdev_ruling(struct g5p_rdev *rd) ++{ ++ bool executing = ++ smp_load_acquire(&rd->executing); /* before `recovering` */ ++ ++ return g5p_ru_ruling(executing, READ_ONCE(rd->recovering)); ++} ++ ++/* A render context. Its firmware state is the slot pair (slot[], bound), owned here the way ++ * an Asahi queue owns its firmware context; the channels and the PM record are the device's. ++ */ ++struct g5p_render { ++ struct g5p_context *ctx; ++ u32 slot[2]; /* engine0 TA, engine1 normal3D */ ++ u32 jobs; /* jobs that started on this context */ ++ u32 dc_target; /* DC ring consumer after this context's binding */ ++ struct drm_mm private_mm; ++ struct drm_mm_node aux_node, low_node, firmware_node, private_node; ++ struct drm_mm_node ++ fault_node; /* render_user_test_fault_job's unmapped page */ ++ struct list_head targets; ++ struct g5p_rt *profile1_rt; ++ u32 target_count; ++ u64 fence_context, fence_seqno; ++ atomic_t queued; ++ /* bound: the slot pair is published (for a RAM-only bind: consumed and checked). ++ * retained: the binding holds a context and device reference until its unbind. ++ * closing: the handle is gone; the close item (g5p_render_close_run()) is queued. ++ * live_maps: a job retired on the binding, so VM_BIND MAP may run beside queued jobs. ++ */ ++ bool claimant, bound, retained, aux_mapped, detached, stage_done, ++ unhooked; ++ bool closing, live_maps; ++ bool lost_drained; /* a lost job of it drained: the firmware holds no use of it */ ++ struct g5p_rs_entity ++ entity; /* its place in the scheduler (drm-render-sched.inc) */ ++ struct list_head rdev_link; /* in rdev->contexts */ ++ /* Mappings unmapped beside queued jobs, freed as those retire (g5p_render_reap()). */ ++ spinlock_t zlock; ++ struct list_head zombies; ++ /* Latched at creation; mirrors whatever qualifies the fixed path. */ ++ u32 stage, cache_variant, mmu_variant, timestamp_variant, wait_ms; ++ u32 soft_ms, hard_ms, obs_ms; /* latency101; hard_ms == 0: off */ ++ u32 touch_ms; /* RC4a; 0: off (always 0 when hard_ms is) */ ++ u32 settle_ms; /* repeat109 minimum coherent run before the 3D event completes */ ++ u32 stall_job; /* render_user_test_stall_job; 0: off */ ++ u32 fault_job; /* render_user_test_fault_job; 0: off */ ++ u32 wake, wake_seed; /* wake110 (render-user-wake.h); wake 0: off */ ++ u32 wake_rang; /* wake110: 0xb00 writes of this context */ ++ u32 pm_policy, vdm_control; ++ u32 fw_window; /* render_user_fw_window, G5P_RU_FW_* */ ++ bool after_ta, cache_repeat; ++ struct g5p_r25_variants v; /* pm, memory, rt<=1, slots; others 0 */ ++ u32 target[2], status[2], selector[2]; ++ bool submitted[2], consumed[2], retired[2]; ++ /* TYPED_V3 (drm-render-v3.inc): serial jobs, the context is retained from its binding on. */ ++ bool v3_accepted; ++ /* perf60 render_user_pm_switch 2: this context's own PM state (S + aux) after its last job */ ++ u8 *pm_save; ++ ++ /* perf60 stage 4 (render_user_pm_ctx): its own PM set (-1: the device PM), pool pages, ++ * aux pages mapped at G5P_RU_PRIVATE_VA in its VM, and the epoch they were built for. ++ */ ++ int pm_set; ++ u32 pm_blocks, pm_epoch; ++ struct g5p_dma_page *pm_pool; ++ struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; ++ bool pm_aux_mapped; ++ bool pm_first; /* it brought the epoch up (attach): the device aux is its own */ ++ bool close_defer_run; /* render_user_close_defer: its deferred close is running */ ++}; ++ ++/* The lost job the channel waits behind (render-user-loss.h): its context (a reference held), ++ * its per-job state as the executor left it, and the drain bound read at the loss. ++ */ ++struct g5p_rstuck { ++ struct g5p_render *r; ++ struct g5p_rv3 v; ++ u32 seq, engine, drain_ms; ++ u64 lost_ns; ++}; ++ ++#include "cache-receipt.inc" ++ ++/* A listed BO that needs implicit fences and/or kernel cache maintenance. */ ++struct g5p_robj { ++ struct drm_gem_object *obj; /* reference held by the job */ ++ u32 access; ++ /* ACCESS_CACHE window (render_user_sync_range bit 1: the resource's range; else whole BO) */ ++ u64 lo, hi; ++}; ++ ++struct g5p_rjob { ++ struct g5p_cache_receipt cache; ++ struct g5p_rs_job node; /* in its context's FIFO until picked */ ++ struct g5p_drm *gpu; ++ struct g5p_context *ctx; ++ struct g5p_fence *fence; ++ struct dma_fence *inputs[G5P_MAX_SYNCS - 1]; ++ u32 input_count; ++ struct g5p_robj *objs; ++ u32 obj_count; ++ struct dma_fence **deps; /* implicit dependencies from dma_resv */ ++ u32 dep_count; ++ int setup_error; /* post-acceptance setup failure: fail without hardware */ ++ /* Inputs then deps, in order: dep_next is the first not yet seen signalled, cb_fence the ++ * one whose callback is armed (scheduler lock), dep_error a failed input or the deadline. ++ */ ++ u32 dep_next; ++ struct dma_fence *cb_fence; ++ struct dma_fence_cb cb; ++ bool cb_fired; ++ int dep_error; ++ u64 id; ++ unsigned long deadline; ++ u8 ta_w[G5P_RU_TA_W_BYTES], td_w[G5P_RU_3D_W_BYTES]; ++ bool v3; ++ bool compute; /* TYPED_COMPUTE (drm-compute-v1.inc): a CDM job */ ++ struct g5p_cu_regs cu; ++ struct g5p_rv3 rv3; ++ u64 submit_ns; /* latency123: the submit ioctl */ ++}; ++ ++static u8 *g5p_ru_page(struct g5p_boot *b, u32 engine) ++{ ++ return b->shared[(engine ? G5P_R22_3D : G5P_R22_TA) / G5P_GPU_PAGE].cpu; ++} ++ ++static u32 g5p_ru_meta(u32 engine) ++{ ++ return g5p_meta_address(G5P_SHARED_VA + ++ (engine ? G5P_R22_3D : G5P_R22_TA)); ++} ++ ++static u8 *g5p_ru_pm_cpu(struct g5p_boot *b) ++{ ++ return b->shared[G5P_R22_PM / G5P_GPU_PAGE].cpu; ++} ++ ++static u8 *g5p_ru_global(struct g5p_boot *b, u32 engine) ++{ ++ u32 off = b->layout.offset[0x47 + engine]; ++ ++ return b->root[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; ++} ++ ++static void g5p_ru_free_page(struct device *dev, struct g5p_dma_page *p) ++{ ++ if (p->cpu) ++ dma_free_coherent(dev, G5P_GPU_PAGE, p->cpu, p->dma); ++ p->cpu = NULL; ++} ++ ++/* Probe: the device render state and its scheduler. Its pages are allocated on demand and ++ * kept until reboot. ++ */ ++static void g5p_rsched_work(struct work_struct *work); ++static void g5p_oc_work(struct work_struct *work); ++ ++static int g5p_render_init(struct g5p_drm *d) ++{ ++ d->rdev = drmm_kzalloc(&d->drm, sizeof(*d->rdev), GFP_KERNEL); ++ if (!d->rdev) ++ return -ENOMEM; ++ d->rdev->drm = d; ++ INIT_LIST_HEAD(&d->rdev->contexts); ++ spin_lock_init(&d->rdev->sched_lock); ++ g5p_rs_init(&d->rdev->sched); ++ INIT_DELAYED_WORK(&d->rdev->sched_work, g5p_rsched_work); ++ INIT_DELAYED_WORK(&d->rdev->oc_work, g5p_oc_work); ++ return drmm_mutex_init(&d->drm, &d->rdev->lock); ++} ++ ++/* rdev->lock held: the aux pages (once) and a PM pool of at least `blocks`. The pool never ++ * shrinks; every page added reaches PoC like the bring-up pool, whether or not the device's ++ * bring-up has cleaned the pool yet. The pool, list and F are rebuilt from it before every ++ * job, so growth takes effect at the next job, and the job in flight keeps its pages. ++ */ ++static int g5p_rdev_grow(struct g5p_drm *d, u32 blocks) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_dma_page *physical; ++ u32 have = rd->blocks * 8, i; ++ int ret = 0; ++ ++ for (i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ if (!rd->aux[i].cpu) ++ ret = g5p_vm_alloc_page(d->drm.dev, &rd->aux[i]); ++ if (ret || blocks <= rd->blocks) ++ return ret; ++ if (blocks > G5P_RU_MAX_BLOCKS) ++ return -EINVAL; ++ physical = kvcalloc(blocks * 8, sizeof(*physical), GFP_KERNEL); ++ if (!physical) ++ return -ENOMEM; ++ if (have) ++ memcpy(physical, rd->physical, have * sizeof(*physical)); ++ for (i = have; !ret && i < blocks * 8; i++) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &physical[i]); ++ if (!ret) ++ g5p_table_clean_page(d->core, &physical[i]); ++ } ++ if (ret) { ++ while (i-- > have) ++ g5p_ru_free_page(d->drm.dev, &physical[i]); ++ kvfree(physical); ++ return ret; ++ } ++ kvfree(rd->physical); ++ rd->physical = physical; ++ rd->blocks = blocks; ++ return 0; ++} ++ ++/* rdev->lock held: the pool's page addresses, for the PM builders. */ ++static u64 *g5p_rdev_addresses(struct g5p_rdev *rd) ++{ ++ u64 *physical = ++ kvmalloc_array(rd->blocks * 8, sizeof(*physical), GFP_KERNEL); ++ ++ for (u32 i = 0; physical && i < rd->blocks * 8; i++) ++ physical[i] = rd->physical[i].dma; ++ return physical; ++} ++ ++/* From the moment the firmware may hold a context's slot pair, the context and everything it ++ * owns stay alive whatever its files do, until its close item has unbound the pair ++ * (g5p_render_close_run()); a pair that cannot be unbound keeps them until reboot. ++ */ ++static void g5p_render_retain(struct g5p_render *r) ++{ ++ if (r->retained) ++ return; ++ kref_get(&r->ctx->ref); ++ drm_dev_get(&r->ctx->gpu->drm); ++ r->retained = true; ++} ++ ++/* The RAM words of the binding of `slot` through the DC records at start and start + 1. */ ++static void g5p_render_bind_mem(struct g5p_boot *b, const u32 *slot, u32 start, ++ struct g5p_ru_bind_mem *m) ++{ ++ for (u32 i = 0; i < 2; i++) ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + ++ ((start + i) & 255) * 20 + j * 4; ++ ++ m->rec[i][j] = b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE; ++ } ++ for (u32 e = 0; e < 2; e++) { ++ m->pa[e] = g5p_boot_shared(b, 0x86) + slot[e] * 8; ++ m->host[e] = g5p_boot_shared(b, 0x84) + slot[e] * 4; ++ m->cached[e] = g5p_boot_shared(b, 0x85) + slot[e] * 4; ++ } ++} ++ ++/* ++ * Called with the file lock. Idle: no accepted job is queued or running. A VM_BIND (`live`) ++ * of a context with live_maps may also run beside its queued jobs, the way an Asahi VM_BIND ++ * does not wait for the queue: a compositor imports and releases client buffers while its ++ * previous frame is in flight. Its tag reaches the firmware at the next kick of the slots. A ++ * MAP only adds entries for VAs no accepted job references; an UNMAP clears its entries now but ++ * keeps the object pinned until the queued jobs have retired (g5p_resident_defer()), so a job ++ * that still used the range faults on the hardware instead of reaching freed memory. Not ++ * before a job has retired on the binding: the epoch's first job checks that the tables did not ++ * change since its attach, and a bind in flight computes its tag from them. Render-target ++ * edits still need the context idle. ++ */ ++static int g5p_render_vm_idle(struct g5p_context *c, bool live) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_boot *b = c->gpu->core->boot; ++ /* During a recovery no epoch may receive a tag. */ ++ bool busy = (atomic_read(&r->queued) && ++ !(live && READ_ONCE(r->live_maps))) || ++ READ_ONCE(c->gpu->rdev->recovering); ++ ++ if (c->bad || r->detached || busy) { ++ dev_warn_ratelimited( ++ c->gpu->drm.dev, ++ "G5P VM_BIND busy=render-idle bad=%u detached=%u queued=%d bound=%u\n", ++ c->bad, r->detached, atomic_read(&r->queued), r->bound); ++ return -EBUSY; ++ } ++ if (r->bound && (!b || READ_ONCE(b->failed) || READ_ONCE(b->stopped))) ++ return g5p_rdev_ruling(c->gpu->rdev) ? -EBUSY : -EIO; ++ return 0; ++} ++ ++/* perf60 MMU invalidation arm, default 0 (render-user-mmu.h): bit 0 every edit publication asks ++ * the firmware for a PC-level invalidate, bit 1 a bound context's job after another context's ++ * job retags its slots (PC), bit 2 NC-reads every table line before the tag store, bit 3 retags ++ * on every job. RAM only, like the tag path itself. ++ */ ++ ++/* An NC read of every 64-byte line of the VM's tables (g5p_rv3_fw_touch()'s visibility read). */ ++static void g5p_render_table_touch(struct g5p_vm *vm) ++{ ++ struct g5p_pt *pt; ++ ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(vm->root.cpu + at)); ++ list_for_each_entry(pt, &vm->tables, link) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)(pt->page.cpu + at)); ++} ++ ++/* File lock held, context idle (or a MAP with live_maps). After a PTE edit on a bound render ++ * VM: tables to PoC, then the packed tag to both of its slots. RAM only: no register access ++ * is safe after retirement, and the firmware reads these words only at this context's next ++ * kick, which then invalidates the slots' PC level. ++ */ ++static int g5p_render_invalidate(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_diag *g = c->gpu->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = c->vm; ++ struct g5p_pt *pt; ++ u32 tag, cached[2], bumps = 0, mode = (0); ++ ++ if (!READ_ONCE(r->bound) || r->detached) ++ return 0; ++ if (!b || c->bad || b->failed || b->stopped) ++ return -EIO; ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ dsb(sy); ++ if (mode & G5P_RU_MMU_TOUCH) ++ g5p_render_table_touch(vm); ++ tag = g5p_vm_generation(&vm->mutation); ++ for (u32 e = 0; e < 2; e++) ++ cached[e] = g5p_le32(g5p_boot_shared(b, 0x85) + r->slot[e] * 4); ++ while (tag == cached[0] || tag == cached[1]) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ bumps++; ++ } ++ if (bumps) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER generation collision cached=%08x/%08x bumps=%u tag=%08x\n", ++ cached[0], cached[1], bumps, tag); ++ if (mode & G5P_RU_MMU_EDIT) { ++ /* Advance the VM's own PC counter to the PC field the helper picked. */ ++ u32 pc = g5p_ru_mmu_pc_tag(tag, cached[0], cached[1]); ++ ++ if (pc) { ++ vm->mutation.generation[0] += ++ ((pc >> 22) - (tag >> 22)) & 0x3ff; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ } ++ for (u32 e = 0; e < 2; e++) ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, tag); ++ dma_wmb(); ++ if (mode) ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick mmu_inval=%x " ++ "cached=%08x/%08x\n", ++ tag, r->slot[0], r->slot[1], mode, cached[0], ++ cached[1]); ++ else ++ g5p_mmu_record( ++ g, ++ "RENDER-USER tag=%08x slots=%u/%u invalidate=firmware-on-kick\n", ++ tag, r->slot[0], r->slot[1]); ++ return 0; ++} ++ ++/* DC3 consumption precedes this CBASE check and selected-slot invalidation; ++ * the r22 diagnostic sequence with the user catalogue instead of a private PC. ++ */ ++ ++/* CPU-written firmware objects of one job: TA/3D pages, PM control and the ++ * list/free pages. Coherent allocations plus explicit PoC, as r22 does. ++ */ ++static void g5p_render_clean_job(struct g5p_diag *g) ++{ ++ struct g5p_boot *b = g->boot; ++ ++ for (u32 off = G5P_R22_TA; off <= G5P_R22_PM; off += G5P_GPU_PAGE) ++ g5p_table_clean_page(g, &b->shared[off / G5P_GPU_PAGE]); ++ for (u32 i = 0; i < 2; i++) ++ g5p_table_clean_page(g, &b->render_meta[i]); ++ dsb(sy); ++ dma_wmb(); ++} ++ ++/* RAM only: the DC ring is idle and neither slot of this context has a PA or a tag. */ ++static int g5p_render_vm_ready(struct g5p_boot *b, struct g5p_render *r) ++{ ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ struct g5p_ru_bind_sample s; ++ struct g5p_ru_bind_mem m; ++ u32 start = READ_ONCE(ctl[4]); ++ ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_read(&m, READ_ONCE(ctl[0]), start, &s); ++ return g5p_ru_bind_busy(&s) ? -EBUSY : 0; ++} ++ ++/* The epoch's first job, core lock held, device not up: the PM record from the device pool ++ * and both channels, and every page they reach cleaned to PoC. Doing this again with the ++ * device up would reset the channels and the PM state that every context shares. ++ */ ++static int g5p_render_device_prepare(struct g5p_drm *d, ++ const struct g5p_r25_variants *v) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u64 *physical; ++ int ret; ++ ++ mutex_lock(&rd->lock); ++ physical = g5p_rdev_addresses(rd); ++ ret = physical ? 0 : -ENOMEM; ++ if (!ret) ++ ret = g5p_ru_pm(g5p_ru_pm_cpu(b), b->render_meta[0].cpu, ++ b->render_meta[1].cpu, ++ G5P_SHARED_VA + G5P_SHARED_SIZE, ++ G5P_SHARED_VA + G5P_SHARED_SIZE + G5P_GPU_PAGE, ++ physical, rd->blocks, G5P_RU_PRIVATE_VA); ++ if (!ret) ++ g5p_ru_pm_variant(g5p_ru_pm_cpu(b), b->render_meta[1].cpu, ++ physical, rd->blocks, v->pm); ++ kvfree(physical); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) ++ g5p_table_clean_page(g, &rd->aux[i]); ++ for (u32 i = 0; !ret && i < rd->blocks * 8; i++) ++ g5p_table_clean_page(g, &rd->physical[i]); ++ mutex_unlock(&rd->lock); ++ if (ret) ++ return ret; ++ rd->pm_serial = 0; ++ for (u32 e = 0; e < 2; e++) ++ g5p_ru_channel_init(g5p_ru_page(b, e), g5p_ru_meta(e)); ++ g5p_cu_device_init(d); ++ g5p_render_clean_job(g); ++ return 0; ++} ++ ++/* ++ * Core lock held, firmware up: the root entry of the firmware GiB for a VM in mode `fw_window`. ++ * The window is built at its first use from the firmware's leaf entries of the pages it keeps ++ * (render-user-window.h), and then shared, unchanged, by every render VM. RAM only. ++ */ ++static int g5p_render_fw_entry(struct g5p_drm *d, struct g5p_render *r, ++ u32 *entry) ++{ ++ struct g5p_rwin *w = &d->rdev->window; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u8 *leaves[ARRAY_SIZE(b->leaf)]; ++ int ret = 0; ++ ++ *entry = 0; ++ if (r->fw_window == G5P_RU_FW_FULL) ++ *entry = g5p_le32(b->catalogue.cpu + G5P_RU_FIRMWARE_CI * 4); ++ if (r->fw_window != G5P_RU_FW_WINDOW) ++ return 0; ++ if (w->built) { ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ return 0; ++ } ++ for (u32 i = 0; i < b->leaves; i++) ++ leaves[i] = b->leaf[i].cpu; ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->dir); ++ /* perf60 stage 4: the per-context PM sets reserved at boot, after the device PM's pages */ ++ for (u32 i = 0; ++ !ret && i < G5P_RU_FW_PAGES + b->pm_sets * G5P_PMSET_PAGES; i++) { ++ u32 j = i - G5P_RU_FW_PAGES; ++ u64 va = i < G5P_RU_FW_PAGES ? ++ g5p_ru_fw_window_va[i] : ++ g5p_pmset_va(j / G5P_PMSET_PAGES, ++ j % G5P_PMSET_PAGES); ++ u64 leaf = g5p_ru_fw_leaf(leaves, b->leaf_index, b->leaves, va); ++ u32 k = g5p_ru_fw_slot(w->leaf_di, w->leaves, va); ++ ++ if (!(leaf & 1)) { ++ w->missing++; ++ continue; ++ } ++ if (k == w->leaves) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &w->leaf[k]); ++ if (ret) ++ break; ++ w->leaf_di[k] = g5p_ru_fw_di(va); ++ g5p_put64(w->dir.cpu + w->leaf_di[k] * 8, ++ g5p_directory_entry(w->leaf[k].dma)); ++ w->leaves++; ++ } ++ g5p_put64(w->leaf[k].cpu + g5p_ru_fw_ti(va) * 8, leaf); ++ w->pages++; ++ } ++ if (ret) { ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_ru_free_page(d->drm.dev, &w->leaf[i]); ++ g5p_ru_free_page(d->drm.dev, &w->dir); ++ memset(w, 0, sizeof(*w)); ++ return ret; ++ } ++ for (u32 i = 0; i < w->leaves; i++) ++ g5p_table_clean_page(g, &w->leaf[i]); ++ g5p_table_clean_page(g, &w->dir); ++ w->built = true; ++ *entry = g5p_catalogue_entry(w->dir.dma); ++ dev_info( ++ g->dev, ++ "RENDER-V3 fw-window pages=%u missing=%u leaves=%u dir=%pad end=v3\n", ++ w->pages, w->missing, w->leaves, &w->dir.dma); ++ return 0; ++} ++ ++/* Core lock held, between jobs: the DC3 pair that points this context's slots at its own page ++ * catalogue. Maps the firmware window (g5p_render_fw_entry()) and borrows the sink catalogue ++ * entry, cleans the tables and this ++ * context's render targets, then writes the PA and tag of both slots and the two records. ++ * RAM only, nothing is written when the ring is busy or a slot is in use; the caller stores ++ * the producer. ++ */ ++static int g5p_render_vm_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_vm *vm = r->ctx->vm; ++ u32 *ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ u32 start = READ_ONCE(ctl[4]), tag, entry; ++ struct g5p_ru_bind_mem m; ++ struct g5p_pt *pt; ++ struct g5p_rt *rt; ++ int ret; ++ ++ ret = g5p_render_vm_ready(b, r); ++ if (!ret) ++ ret = g5p_render_fw_entry(d, r, &entry); ++ if (ret) ++ return ret; ++ g5p_vm_entry(&vm->mutation, vm->root.cpu + G5P_RU_FIRMWARE_CI * 4, ++ entry, 0); ++ if (b->sink_ready) ++ g5p_vm_entry(&vm->mutation, vm->root.cpu, ++ g5p_le32(b->catalogue.cpu), 0); ++ g5p_table_clean_page(g, &vm->root); ++ list_for_each_entry(pt, &vm->tables, link) ++ g5p_table_clean_page(g, &pt->page); ++ list_for_each_entry(rt, &r->targets, link) ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(g, &rt->pages[i]); ++ dsb(sy); ++ tag = g5p_vm_generation(&vm->mutation); ++ /* A zero tag would read as unbound: the unbind check takes a zero cached tag as proof ++ * that the DC3 handler ran. ++ */ ++ while (!tag && vm->mutation.generations) { ++ vm->mutation.generation[2]++; ++ tag = g5p_vm_generation(&vm->mutation); ++ } ++ g5p_render_bind_mem(b, r->slot, start, &m); ++ g5p_ru_bind_publish(&m, r->slot, vm->root.dma, tag); ++ *start_out = start; ++ *tag_out = tag; ++ return 0; ++} ++ ++/* The epoch's first job, core lock held: build the device, then bind this context's pair. ++ * Any failure leaves every allocation and binding retained. ++ */ ++static int g5p_render_attach_prepare(struct g5p_drm *d, struct g5p_render *r, ++ u32 *start_out, u32 *tag_out) ++{ ++ int ret = g5p_render_vm_ready(d->core->boot, r); ++ ++ if (!ret) ++ ret = g5p_render_device_prepare(d, &r->v); ++ return ret ?: g5p_render_vm_prepare(d, r, start_out, tag_out); ++} ++ ++/* The legacy profiles' attach: the epoch's first job, with a doorbell and a host MMU pass. */ ++ ++/* Pointer contents, channel producer, global producer and kick, each after a ++ * barrier. The channel must have retired the previous job's endpoint. ++ */ ++ ++/* Same predicates as g5p_r22_observe() for sequence `seq`. */ ++ ++#include "drm-render-v3.inc" ++#include "drm-render-sched.inc" ++#include "drm-render-unbind.inc" ++ ++static void g5p_render_job_free(struct g5p_rjob *job) ++{ ++ for (u32 i = 0; i < job->input_count; i++) ++ dma_fence_put(job->inputs[i]); ++ for (u32 i = 0; i < job->dep_count; i++) ++ dma_fence_put(job->deps[i]); ++ kfree(job->deps); ++ for (u32 i = 0; i < job->obj_count; i++) ++ drm_gem_object_put(job->objs[i].obj); ++ kvfree(job->objs); ++ if (job->fence) ++ dma_fence_put(&job->fence->base); ++ if (job->ctx) ++ g5p_context_put(job->ctx); ++ kfree(job); ++} ++ ++/* CPU cache maintenance on the G5P mapping (for an imported BO, the attachment's sg_table): clean ++ * before the job, invalidate after retirement. Whole BO, or (render_user_sync_range bit 1, latched ++ * at submit) the resource's BO range, the same window both ways. ++ */ ++static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, ++ bool to_cpu) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct g5p_robj *o = &job->objs[i]; ++ struct sg_table *sgt = to_drm_gem_shmem_obj(o->obj)->sgt; ++ ++ if (!(o->access & DRM_G5P_RENDER_ACCESS_CACHE) || !sgt) ++ continue; ++ /* A buffer the job only reads (a sampled client dma-buf or wl_shm udmabuf) ++ * was cleaned before the job and the GPU wrote none of it: no invalidate, ++ * which could discard what the CPU side wrote meanwhile into lines of the ++ * pages the buffer shares (a udmabuf covers whole 16 KiB pages). ++ */ ++ if (to_cpu && !(o->access & DRM_G5P_RENDER_ACCESS_WRITE)) ++ continue; ++ struct g5p_cache_receipt *cr = &job->cache; ++ bool imported = drm_gem_is_imported(o->obj); ++ bool exported = READ_ONCE(o->obj->dma_buf) != NULL; ++ bool wc = to_drm_gem_shmem_obj(o->obj)->map_wc; ++ int cls = g5p_cache_class(imported, exported, wc); ++ u64 bytes = o->hi ? o->hi - o->lo : o->obj->size; ++ u64 cpu = 0, wall = 0; ++ if (cr->enabled) ++ cr->requested[to_cpu][cls] += bytes; ++ /* Native pages are initially DMA-mapped with no SKIP_CPU_SYNC. ++ * All later CPU views of a private map_wc object (GEM mmap/vmap, ++ * including command validation) are write-combined. There is no ++ * cacheable client alias to clean/invalidate. Shared/foreign BOs ++ * remain conservative. Barriers below still order WC accesses. ++ */ ++ bool imported_uncached = imported && ++ drm_gem_dma_buf_is_cpu_uncached(o->obj->import_attach->dmabuf); ++ if ((imported_uncached && cr->import_policy) || g5p_cache_omit(cr->policy, imported, exported, wc)) ++ continue; ++ if (cr->enabled) { cpu = task_sched_runtime(current); wall = ktime_get_ns(); } ++ if (o->hi && (o->lo || o->hi != o->obj->size)) ++ g5p_sync_sgt_range(d->drm.dev, sgt, o->lo, o->hi, ++ to_cpu); ++ else if (to_cpu) ++ dma_sync_sgtable_for_cpu(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ else ++ dma_sync_sgtable_for_device(d->drm.dev, sgt, ++ DMA_BIDIRECTIONAL); ++ if (cr->enabled) { ++ cr->maintained[to_cpu][cls] += bytes; ++ cr->cpu[to_cpu][cls] += task_sched_runtime(current) - cpu; ++ cr->wall[to_cpu][cls] += ktime_get_ns() - wall; ++ } ++ } ++ if (to_cpu) dma_rmb(); else dma_wmb(); ++} ++ ++/* After acceptance, render_submit_lock held, no file lock: one dma_resv at a ++ * time, so no ww ordering is needed. Our own context's earlier fences are ++ * already ordered by the queue. A failure fails the job before hardware. ++ */ ++static int g5p_render_implicit(struct g5p_rjob *job, u64 own_context) ++{ ++ for (u32 i = 0; i < job->obj_count; i++) { ++ struct dma_resv *resv = job->objs[i].obj->resv; ++ bool write = job->objs[i].access & DRM_G5P_RENDER_ACCESS_WRITE; ++ struct dma_fence **fences = NULL, **grown; ++ unsigned int count = 0; ++ int ret; ++ ++ if (!(job->objs[i].access & DRM_G5P_RENDER_ACCESS_IMPLICIT)) ++ continue; ++ dma_resv_lock(resv, NULL); ++ ret = dma_resv_get_fences(resv, ++ write ? DMA_RESV_USAGE_READ : ++ DMA_RESV_USAGE_WRITE, ++ &count, &fences); ++ if (!ret) ++ ret = dma_resv_reserve_fences(resv, 1); ++ if (!ret) ++ dma_resv_add_fence(resv, &job->fence->base, ++ write ? DMA_RESV_USAGE_WRITE : ++ DMA_RESV_USAGE_READ); ++ dma_resv_unlock(resv); ++ grown = ret || !count ? ++ NULL : ++ krealloc_array(job->deps, ++ job->dep_count + count, ++ sizeof(*grown), GFP_KERNEL); ++ if (!ret && count && !grown) ++ ret = -ENOMEM; ++ if (grown) ++ job->deps = grown; ++ for (u32 j = 0; j < count; j++) { ++ if (!ret && fences[j]->context != own_context) ++ job->deps[job->dep_count++] = fences[j]; ++ else ++ dma_fence_put(fences[j]); ++ } ++ kfree(fences); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++/* Free the mappings unmapped beside jobs up to fence seqno `seqno` (all: every one). */ ++static void g5p_render_reap(struct g5p_render *r, u64 seqno, bool all) ++{ ++ struct g5p_mapping *m, *t; ++ unsigned long flags; ++ LIST_HEAD(done); ++ ++ spin_lock_irqsave(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &r->zombies, link) ++ if (all || m->retire <= seqno) ++ list_move_tail(&m->link, &done); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ list_for_each_entry_safe(m, t, &done, link) { ++ list_del(&m->link); ++ g5p_vm_obj_put(r->ctx->vm, m->obj); ++ drm_gem_object_put(m->obj); ++ kfree(m); ++ } ++} ++ ++/* One job, picked by the scheduler once its inputs signalled (or failed, or timed out). */ ++static void g5p_render_run(struct g5p_rjob *job) ++{ ++ struct g5p_drm *d = job->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_render *r = job->ctx->render; ++ unsigned long lock_deadline, flags; ++ bool hardware = false; ++ int ret; ++ ++ job->cache.enabled = READ_ONCE(render_user_cache_receipts); ++ job->cache.policy = READ_ONCE(render_user_cache_private_wc); ++ job->cache.import_policy = READ_ONCE(render_user_cache_import_wc); ++ if (job->cache.enabled) { ++ job->cache.job = job->id; ++ job->cache.start_cpu = task_sched_runtime(current); ++ job->cache.start_wall = ktime_get_ns(); ++ } ++ /* Admission closes only for good (reboot quiesce, FINAL teardown): a job queued before ++ * that fails here, without waiting for its inputs and without competing for the command ++ * lock. ++ */ ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit)) ++ ret = -ESHUTDOWN; ++ else ++ ret = job->setup_error ?: job->dep_error; ++ if (!ret && (job->ctx->bad || READ_ONCE(g->boot->failed))) ++ ret = -EIO; ++ if (!ret) { ++ /* Bounded: never queue forever behind a command stuck in MMIO. */ ++ lock_deadline = jiffies + 2 * HZ; ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, lock_deadline)) { ++ ret = -EBUSY; ++ break; ++ } ++ msleep(5); ++ } ++ } ++ if (!ret) { ++ if (g->boot->failed || g->boot->stopped) { ++ ret = -EIO; ++ } else if (!1 && !g5p_ru_mmio_ok(&g5p_rv3_term)) { ++ /* A V3 epoch is terminal: the legacy path would read and write registers. */ ++ ret = -EIO; ++ } else { ++ /* RAM-only from the DC3 attach on: no snapshot (it reads R0), no ++ * export, no register access of any kind after a timeout. ++ */ ++ g5p_render_cache(d, job, false); ++ WRITE_ONCE(d->rdev->executing, true); ++ ret = g5p_rv3_execute(d, r, job); ++ /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ ++ smp_store_release(&d->rdev->executing, false); ++ hardware = job->rv3.touched || job->rv3.bind_failed; ++ if (!ret || ret == -ECANCELED) ++ g5p_render_cache( ++ d, job, ++ true); /* before status and fence */ ++ if (!ret && r->bound) ++ WRITE_ONCE(r->live_maps, true); ++ } ++ g->last_result = ret; ++ mutex_unlock(&g->lock); ++ } ++ spin_lock_irqsave(&d->status_lock, flags); ++ /* A failed device fails every later job; one context's lost job or failed bind does not. */ ++ if (READ_ONCE(g->boot->failed) || READ_ONCE(g->boot->event_failed)) ++ ret = ret ?: -EIO; ++ job->fence->state = DRM_G5P_JOB_COMPLETE; ++ job->fence->error = ret < 0 ? ret : 0; ++ d->last_job = job->id; ++ d->last_error = job->fence->error; ++ d->job_error = job->fence->error; ++ d->job_state = DRM_G5P_JOB_COMPLETE; ++ if (ret < 0 && ret != -ECANCELED && hardware) { ++ /* No reset protocol: quarantine the context and the device. A TYPED_V3 ++ * timeout is one exception for the device: nothing is wrong with the ++ * protocol, the firmware may still complete, and the drain must keep ++ * decoding (RAM only) so that a late completion is recorded. A failed ++ * bind is another: the DC ring is not on the path of the contexts ++ * already bound, so only this context is lost. A lost job is the last: ++ * the executor already ruled (the drain, or the recovery that replaces ++ * this epoch), and only its context is lost. ++ */ ++ WRITE_ONCE(job->ctx->bad, true); ++ if (g5p_ru_job_fails_device(1, ret, job->rv3.bind_failed, ++ job->rv3.lost)) ++ WRITE_ONCE(g->boot->failed, true); ++ } ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (ret < 0) ++ dma_fence_set_error(&job->fence->base, ret); ++ atomic_dec(&r->queued); ++ atomic_dec(&d->r8_queued); ++ dma_fence_signal(&job->fence->base); ++ /* A job that may still run on the hardware (a bad context) keeps what it might use. */ ++ if (!READ_ONCE(job->ctx->bad)) ++ g5p_render_reap(r, job->fence->base.seqno, false); ++ /* a failed job (complete=-N) always prints; a completed one is routine: job_log */ ++ if (ret) ++ dev_info( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ else ++ g5p_job_printk( ++ g->dev, ++ "RENDER-USER job=%llu seq=%u complete=%d slots=%u/%u\n", ++ job->id, job->v3 ? job->rv3.seq : d->rdev->sequence, ++ ret, r->slot[0], r->slot[1]); ++ if (job->cache.enabled) { ++ job->cache.total_cpu = task_sched_runtime(current) - job->cache.start_cpu; ++ job->cache.total_wall = ktime_get_ns() - job->cache.start_wall; ++ spin_lock_irqsave(&g5p_cache_ring_lock, flags); ++ job->cache.serial = g5p_cache_head + 1; ++ g5p_cache_ring[g5p_cache_head++ % G5P_CACHE_RECORDS] = job->cache; ++ spin_unlock_irqrestore(&g5p_cache_ring_lock, flags); ++ } ++ g5p_render_job_free(job); ++ ++ drm_dev_put(&d->drm); ++} ++ ++/* Private PM-aux/RT page with the latched r25 leaf flags (c by default). */ ++static int g5p_render_map_page(struct g5p_drm *d, struct g5p_render *r, u64 va, ++ struct g5p_dma_page *page) ++{ ++ u8 *pte = g5p_vm_pte(r->ctx->vm, d->drm.dev, va, true); ++ ++ if (IS_ERR_OR_NULL(pte)) ++ return pte ? PTR_ERR(pte) : -ENOMEM; ++ if (g5p_le32(pte) || g5p_le32(pte + 4)) ++ return -EBUSY; ++ g5p_vm_entry(&r->ctx->vm->mutation, pte, ++ g5p_leaf_entry(page->dma, g5p_r25_attributes(&r->v, va)), ++ 2); ++ return 0; ++} ++ ++/* clear: unmap first (file lock held, VM still hooked). */ ++static void g5p_rt_release_memory(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rt *rt, bool clear) ++{ ++ if (clear && rt->page_count) ++ g5p_vm_clear(r->ctx->vm, d->drm.dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_ru_free_page(d->drm.dev, &rt->pages[i]); ++ kvfree(rt->pages); ++ rt->pages = NULL; ++ rt->page_count = 0; ++ if (drm_mm_node_allocated(&rt->node)) ++ drm_mm_remove_node(&rt->node); ++} ++ ++/* File lock held, context idle; RAM only. Main ring then secondary in one ++ * RT_ALIGN node: the first 64x64 target lands exactly on r22's ++ * 101020000/101030000. ++ */ ++static int g5p_rt_create(struct g5p_drm *d, struct g5p_render *r, ++ const struct g5p_ru_rt *geometry, u32 width, ++ u32 height, struct g5p_rt **out) ++{ ++ struct g5p_rt *rt; ++ u64 bytes; ++ int ret; ++ ++ if (r->target_count >= G5P_RU_MAX_TARGETS) ++ return -ENOSPC; ++ rt = kzalloc(sizeof(*rt), GFP_KERNEL); ++ if (!rt) ++ return -ENOMEM; ++ rt->ctx = r->ctx; ++ rt->geometry = *geometry; ++ rt->width = width; ++ rt->height = height; ++ ret = g5p_ru_geometry_check(geometry, width, height, &rt->sizes); ++ if (ret) ++ goto err_free; ++ bytes = rt->sizes.main + rt->sizes.secondary_map; ++ ret = drm_mm_insert_node_in_range( ++ &r->private_mm, &rt->node, bytes, G5P_RU_RT_ALIGN, 0, ++ G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE, ++ DRM_MM_INSERT_LOW); ++ if (ret) ++ goto err_free; ++ rt->pages = ++ kvcalloc(bytes / G5P_GPU_PAGE, sizeof(*rt->pages), GFP_KERNEL); ++ if (!rt->pages) { ++ ret = -ENOMEM; ++ goto err_node; ++ } ++ for (; rt->page_count < bytes / G5P_GPU_PAGE; rt->page_count++) { ++ struct g5p_dma_page *p = &rt->pages[rt->page_count]; ++ ++ ret = g5p_vm_alloc_page(d->drm.dev, p); ++ if (!ret) ++ ret = g5p_render_map_page( ++ d, r, ++ rt->node.start + rt->page_count * G5P_GPU_PAGE, ++ p); ++ if (ret) { ++ g5p_ru_free_page(d->drm.dev, p); ++ goto err_pages; ++ } ++ } ++ if (r->bound) { ++ for (u32 i = 0; i < rt->page_count; i++) ++ g5p_table_clean_page(d->core, &rt->pages[i]); ++ ret = g5p_render_invalidate(r->ctx); ++ if (ret) { ++ /* Hardware may hold the new tag: retain the target. */ ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ return ret; ++ } ++ } ++ list_add_tail(&rt->link, &r->targets); ++ r->target_count++; ++ *out = rt; ++ g5p_vmlog_add(G5P_VMLOG_RT_NEW, r->ctx->vm, 0, rt->node.start, bytes, ++ rt->width << 16 | rt->height, rt->samples); ++ return 0; ++err_pages: ++ g5p_rt_release_memory(d, r, rt, true); ++ goto err_free; ++err_node: ++ drm_mm_remove_node(&rt->node); ++err_free: ++ kfree(rt); ++ return ret; ++} ++ ++/* Detach every VM-side trace: private PTEs and the reserved VA nodes. File ++ * lock held, or the file is closing (no concurrent VM ioctl can exist). ++ */ ++static void g5p_render_unhook(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct device *dev = c->gpu->drm.dev; ++ struct g5p_rt *rt; ++ ++ if (r->unhooked) ++ return; ++ list_for_each_entry(rt, &r->targets, link) ++ if (rt->page_count) ++ g5p_vm_clear(c->vm, dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ if (r->aux_mapped) ++ g5p_vm_clear(c->vm, dev, G5P_RU_PRIVATE_VA, ++ G5P_RU_AUX_PAGES * G5P_GPU_PAGE); ++ for (u32 i = 0; i < 4; i++) { ++ struct drm_mm_node *n = i == 0 ? &r->low_node : ++ i == 1 ? &r->firmware_node : ++ i == 2 ? &r->private_node : ++ &r->fault_node; ++ ++ if (drm_mm_node_allocated(n)) ++ drm_mm_remove_node(n); ++ } ++ if (c->vm->resident == c) ++ c->vm->resident = NULL; ++ r->unhooked = true; ++} ++ ++/* Last context reference: only reached unbound or after a successful DC3 ++ * unbind, so no firmware or GPU reference to this memory remains. The aux ++ * pages and the PM pool are the device's. ++ */ ++static void g5p_render_release(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_rt *rt, *tmp; ++ unsigned long flags; ++ ++ g5p_render_unhook(c); ++ g5p_render_reap(r, 0, true); ++ if (!list_empty(&r->rdev_link)) { ++ mutex_lock(&d->rdev->lock); ++ list_del_init(&r->rdev_link); ++ mutex_unlock(&d->rdev->lock); ++ } ++ list_for_each_entry_safe(rt, tmp, &r->targets, link) { ++ g5p_rt_release_memory(d, r, rt, false); ++ list_del(&rt->link); ++ kfree(rt); ++ } ++ if (drm_mm_node_allocated(&r->aux_node)) ++ drm_mm_remove_node(&r->aux_node); ++ drm_mm_takedown(&r->private_mm); ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ for (u32 e = 0; e < 2; e++) ++ if (r->slot[e]) ++ __clear_bit(r->slot[e], d->core->user_slots); ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ { ++ u32 aux = 0; ++ ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ aux += !!r->pm_aux[i].cpu; ++ /* release8 accounting: what this context frees, and the device's live GEM total */ ++ dev_info( ++ d->drm.dev, ++ "RENDER-V3 release slots=%u/%u jobs=%u set=%d pool_pages=%u aux_pages=%u " ++ "vm_pinned=%lluK live_bos=%u live_bo_bytes=%lluK end=v3\n", ++ r->slot[0], r->slot[1], r->jobs, r->pm_set, ++ r->pm_pool ? r->pm_blocks * 8 : 0, aux, ++ c->vm ? c->vm->pinned_bytes >> 10 : 0, ++ READ_ONCE(g5p_bo_count), READ_ONCE(g5p_bo_bytes) >> 10); ++ } ++ g5p_rv3_pmctx_release(d, r); ++ if (d->rdev->pm_last == r) ++ d->rdev->pm_last = NULL; ++ if (d->rdev->pm_prev == r) ++ d->rdev->pm_prev = NULL; ++ kvfree(r->pm_save); ++ kfree(r); ++ c->render = NULL; ++} ++ ++/* CREATE_CONTEXT type RENDER. Exclusive VM: reserves [64KiB, 1GiB) for the ++ * borrowed VA0 sink, the whole firmware GiB and the private RT/PM window. ++ */ ++static int g5p_render_create_context(struct drm_device *drm, ++ struct drm_file *file, ++ struct drm_g5p_create_context *a) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_r25_variants v = { ++ .pm = (0), ++ .memory = (0), ++ .rt = (0), ++ .slots = (0), ++ }; ++ unsigned long flags; ++ u32 id, fw_window; ++ int ret; ++ ++ BUILD_BUG_ON(G5P_USER_RESERVED_START & ((1ULL << 30) - 1)); ++ if (a->flags || a->handle) ++ return -EINVAL; ++ ; ++ ; ++ fw_window = (G5P_RU_FW_WINDOW); ++ /* Unprivileged unless the VM would map the whole firmware GiB (the kill switch). */ ++ if (fw_window >= G5P_RU_FW_MODES || ++ (fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO))) ++ return fw_window >= G5P_RU_FW_MODES ? -EINVAL : -EPERM; ++ if ((0) > 2 || ++ ((G5P_RU_DEF_CACHE_VARIANT) != 0 && ++ (G5P_RU_DEF_CACHE_VARIANT) != 3) || ++ (G5P_RU_DEF_MMU_VARIANT) > 1 || (0) > 1 || (0) > 1 || ++ READ_ONCE(render_user_wait_ms) < ((1) ? G5P_RU_V3_WAIT_MS : 3000) || ++ READ_ONCE(render_user_wait_ms) > G5P_RU_V3_CAP_MS || ++ !g5p_ru_lat_params_valid() || !(32) || (32) > G5P_RU_MAX_BLOCKS || ++ ((G5P_RU_DEF_PROFILE1_VDM_CONTROL) & 0xf) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) > G5P_RU_MAX_VDM_CONTROL || ++ !g5p_ru_r25_valid(&v)) ++ return -EINVAL; ++ c = kzalloc(sizeof(*c), GFP_KERNEL); ++ r = kzalloc(sizeof(*r), GFP_KERNEL); ++ if (!c || !r) { ++ kfree(c); ++ kfree(r); ++ return -ENOMEM; ++ } ++ r->pm_set = -1; ++ c->gpu = d; ++ c->render = r; ++ r->ctx = c; ++ INIT_LIST_HEAD(&r->targets); ++ INIT_LIST_HEAD(&r->rdev_link); ++ g5p_rs_entity_init(&r->entity); ++ spin_lock_init(&r->zlock); ++ INIT_LIST_HEAD(&r->zombies); ++ atomic_set(&r->queued, 0); ++ drm_mm_init(&r->private_mm, G5P_RU_PRIVATE_VA, G5P_RU_PRIVATE_SIZE); ++ r->stage = (0); ++ r->cache_variant = (G5P_RU_DEF_CACHE_VARIANT); ++ r->mmu_variant = (G5P_RU_DEF_MMU_VARIANT); ++ r->timestamp_variant = (0); ++ r->wait_ms = READ_ONCE(render_user_wait_ms); ++ r->hard_ms = READ_ONCE(render_user_hard_ms); ++ r->soft_ms = r->hard_ms ? ++ g5p_ru_lat_soft(READ_ONCE(render_user_soft_ms), ++ r->hard_ms) : ++ 0; ++ r->obs_ms = r->hard_ms ? READ_ONCE(render_user_obs_ms) : 0; ++ r->touch_ms = r->hard_ms ? (0) : 0; ++ r->settle_ms = (G5P_RU_RT_DEF_SETTLE_MS); ++ r->stall_job = (0); ++ r->fault_job = capable(CAP_SYS_RAWIO) ? (0) : 0; ++ r->wake = r->hard_ms && !r->touch_ms ? (2) : 0; ++ r->wake = g5p_wk_mode_valid(r->wake) ? r->wake : 0; ++ r->wake_seed = (0); ++ r->after_ta = (0); ++ r->pm_policy = (0); ++ r->cache_repeat = (0); ++ r->vdm_control = (G5P_RU_DEF_PROFILE1_VDM_CONTROL); ++ r->fw_window = fw_window; ++ r->v = v; ++ r->fence_context = dma_fence_context_alloc(1); ++ mutex_lock(&f->lock); ++ c->vm = xa_load(&f->vms, a->vm); ++ if (!c->vm) { ++ ret = -ENOENT; ++ goto err_unlock; ++ } ++ if (c->vm->contexts || c->vm->resident || c->vm->resident_gone || ++ c->vm->frozen || c->vm->serial_context) { ++ ret = -EBUSY; ++ goto err_unlock; ++ } ++ /* Memory mode 4 covers every render leaf, so no BO may be mapped yet. */ ++ if (v.memory == 4 && c->vm->map_count) { ++ ret = -EINVAL; ++ goto err_unlock; ++ } ++ r->low_node.start = G5P_USER_VA_START; ++ r->low_node.size = G5P_RU_USER_VA_MIN - G5P_USER_VA_START; ++ r->private_node.start = G5P_RU_PRIVATE_VA; ++ r->private_node.size = G5P_RU_PRIVATE_SIZE; ++ r->firmware_node.start = G5P_USER_RESERVED_END; ++ r->firmware_node.size = ++ ((G5P_RU_FIRMWARE_CI + 1) << 30) - G5P_USER_RESERVED_END; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->low_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->private_node) || ++ drm_mm_reserve_node(&c->vm->mm, &r->firmware_node) || ++ g5p_le32(c->vm->root.cpu + G5P_RU_FIRMWARE_CI * 4) || ++ g5p_le32(c->vm->root.cpu)) { ++ ret = -EINVAL; /* VM already maps a range this context needs */ ++ goto err_unlock; ++ } ++ if (r->fault_job) { ++ r->fault_node.start = G5P_RU_TEST_FAULT_VA; ++ r->fault_node.size = G5P_GPU_PAGE; ++ if (drm_mm_reserve_node(&c->vm->mm, &r->fault_node)) { ++ dev_warn( ++ d->drm.dev, ++ "RENDER-V3 test-fault off: va=%#llx already used in this VM end=v3\n", ++ G5P_RU_TEST_FAULT_VA); ++ r->fault_job = 0; ++ } ++ } ++ spin_lock_irqsave(&d->core->slot_lock, flags); ++ if (g5p_ru_slot_pair(d->core->user_slots, v.slots, r->slot)) { ++ __set_bit(r->slot[0], d->core->user_slots); ++ __set_bit(r->slot[1], d->core->user_slots); ++ } else { ++ r->slot[0] = 0; ++ r->slot[1] = 0; ++ } ++ spin_unlock_irqrestore(&d->core->slot_lock, flags); ++ if (!r->slot[0]) { ++ ret = -ENOSPC; ++ goto err_unlock; ++ } ++ /* The device's PM pool (at least this context's latched size) and aux pages, mapped at ++ * the same VA in every render VM, so the PM state each TA reloads is the one the ++ * previous job saved, whichever VM ran it. ++ */ ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, (32)); ++ mutex_unlock(&d->rdev->lock); ++ r->aux_node.start = G5P_RU_PRIVATE_VA; ++ r->aux_node.size = G5P_RU_AUX_PAGES * G5P_GPU_PAGE; ++ if (!ret) ++ ret = drm_mm_reserve_node(&r->private_mm, &r->aux_node); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ ret = g5p_render_map_page(d, r, ++ G5P_RU_PRIVATE_VA + i * G5P_GPU_PAGE, ++ &d->rdev->aux[i]); ++ r->aux_mapped = true; ++ } ++ if (ret) ++ goto err_unlock; ++ ret = xa_alloc(&f->contexts, &id, c, XA_LIMIT(1, G5P_MAX_CONTEXTS), ++ GFP_KERNEL); ++ if (ret) ++ goto err_unlock; ++ kref_init(&c->ref); ++ kref_get(&c->vm->ref); ++ mutex_lock(&d->rdev->lock); ++ list_add_tail(&r->rdev_link, &d->rdev->contexts); ++ mutex_unlock(&d->rdev->lock); ++ c->vm->contexts++; ++ c->vm->serial_context = true; ++ c->vm->resident = c; ++ c->vm->zero_leaf_flags = v.memory == 4; ++ a->handle = id; ++ mutex_unlock(&f->lock); ++ return 0; ++err_unlock: ++ if (c->vm) ++ g5p_render_release(c); ++ else ++ kfree(r); ++ mutex_unlock(&f->lock); ++ kfree(c); ++ return ret; ++} ++ ++/* DESTROY_CONTEXT, file lock held. A TYPED_V3 context that submitted: its close item, queued ++ * behind its jobs, unbinds and releases it (g5p_render_close()). Unbound: plain release. A ++ * legacy context that brought the device up, bound and idle: zero PA/tag and consume one DC3 ++ * per slot, as r21 detach does. Failure or a quarantined legacy context retains everything ++ * until reboot. ++ */ ++static int g5p_render_destroy(struct g5p_context *c) ++{ ++ struct g5p_render *r = c->render; ++ struct g5p_drm *d = c->gpu; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ u32 *ctl, start, target; ++ unsigned long deadline; ++ int ret; ++ ++ if (r->v3_accepted) { ++ g5p_render_close(c); ++ return 0; ++ } ++ if (atomic_read(&r->queued) || c->bad) ++ return -EBUSY; ++ if (!r->bound) { ++ g5p_render_unhook(c); ++ if (r->claimant) { ++ /* Claimed but never touched hardware: drop the claim's reference. The ++ * device stays claimed for render: other contexts may be queued on it. ++ */ ++ r->claimant = false; ++ d->quarantined_context = NULL; ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ } ++ return 0; ++ } ++ /* The legacy unbind rings a doorbell: retained once the epoch is terminal. */ ++ if (!r->claimant || !g5p_ru_mmio_ok(&g5p_rv3_term)) ++ return -EBUSY; ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ ret = -EIO; ++ if (b->failed || b->stopped) ++ goto out; ++ ctl = (u32 *)g5p_boot_shared(b, 0x6c); ++ start = READ_ONCE(ctl[4]); ++ ret = -EBUSY; ++ if (start > 255 || READ_ONCE(ctl[0]) != start) ++ goto out; ++ for (u32 e = 0; e < 2; e++) { ++ u32 index = (start + e) & 255; ++ ++ g5p_put64(g5p_boot_shared(b, 0x86) + r->slot[e] * 8, 0); ++ g5p_put32(g5p_boot_shared(b, 0x84) + r->slot[e] * 4, 0); ++ for (u32 j = 0; j < 5; j++) { ++ u32 at = b->layout.offset[0x6d] + index * 20 + j * 4; ++ ++ g5p_put32(b->shared[at / G5P_GPU_PAGE].cpu + ++ at % G5P_GPU_PAGE, ++ j == 0 ? 3 : ++ j == 1 ? r->slot[e] : ++ 0); ++ } ++ } ++ target = (start + 2) & 255; ++ dma_wmb(); ++ WRITE_ONCE(ctl[4], target); ++ dma_wmb(); ++ ret = g5p_prepare_kick(g, "RENDER-USER-unbind"); ++ if (ret) ++ goto fail; ++ writel(0x10, g->rgx + 0xb00); ++ deadline = jiffies + 3 * HZ; ++ do { ++ ret = g5p_boot_drain(g); ++ if (ret || b->failed) { ++ ret = ret ?: -EIO; ++ goto fail; ++ } ++ dma_rmb(); ++ if (READ_ONCE(ctl[0]) == target) ++ break; ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (READ_ONCE(ctl[0]) != target) { ++ ret = -ETIMEDOUT; ++ goto fail; ++ } ++ r->detached = true; ++ r->bound = false; ++ r->claimant = false; ++ WRITE_ONCE(c->live, false); ++ g5p_render_unhook(c); ++ /* One render attachment per boot: accepted stays held, so neither a new ++ * render owner nor compute can reuse the shared channel pages. ++ */ ++ d->quarantined_context = NULL; ++ g5p_mmu_record( ++ g, ++ "RENDER-USER detached slots=%u/%u; resources freed on release\n", ++ r->slot[0], r->slot[1]); ++ mutex_unlock(&g->lock); ++ g5p_context_put(c); ++ drm_dev_put(&d->drm); ++ return 0; ++fail: ++ c->bad = true; ++ b->failed = true; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, ++ const struct drm_g5p_sync *syncs, u32 count, ++ struct drm_syncobj **signal, u64 *point, ++ struct dma_fence_chain **chain) ++{ ++ u32 signal_handle = 0; ++ int ret; ++ ++ if (!count || count > G5P_MAX_SYNCS) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) { ++ for (u32 j = 0; j < i; j++) ++ if (syncs[i].handle == syncs[j].handle && ++ syncs[i].point == syncs[j].point && ++ syncs[i].flags == syncs[j].flags) ++ return -EINVAL; ++ if (syncs[i].flags & ~DRM_G5P_SYNC_SIGNAL) ++ return -EINVAL; ++ if (syncs[i].flags) { ++ if (*signal) ++ return -EINVAL; ++ *signal = drm_syncobj_find(file, syncs[i].handle); ++ if (!*signal) ++ return -ENOENT; ++ *point = syncs[i].point; ++ signal_handle = syncs[i].handle; ++ continue; ++ } ++ if (job->input_count == ARRAY_SIZE(job->inputs)) ++ return -E2BIG; ++ ret = drm_syncobj_find_fence(file, syncs[i].handle, ++ syncs[i].point, 0, ++ &job->inputs[job->input_count]); ++ if (ret) ++ return ret; ++ job->input_count++; ++ } ++ if (!*signal) ++ return -EINVAL; ++ for (u32 i = 0; i < count; i++) ++ if (!syncs[i].flags && syncs[i].handle == signal_handle && ++ (!*point || !syncs[i].point || syncs[i].point >= *point)) ++ return -EINVAL; ++ if (*point) { ++ struct dma_fence *old = drm_syncobj_fence_get(*signal); ++ ++ ret = old && (!to_dma_fence_chain(old) || ++ old->seqno >= *point) ? ++ -EINVAL : ++ 0; ++ dma_fence_put(old); ++ if (ret) ++ return ret; ++ *chain = dma_fence_chain_alloc(); ++ if (!*chain) ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++/* File lock held. One mapping of the listed BO must contain each range at a ++ * consistent BO offset; access is a subset of the BO's immutable roles. ++ */ ++static int g5p_render_resolve(struct drm_file *file, struct g5p_vm *vm, ++ const struct drm_g5p_render_resource *res, ++ u32 count, struct g5p_ru_range *ranges, ++ struct g5p_robj *objs, u32 *obj_count) ++{ ++ u32 known = DRM_G5P_RENDER_ACCESS_READ | DRM_G5P_RENDER_ACCESS_WRITE | ++ DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ ++ for (u32 i = 0; i < count; i++) { ++ struct drm_gem_object *obj; ++ struct g5p_mapping *m, *found = NULL; ++ u32 roles, access; ++ ++ for (u32 j = 0; j < i; j++) ++ if (res[i].handle == res[j].handle) ++ return -EINVAL; ++ obj = drm_gem_object_lookup(file, res[i].handle); ++ if (!obj) ++ return -ENOENT; ++ roles = to_g5p_bo(obj)->roles; ++ { ++ access = res[i].kind; ++ if (!(access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE)) || ++ access & ~known || ++ ((access & G5P_RU_ACCESS_READ) && ++ !(roles & DRM_G5P_BO_READ)) || ++ ((access & G5P_RU_ACCESS_WRITE) && ++ !(roles & DRM_G5P_BO_WRITE))) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ /* A foreign buffer always gets cache maintenance and fences. */ ++ if (drm_gem_is_imported(obj)) ++ access |= DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE; ++ } ++ list_for_each_entry(m, &vm->maps, link) ++ if (m->obj == obj && res[i].va >= m->node.start && ++ res[i].size && ++ res[i].va - m->node.start < m->node.size && ++ res[i].size <= m->node.size - (res[i].va - ++ m->node.start) && ++ res[i].offset == ++ m->offset + (res[i].va - m->node.start)) ++ found = m; ++ if (!found) { ++ drm_gem_object_put(obj); ++ return -EINVAL; ++ } ++ if (access & (DRM_G5P_RENDER_ACCESS_IMPLICIT | ++ DRM_G5P_RENDER_ACCESS_CACHE)) { ++ struct g5p_robj *o = &objs[*obj_count]; ++ ++ o->obj = obj; /* the job inherits this reference */ ++ o->access = access; ++ o->lo = 0; ++ o->hi = obj->size; ++ if ((READ_ONCE(render_user_sync_range) & 2) && ++ !g5p_sync_window(res[i].offset, res[i].size, ++ obj->size, &o->lo, &o->hi)) { ++ o->lo = 0; ++ o->hi = obj->size; ++ } ++ (*obj_count)++; ++ } else { ++ drm_gem_object_put(obj); ++ } ++ ranges[i].va = res[i].va; ++ ranges[i].size = res[i].size; ++ ranges[i].access = access & ++ (G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE); ++ /* The BO's EXEC role, for TYPED_V3 only; an imported BO never has it. */ ++ if ((roles & DRM_G5P_BO_EXEC)) ++ ranges[i].access |= G5P_RU_ACCESS_EXEC; ++ } ++ return 0; ++} ++ ++/* LINEAR64_V1 admits only the exact round16 immutable inputs. Writes stay ++ * privileged and caller-serialized, as for the r20 first-store dialect. ++ */ ++ ++/* File lock held. Copy `len` bytes at GPU VA `va` out of the BO the VM maps ++ * there, through the BO's own pages, as g5p_render_profile1_bytes() does. The ++ * mapping is found again by VA and its own (VM-referenced) object is read, so ++ * a handle closed or replaced since g5p_render_resolve() cannot redirect the ++ * read. Only a native BO has shmem pages: an imported one (never EXEC, so ++ * never a TA program) is refused for the stream too. ++ */ ++static int g5p_render_read_va(struct drm_device *drm, struct g5p_vm *vm, u64 va, ++ void *dst, u32 len) ++{ ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); ++ struct g5p_mapping *m, *found = NULL; ++ struct drm_gem_object *obj; ++ struct sg_table *sgt; ++ u64 off; ++ int ret; ++ ++ list_for_each_entry(m, &vm->maps, link) ++ if (va >= m->node.start && va - m->node.start < m->node.size && ++ len <= m->node.size - (va - m->node.start)) ++ found = m; ++ if (!found) ++ return -ENOENT; ++ obj = found->obj; ++ off = found->offset + (va - found->node.start); ++ if (drm_gem_is_imported(obj) || off > obj->size || ++ len > obj->size - off) ++ return -EINVAL; ++ sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj)); ++ ret = IS_ERR(sgt) ? PTR_ERR(sgt) : drm_gem_vmap(obj, &map); ++ if (ret) ++ return ret; ++ /* Keep content validation. Native private WC has no CPU cache alias; ++ * all other supported mappings retain a bounded DMA ownership sync. ++ */ ++ bool range = READ_ONCE(render_user_cache_read_va); ++ bool omit = range && g5p_cache_omit(READ_ONCE(render_user_cache_private_wc), ++ false, READ_ONCE(obj->dma_buf) != NULL, to_drm_gem_shmem_obj(obj)->map_wc); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, true); ++ else dma_sync_sgtable_for_cpu(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_rmb(); ++ if (map.is_iomem) ++ ret = -EINVAL; ++ else ++ memcpy(dst, (const u8 *)map.vaddr + off, len); ++ if (!omit) { ++ if (range) g5p_sync_sgt_range(drm->dev, sgt, off, off + len, false); ++ else dma_sync_sgtable_for_device(drm->dev, sgt, DMA_BIDIRECTIONAL); ++ } ++ dma_wmb(); ++ drm_gem_vunmap(obj, &map); ++ return ret; ++} ++ ++static void g5p_render_pass_from_uapi(struct g5p_ru_pass *p, ++ const struct drm_g5p_render_pass *u) ++{ ++ p->vdm_va = u->vdm_va; ++ p->vdm_size = u->vdm_size; ++ p->vdm_control = u->vdm_control; ++ p->scratch_va = u->scratch_va; ++ p->scratch_size = u->scratch_size; ++ memcpy(p->scratch_ptr, u->scratch_ptr, sizeof(p->scratch_ptr)); ++ p->pool7_va = u->pool7_va; ++ p->pool7_size = u->pool7_size; ++ p->pool8_va = u->pool8_va; ++ p->pool8_size = u->pool8_size; ++ memcpy(p->context_pds, u->context_pds, sizeof(p->context_pds)); ++ memcpy(p->resume_pds, u->resume_pds, sizeof(p->resume_pds)); ++ memcpy(p->ta_pds, u->ta_pds, sizeof(p->ta_pds)); ++ memcpy(p->background, u->background, sizeof(p->background)); ++ memcpy(p->eot, u->eot, sizeof(p->eot)); ++ memcpy(p->eot_resume, u->eot_resume, sizeof(p->eot_resume)); ++ memcpy(p->restore, u->restore, sizeof(p->restore)); ++} ++ ++static void g5p_render_pass2_from_uapi(struct g5p_ru_pass2 *e, ++ const struct drm_g5p_render_pass_ext *u) ++{ ++ *e = (struct g5p_ru_pass2){ ++ .scissor_va = u->scissor_va, ++ .dbias_va = u->dbias_va, ++ .background_texture = u->background_texture, ++ .occlusion_query_va = u->occlusion_query_va, ++ .occlusion_query_size = u->occlusion_query_size, ++ .scissor_count = u->scissor_count, ++ .dbias_count = u->dbias_count, ++ .bgobj_depth = u->bgobj_depth, ++ .bgobj_vals = u->bgobj_vals, ++ .pixel_width = u->pixel_output_width, ++ .zls_mask = u->zls_address_mask, ++ }; ++ memcpy(e->zls, u->zls, sizeof(e->zls)); ++} ++ ++static void g5p_render_pass3_from_uapi(struct g5p_ru_pass3 *v, ++ const struct drm_g5p_render_pass_v3 *u) ++{ ++ *v = (struct g5p_ru_pass3){ ++ .ta_program_va = u->ta_program_va, ++ .ta_program_size = u->ta_program_size, ++ .vdm_end = u->vdm_end, ++ .pixel_width = u->pixel_output_width, ++ .pad = u->pad, ++ }; ++} ++ ++static_assert(offsetof(struct drm_g5p_render_pass, samples) == 296); ++static_assert(offsetof(struct drm_g5p_render_pass, sample_positions) == 304); ++static_assert(offsetof(struct drm_g5p_render_pass, reserved) == 312); ++static_assert(DRM_G5P_CAP_RENDER_MSAA == BIT_ULL(31)); ++static_assert(DRM_G5P_CAP_RENDER_OCCLUSION_QUERY == BIT_ULL(33)); ++static_assert(DRM_G5P_RENDER_MSAA_RESOLVE == G5P_RU_MSAA_RESOLVE); ++ ++/* CAP_RENDER_MSAA: the target's geometry is kernel-derived (it holds the sample ++ * count), and only TYPED_V3 carries the pass fields. ++ */ ++static bool g5p_render_msaa_enabled(void) ++{ ++ return (1) && (1); ++} ++ ++/* The base pass's MSAA fields: zero unless TYPED_V3 with the capability. */ ++static int g5p_render_msaa_from_uapi(struct g5p_ru_msaa *m, ++ const struct drm_g5p_render_pass *u, ++ bool v3) ++{ ++ *m = (struct g5p_ru_msaa){ ++ .samples = u->samples, ++ .flags = u->msaa_flags, ++ .positions = u->sample_positions, ++ }; ++ if ((m->samples || m->flags || m->positions) && ++ (!v3 || !g5p_render_msaa_enabled())) ++ return -EINVAL; ++ return 0; ++} ++ ++/* Render submit lock held. The first accepted job claims the device for render until reboot; ++ * same exclusions as compute/r22. Every later job, of any context, passes while the device is ++ * neither failed nor stopped, without the core lock (a job may hold it for its whole run). The ++ * claimant keeps the reference d->quarantined_context has always held. The IRQ-test term is ++ * the r74 one (render-fixed.inc, render-r72.inc): the r74 boots refuse the IRQ test, so ++ * irq_tested must stay 0 there and must be 1 on every other boot. ++ */ ++static int g5p_render_device_claim(struct g5p_drm *d, struct g5p_context *c) ++{ ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_render *r = c->render; ++ int ret = -EBUSY; ++ ++ if (READ_ONCE(d->render_claimed)) { ++ /* A recovery swaps the epoch, and a failure noted mid-job may lead to one: the job ++ * waits for the ruling in the scheduler, which fails it if the epoch ends. ++ */ ++ if (g5p_rdev_ruling(d->rdev)) ++ return 0; ++ b = READ_ONCE(g->boot); ++ return !b || READ_ONCE(b->failed) || READ_ONCE(b->stopped) ? ++ -EIO : ++ 0; ++ } ++ if (!mutex_trylock(&g->lock)) ++ return -EBUSY; ++ if (b->failed || b->stopped || !b->control_complete || b->irq_tested || ++ b->user_compute_owned || d->quarantined_context || ++ !b->render_meta[0].cpu || !b->render_meta[1].cpu || ++ b->layout.used[1] > G5P_R22_TA || ++ /* recovery108: expose the GPU only with a current-generation ALIVE proof. */ ++ g5p_ds_hook_claim(g) || atomic_cmpxchg(&d->accepted, 0, 1)) ++ goto out; ++ b->user_compute_owned = true; /* blocks diagnostic commands, latched */ ++ r->claimant = true; ++ kref_get(&c->ref); ++ drm_dev_get(&d->drm); ++ d->quarantined_context = c; ++ WRITE_ONCE(d->render_claimed, true); ++ ret = 0; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++/* Evict the oldest completed result beyond the per-file bound. */ ++static int g5p_render_result_slot(struct g5p_file *f, struct g5p_drm *d) ++{ ++ unsigned long index = 0, flags; ++ struct g5p_fence *old; ++ bool done; ++ ++ if (f->render_results < G5P_RU_RESULTS) ++ return 0; ++ old = xa_find(&f->results, &index, ULONG_MAX, XA_PRESENT); ++ if (!old) ++ return 0; ++ spin_lock_irqsave(&d->status_lock, flags); ++ done = old->state == DRM_G5P_JOB_COMPLETE; ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (!done) ++ return -EBUSY; ++ xa_erase(&f->results, index); ++ dma_fence_put(&old->base); ++ f->render_results--; ++ return 0; ++} ++ ++static int g5p_render_submit_admitted(struct drm_device *drm, ++ struct drm_file *file, u32 context, ++ u32 profile, u32 rt_handle, u64 res_ptr, ++ u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, ++ u32 pass_size, u64 *job_id) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_boot *b = READ_ONCE(d->core->boot); ++ struct drm_g5p_render_resource *res = NULL; ++ struct drm_g5p_render_pass_v3 *upass = NULL; ++ struct drm_g5p_sync *syncs = NULL; ++ struct drm_syncobj *signal = NULL; ++ struct dma_fence_chain *chain = NULL; ++ struct g5p_ru_range *ranges = NULL; ++ struct g5p_ru_pass pass; ++ struct g5p_ru_pass2 pass2 = {}; ++ struct g5p_ru_pass3 pass3 = {}; ++ struct g5p_ru_v3_loc loc = {}; ++ struct g5p_ru_msaa msaa = {}; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rjob *job; ++ struct g5p_rt *rt = NULL; ++ const bool v2 = false; ++ const bool v3 = true; ++ u64 point = 0; ++ int ret; ++ ++ ; ++ ; ++ /* TYPED_V3 is unprivileged (checked per context below); the legacy profiles touch ++ * registers and stay with CAP_SYS_RAWIO. ++ */ ++ ; ++ ; ++ /* During a recovery the epoch is being replaced: accept, the scheduler holds the job. */ ++ if (!READ_ONCE(d->rdev->recovering)) { ++ if (!b || !READ_ONCE(b->alive) || !READ_ONCE(b->released)) ++ return -EAGAIN; ++ if (READ_ONCE(b->stopped)) ++ return -ESHUTDOWN; ++ /* a failure noted mid-job waits for the executor's ruling (g5p_rdev_ruling()) */ ++ if (READ_ONCE(b->failed) && !g5p_rdev_ruling(d->rdev)) ++ return -EIO; ++ } ++ res = memdup_array_user(u64_to_user_ptr(res_ptr), res_count, ++ sizeof(*res)); ++ if (IS_ERR(res)) ++ return PTR_ERR(res); ++ syncs = memdup_array_user(u64_to_user_ptr(sync_ptr), sync_count, ++ sizeof(*syncs)); ++ if (IS_ERR(syncs)) { ++ ret = PTR_ERR(syncs); ++ syncs = NULL; ++ goto out; ++ } ++ { ++ /* pass_size is the exact V1, V2 or V3 size; ext exists for V2 and V3, ++ * the program fields for V3 only. ++ */ ++ upass = memdup_user(u64_to_user_ptr(pass_ptr), pass_size); ++ if (IS_ERR(upass)) { ++ ret = PTR_ERR(upass); ++ upass = NULL; ++ goto out; ++ } ++ if (upass->base.size != pass_size || upass->base.flags || ++ memchr_inv(upass->base.reserved, 0, ++ sizeof(upass->base.reserved)) || ++ (true && memchr_inv(upass->ext.reserved, 0, ++ sizeof(upass->ext.reserved))) || ++ (v3 && ++ memchr_inv(upass->reserved, 0, sizeof(upass->reserved)))) { ++ ret = -EINVAL; ++ goto out; ++ } ++ g5p_render_pass_from_uapi(&pass, &upass->base); ++ ret = g5p_render_msaa_from_uapi(&msaa, &upass->base, v3); ++ if (ret) ++ goto out; ++ { ++ g5p_render_pass2_from_uapi(&pass2, &upass->ext); ++ }; ++ { ++ g5p_render_pass3_from_uapi(&pass3, upass); ++ }; ++ } ++ ranges = kvcalloc(res_count, sizeof(*ranges), GFP_KERNEL); ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (job) { ++ job->submit_ns = ktime_get_ns(); ++ job->objs = kvcalloc(res_count, sizeof(*job->objs), GFP_KERNEL); ++ } ++ if (!ranges || !job || !job->objs) { ++ if (job) ++ kvfree(job->objs); ++ kfree(job); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = g5p_render_syncs(file, job, syncs, sync_count, &signal, &point, ++ &chain); ++ if (ret) ++ goto out_job; ++ job->fence = kzalloc(sizeof(*job->fence), GFP_KERNEL); ++ if (!job->fence) { ++ ret = -ENOMEM; ++ goto out_job; ++ } ++ spin_lock_init(&job->fence->lock); ++ /* Acceptance, implicit fence attach and queueing happen in one order. */ ++ ret = mutex_lock_interruptible(&d->render_submit_lock); ++ if (ret) ++ goto out_job; ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto unlock; ++ } ++ r = c->render; ++ if (c->bad || r->detached) { ++ ret = -EIO; ++ goto unlock; ++ } ++ if (r->fw_window == G5P_RU_FW_FULL && !capable(CAP_SYS_RAWIO)) { ++ ret = -EPERM; ++ goto unlock; ++ } ++ { ++ ret = g5p_rv3_admit_submit(d, r); ++ if (ret) ++ goto unlock; ++ } ++ if (atomic_read(&r->queued) >= G5P_RU_MAX_QUEUED) { ++ ret = -EBUSY; ++ goto unlock; ++ } ++ ret = g5p_render_resolve(file, c->vm, res, res_count, ranges, job->objs, ++ &job->obj_count); ++ if (ret) ++ goto unlock; ++ { ++ rt = xa_load(&f->targets, rt_handle); ++ if (!rt || rt->ctx != c || rt->destroyed) { ++ ret = -ENOENT; ++ goto unlock; ++ } ++ } ++ { ++ static const struct g5p_ru_v3_limits lim = { ++ G5P_RU_PDS_BASE, ++ G5P_RU_CODE_WINDOW, ++ G5P_RU_HW_VA_LIMIT, ++ }; ++ u8 head[G5P_RU_PROGRAM_MIN], tail[4]; ++ ++ /* Structure first; the program head and the stream's last dword are ++ * read only once every range is proven backed, READ and (program) EXEC. ++ */ ++ ret = g5p_ru_pass3_check(&pass, &pass2, &pass3, &lim, ranges, ++ res_count, &loc); ++ if (!ret) ++ ret = g5p_render_read_va(drm, c->vm, ++ pass3.ta_program_va, head, ++ sizeof(head)); ++ if (!ret) ++ ret = g5p_render_read_va( ++ drm, c->vm, pass.vdm_va + pass3.vdm_end - 4, ++ tail, sizeof(tail)); ++ if (!ret) ++ ret = g5p_ru_pass3_content(head, tail); ++ } ++ /* Every profile: a one-sample pass needs a one-sample target, and back. */ ++ if (!ret) ++ ret = g5p_ru_msaa_check(&msaa, rt->samples ? rt->samples : 1); ++ if (ret) ++ goto unlock; ++ g5p_ru_fill_ta(job->ta_w, &pass); ++ { ++ g5p_ru_fill_ta_v3(job->ta_w, loc.program_off); ++ }; ++ g5p_ru_fill_3d(job->td_w, &pass); ++ /* TYPED_V3 with an all-zero ext is the V1 dialect: the V1 3D words, plus ++ * the output width words (width 2 rewrites the frozen dialect-1 values). ++ */ ++ if (v2 || loc.dialect2) ++ g5p_ru_fill_3d_v2(job->td_w, &pass2); ++ else { ++ g5p_ru_fill_3d_width(job->td_w, loc.pixel_width); ++ }; ++ /* r25 RT variant 1 moves only TA.W28/3D.W20 one slice into the ring. */ ++ g5p_ru_rt_fill(job->ta_w, job->td_w, &rt->geometry, ++ rt->node.start + (r->v.rt == 1 ? rt->sizes.slice : 0), ++ rt->node.start + rt->sizes.main); ++ { ++ g5p_ru_fill_msaa(job->ta_w, job->td_w, &msaa, &rt->geometry); ++ }; ++ ret = g5p_render_result_slot(f, d); ++ if (ret) ++ goto unlock; ++ job->id = atomic64_inc_return(&d->job_sequence); ++ dma_fence_init(&job->fence->base, &g5p_fence_ops, &job->fence->lock, ++ r->fence_context, ++r->fence_seqno); ++ job->fence->state = DRM_G5P_JOB_PENDING; ++ job->fence->error = -EINPROGRESS; ++ ret = xa_insert(&f->results, job->id, job->fence, GFP_KERNEL); ++ if (!ret) { ++ ret = g5p_render_device_claim(d, c); ++ if (ret) ++ xa_erase(&f->results, job->id); ++ } ++ if (ret) { ++ /* Unpublished, never signalled: still a clean rejection. */ ++ dma_fence_put(&job->fence->base); ++ job->fence = NULL; ++ goto unlock; ++ } ++ /* Accepted from here: every later step is infallible. */ ++ dma_fence_get(&job->fence->base); ++ f->render_results++; ++ job->v3 = v3; ++ { ++ r->v3_accepted = true; ++ }; ++ atomic_inc(&r->queued); ++ atomic_inc(&d->r8_queued); ++ kref_get(&c->ref); ++ job->ctx = c; ++ job->gpu = d; ++ job->cache.pid = task_tgid_nr(current); ++ mutex_unlock(&f->lock); ++ if (point) { ++ drm_syncobj_add_point(signal, chain, &job->fence->base, point); ++ chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(signal, &job->fence->base); ++ } ++ *job_id = job->id; ++ job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ job->setup_error = g5p_render_implicit(job, r->fence_context); ++ drm_dev_get(drm); ++ g5p_rsched_push(d, r, job); ++ mutex_unlock(&d->render_submit_lock); ++ job = NULL; ++ ret = 0; ++ goto out_job; ++unlock: ++ mutex_unlock(&f->lock); ++ mutex_unlock(&d->render_submit_lock); ++out_job: ++ if (signal) ++ drm_syncobj_put(signal); ++ dma_fence_chain_free(chain); ++ if (job) { ++ kfree(job->fence); /* only ever an uninitialized fence here */ ++ job->fence = NULL; ++ g5p_render_job_free(job); ++ } ++out: ++ kvfree(ranges); ++ kfree(upass); ++ kfree(syncs); ++ kfree(res); ++ return ret; ++} ++ ++/* recovery108: a submitter is inside the admission window for its whole call; once a ++ * FINAL teardown closed admission it is refused before it looks at anything. ++ */ ++static int g5p_render_submit(struct drm_device *drm, struct drm_file *file, ++ u32 context, u32 profile, u32 rt_handle, ++ u64 res_ptr, u32 res_count, u64 sync_ptr, ++ u32 sync_count, u64 pass_ptr, u32 pass_size, ++ u64 *job_id) ++{ ++ int ret; ++ ++ if (!g5p_r8_enter()) ++ return -ESHUTDOWN; ++ ret = g5p_render_submit_admitted(drm, file, context, profile, rt_handle, ++ res_ptr, res_count, sync_ptr, ++ sync_count, pass_ptr, pass_size, ++ job_id); ++ g5p_r8_exit(); ++ return ret; ++} ++ ++/* The layout the profile dispatch (pass_size selects the profile) and ++ * GET_RENDER_INFO rely on; a header edit that moves any of it fails here. ++ */ ++static_assert(sizeof(struct drm_g5p_render_pass) == 360); ++static_assert(offsetof(struct drm_g5p_render_pass, vdm_control) == 20); ++static_assert(sizeof(struct drm_g5p_render_pass_ext) == 128); ++static_assert(sizeof(struct drm_g5p_render_pass_v2) == 488); ++static_assert(sizeof(struct drm_g5p_render_pass_v3) == 536); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ext) == ++ sizeof(struct drm_g5p_render_pass)); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_va) == 488); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, ta_program_size) == 496); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, vdm_end) == 500); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, pixel_output_width) == ++ 504); ++static_assert(offsetof(struct drm_g5p_render_pass_v3, reserved) == 512); ++static_assert(sizeof(struct drm_g5p_submit_render_typed) == 80); ++static_assert(sizeof(struct drm_g5p_get_render_info) == 120); ++static_assert(offsetof(struct drm_g5p_get_render_info, hw_va_limit) == 104); ++static_assert(offsetof(struct drm_g5p_get_render_info, reserved1) == 112); ++static_assert(DRM_G5P_CAP_TYPED_RENDER_V3 == BIT_ULL(26)); ++static_assert(DRM_G5P_CAP_RENDER_MULTI_CONTEXT == BIT_ULL(27)); ++static_assert(DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH == BIT_ULL(29)); ++static_assert(DRM_G5P_RENDER_PROFILE_TYPED_V3 == 4); ++ ++static int g5p_submit_render_typed(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_submit_render_typed *a = data; ++ u64 id = 0; ++ int ret; ++ if (a->flags || a->pad || a->job_id || a->reserved[0] || ++ a->reserved[1] || a->profile != DRM_G5P_RENDER_PROFILE_TYPED_V3 || ++ !a->render_target || !a->resource_count || ++ a->resource_count > DRM_G5P_RENDER_MAX_RESOURCES || ++ !a->sync_count || a->sync_count > DRM_G5P_RENDER_MAX_SYNCS || ++ a->pass_size != sizeof(struct drm_g5p_render_pass_v3)) ++ return -EINVAL; ++ ret = g5p_render_submit(drm, file, a->context, ++ DRM_G5P_RENDER_PROFILE_TYPED_V3, ++ a->render_target, a->resources, ++ a->resource_count, a->syncs, a->sync_count, ++ a->pass, a->pass_size, &id); ++ a->job_id = id; ++ return ret; ++} ++ ++static void g5p_rt_geometry_to_uapi(struct drm_g5p_rt_geometry *u, ++ const struct g5p_ru_rt *k) ++{ ++ u->header_bytes = k->header; ++ u->region_bytes = k->region; ++ u->packed_state = k->packed; ++ u->state_word = k->state; ++ u->padded_dims = k->padded; ++ memcpy(u->scale, k->scale, sizeof(u->scale)); ++ memcpy(u->ta_state, k->ta_state, sizeof(u->ta_state)); ++ memcpy(u->ta_tiling, k->ta_tiling, sizeof(u->ta_tiling)); ++ u->ta_tiling_word = k->ta_tiling_word; ++ u->tile_count = k->tiles; ++ u->secondary_bytes = k->secondary; ++ u->secondary_align = k->secondary_align; ++} ++ ++static int g5p_create_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_create_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ const struct drm_g5p_rt_geometry *u = &a->geometry; ++ struct g5p_ru_rt geometry; ++ struct g5p_context *c; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ bool user = a->flags & DRM_G5P_RT_USER_GEOMETRY; ++ bool kernel = (1); ++ u64 positions = a->samples == 4 ? G5P_RU_POSITIONS_4X : ++ G5P_RU_POSITIONS_1X; ++ u32 id, blocks = 0; ++ int ret; ++ ++ ; ++ ; ++ if (a->flags & ~DRM_G5P_RT_USER_GEOMETRY || a->handle || a->pad || ++ a->main_va || a->main_bytes || a->secondary_va || ++ a->secondary_bytes || ++ (a->samples != 1 && ++ (a->samples != 4 || !g5p_render_msaa_enabled())) || ++ a->layers != 1 || a->pm_blocks || a->pad2 || a->reserved || ++ u->reserved[0] || u->reserved[1]) ++ return -EINVAL; ++ if (user) { ++ { ++ return -EOPNOTSUPP; ++ } ++ } else if (kernel) { ++ /* 6ca6d5c in integers; equal to the leaf for every size. */ ++ if (memchr_inv(u, 0, sizeof(*u)) || ++ g5p_ru_rt_derive_samples(a->width, a->height, a->samples, ++ positions, &geometry)) ++ return -EINVAL; ++ } else { ++ /* Kernel table: only the offline-executed 6ca6d5c result. */ ++ if (a->width != 64 || a->height != 64 || ++ memchr_inv(u, 0, sizeof(*u))) ++ return -EOPNOTSUPP; ++ geometry = g5p_ru_rt64; ++ } ++ mutex_lock(&f->lock); ++ c = xa_load(&f->contexts, a->context); ++ if (!c || !c->render) { ++ ret = c ? -EINVAL : -ENOENT; ++ goto out; ++ } ++ r = c->render; ++ ret = g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ /* No core lock: another context's job may hold it for its whole run, and nothing here ++ * touches the hardware. The device pool only grows; it serves every target of every ++ * context. ++ */ ++ if (kernel) { ++ mutex_lock(&d->rdev->lock); ++ ret = g5p_rdev_grow(d, g5p_ru_pm_blocks_for(geometry.tiles, ++ d->rdev->blocks)); ++ blocks = d->rdev->blocks; ++ mutex_unlock(&d->rdev->lock); ++ } ++ if (!ret) ++ ret = g5p_rt_create(d, r, &geometry, a->width, a->height, &rt); ++ if (ret) ++ goto out; ++ rt->samples = a->samples; ++ ret = xa_alloc(&f->targets, &id, rt, ++ XA_LIMIT(1, G5P_RU_MAX_TARGETS * 8), GFP_KERNEL); ++ if (ret) { ++ /* Mapped but unpublished to a job: leave it owned by the context. */ ++ rt->destroyed = true; ++ goto out; ++ } ++ kref_get(&c->ref); ++ a->handle = id; ++ a->main_va = rt->node.start; ++ a->main_bytes = rt->sizes.main; ++ a->secondary_va = rt->node.start + rt->sizes.main; ++ a->secondary_bytes = rt->sizes.secondary_map; ++ if (!user) ++ g5p_rt_geometry_to_uapi(&a->geometry, &geometry); ++ if (kernel) ++ a->pm_blocks = blocks; ++out: ++ mutex_unlock(&f->lock); ++ return ret; ++} ++ ++static int g5p_destroy_render_target(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_destroy_render_target *a = data; ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ struct g5p_context *c = NULL; ++ struct g5p_render *r; ++ struct g5p_rt *rt; ++ int ret; ++ ++ if (a->flags) ++ return -EINVAL; ++ mutex_lock(&f->lock); ++ rt = xa_load(&f->targets, a->handle); ++ if (!rt) { ++ ret = -ENOENT; ++ goto out; ++ } ++ c = rt->ctx; ++ r = c->render; ++ ret = r->detached ? 0 : g5p_render_vm_idle(c, false); ++ if (ret) ++ goto out; ++ g5p_vmlog_add(G5P_VMLOG_RT_DEL, c->vm, 0, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map, ++ rt->width << 16 | rt->height, rt->samples); ++ /* RAM only, no core lock: see g5p_create_render_target(). */ ++ if (r->bound) { ++ g5p_vm_clear(c->vm, drm->dev, rt->node.start, ++ rt->sizes.main + rt->sizes.secondary_map); ++ ret = g5p_render_invalidate(c); ++ if (ret) ++ goto out; ++ } ++ if (!r->detached) { ++ g5p_rt_release_memory(d, r, rt, true); ++ list_del(&rt->link); ++ r->target_count--; ++ if (r->profile1_rt == rt) ++ r->profile1_rt = NULL; ++ kfree(rt); ++ } else { ++ rt->destroyed = true; ++ } ++ xa_erase(&f->targets, a->handle); ++out: ++ mutex_unlock(&f->lock); ++ if (!ret && c) ++ g5p_context_put(c); ++ return ret; ++} ++ ++/* True when a context created now would latch a hypothesis knob that differs ++ * from its default: GET_RENDER_INFO has no context to report on. ++ */ ++static bool g5p_render_variant_active(void) ++{ ++ return (0) || (G5P_RU_DEF_CACHE_VARIANT) != G5P_RU_DEF_CACHE_VARIANT || ++ (G5P_RU_DEF_MMU_VARIANT) != G5P_RU_DEF_MMU_VARIANT || (0) || ++ READ_ONCE(render_user_wait_ms) != G5P_RU_DEF_WAIT_MS || (0) || ++ (0) || (0) || ++ (G5P_RU_DEF_PROFILE1_VDM_CONTROL) != ++ G5P_RU_DEF_PROFILE1_VDM_CONTROL || ++ (0) || (0) || (0) || (0); ++} ++ ++static int g5p_get_render_info(struct drm_device *drm, void *data, ++ struct drm_file *file) ++{ ++ struct drm_g5p_get_render_info *a = data; ++ ++ if (a->size != sizeof(*a) || a->flags || ++ memchr_inv(&a->profiles, 0, ++ sizeof(*a) - offsetof(struct drm_g5p_get_render_info, ++ profiles))) ++ return -EINVAL; ++ ; ++ ; ++ if (g5p_render_variant_active()) ++ a->flags = DRM_G5P_RENDER_INFO_VARIANT_ACTIVE; ++ a->profiles = BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ ; ++ ; ++ { ++ a->profiles |= BIT(DRM_G5P_RENDER_PROFILE_TYPED_V3); ++ }; ++ { ++ a->max_render_width = G5P_RU_MAX_DIM; ++ a->max_render_height = G5P_RU_MAX_DIM; ++ a->max_pm_blocks = G5P_RU_MAX_BLOCKS; ++ } ++ a->max_render_targets = G5P_RU_MAX_TARGETS; ++ a->private_va_start = G5P_RU_PRIVATE_VA; ++ a->private_va_end = G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE; ++ a->user_va_min = G5P_RU_USER_VA_MIN; ++ a->usc_base = G5P_RU_PDS_BASE; ++ a->pds_base = a->usc_base; ++ a->code_window = G5P_RU_CODE_WINDOW; ++ a->hw_va_limit = G5P_RU_HW_VA_LIMIT; ++ a->pm_blocks = (32); ++ a->pm_block_bytes = G5P_RU_BLOCK_BYTES; ++ a->max_queued_jobs = G5P_RU_MAX_QUEUED; ++ a->max_vdm_bytes = G5P_RU_MAX_VDM; ++ /* TYPED_V3: what a waiter must outlast is the job's absolute bound (TA publish to the end of the ++ * 3D, render-user-wait.h), not the per-phase module value: a TA that takes its whole phase leaves the ++ * 3D the rest of the job cap. ++ */ ++ { ++ if (READ_ONCE(render_user_hard_ms)) ++ a->job_timeout_ms = g5p_ru_lat_job_ms( ++ READ_ONCE(render_user_hard_ms), ++ READ_ONCE(render_user_obs_ms)); ++ else ++ a->job_timeout_ms = G5P_RU_V3_CAP_MS; ++ }; ++ a->retained_results = G5P_RU_RESULTS; ++ return 0; ++} ++ ++/* VM_BIND hooks for the context resident on a VM (serial compute or render). */ ++static int g5p_resident_idle(struct g5p_context *c, bool live) ++{ ++ return g5p_render_vm_idle(c, live); ++} ++ ++/* VM_BIND UNMAP, file lock held, PTEs already cleared and the tag published: a render ++ * context with queued jobs keeps the mapping's object (its pages and its pin) until those ++ * jobs have retired (g5p_render_reap()). False: free it now. ++ */ ++static bool g5p_resident_defer(struct g5p_context *c, struct g5p_mapping *m) ++{ ++ struct g5p_render *r = c->render; ++ unsigned long flags; ++ ++ if (!r || !atomic_read(&r->queued)) ++ return false; ++ m->retire = r->fence_seqno; ++ spin_lock_irqsave(&r->zlock, flags); ++ list_add_tail(&m->link, &r->zombies); ++ spin_unlock_irqrestore(&r->zlock, flags); ++ return true; ++} ++ ++static int g5p_resident_invalidate(struct g5p_context *c) ++{ ++ return g5p_render_invalidate(c); ++} +diff --git a/drivers/gpu/drm/apple-g5p/drm-submit.inc b/drivers/gpu/drm/apple-g5p/drm-submit.inc +new file mode 100644 +index 000000000..f36998e4d +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/drm-submit.inc +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++/* Asynchronous privileged probes; payloads and firmware VM are kernel-owned. */ ++struct g5p_fence { ++ struct dma_fence base; ++ /* Owned by dma_fence; independent of queue/status locks. */ ++ spinlock_t lock; ++ u32 state; ++ int error; ++}; ++ ++static const char *g5p_fence_driver_name(struct dma_fence *f) ++{ ++ return "apple_g5p"; ++} ++ ++static const char *g5p_fence_timeline_name(struct dma_fence *f) ++{ ++ return "g5p-probe"; ++} ++ ++/* Default release uses dma_fence_free()/RCU; base is the first member. */ ++static const struct dma_fence_ops g5p_fence_ops = { ++ .get_driver_name = g5p_fence_driver_name, ++ .get_timeline_name = g5p_fence_timeline_name, ++}; ++ ++/* recovery108: the compute/probe submitters share the render path's admission window. */ ++ ++/* Legacy JOB_COMPUTE remains unsupported: v1 lacks a bounded stream length. */ +diff --git a/drivers/gpu/drm/apple-g5p/render-user-validate.h b/drivers/gpu/drm/apple-g5p/render-user-validate.h +new file mode 100644 +index 000000000..a466367bb +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/render-user-validate.h +@@ -0,0 +1,384 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_VALIDATE_H ++#define G5P_RENDER_USER_VALIDATE_H ++ ++/* Pure TYPED_V1 checks over already-resolved resource ranges. The caller ++ * proves each range is inside a mapping of the listed BO, in the submitting ++ * context's VM, with BO roles covering `access`. Nothing here is a pointer ++ * the kernel dereferences; GPU addresses are only range-checked. ++ */ ++#define G5P_RU_ACCESS_READ 1U ++#define G5P_RU_ACCESS_WRITE 2U ++/* Software admission bound; TYPED_V3 never sends a stream length to firmware. ++ * vdm_size/end only bound the listed range and the four-byte tail read. ++ */ ++#define G5P_RU_MAX_VDM 0x100000U ++#define G5P_RU_MAX_VDM_CONTROL 0x10000U ++#define G5P_RU_MAX_SCRATCH (64ULL << 20) ++ ++/* Hardware facts GET_RENDER_INFO reports and the TYPED_V3 validator enforces; ++ * one definition so the two cannot drift. Code offsets are 32-bit from the ++ * common USC/PDS base. TA/3D reads at or above 2^38 stall silently on A10 ++ * (works at 0x3_, 0x10_ and 0x20_, stalls at 0x40_ and 0x80_, no MMU fault); ++ * not flown on A9, so the limit is a statement, not a proof (see ++ * docs/gpu/experiments/typed-v3-kernel-implementation.md). ++ */ ++#define G5P_RU_PDS_BASE 0x300000000ULL ++#define G5P_RU_CODE_WINDOW 0x100000000ULL /* 2^32 */ ++#define G5P_RU_HW_VA_LIMIT 0x4000000000ULL /* 2^38 */ ++ ++/* Not a requested access: the resolver sets it, for the typed profiles only, ++ * when the listed BO carries DRM_G5P_BO_EXEC. Only TYPED_V3 reads it; every ++ * other check masks on READ/WRITE. It lives in `access` rather than in a new ++ * field so positional g5p_ru_range initialisers keep compiling. ++ */ ++#define G5P_RU_ACCESS_EXEC 16U ++ ++struct g5p_ru_range { ++ u64 va, size; ++ u32 access; ++}; ++ ++/* [va, va + size) inside one listed range that grants every access bit. */ ++static inline bool g5p_ru_in(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 access) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if ((r[i].access & access) == access && va >= r[i].va && ++ va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) ++ return true; ++ return false; ++} ++ ++/* Everything TYPED_V1 and TYPED_V2 share: all fields except the pools. */ ++static inline int g5p_ru_pass_check_common(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || ++ !g5p_ru_in(r, n, p->vdm_va, p->vdm_size, G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ /* Copied to TA W88/R0+470; meaning unresolved (r23), so bounded. */ ++ if ((p->vdm_control & 0xf) || p->vdm_control > G5P_RU_MAX_VDM_CONTROL) ++ return -EINVAL; ++ if ((p->scratch_va & 0xf) || p->scratch_size < 0x10 || ++ p->scratch_size > G5P_RU_MAX_SCRATCH || ++ !g5p_ru_in(r, n, p->scratch_va, p->scratch_size, rw)) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((p->scratch_ptr[i] & 3) || ++ !g5p_ru_in(r, n, p->scratch_ptr[i], 4, rw)) ++ return -EINVAL; ++ /* Three-word descriptors keep their fourth slot zero. */ ++ if (p->ta_pds[3] || p->eot[3] || p->eot_resume[3]) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V1 and LINEAR64_V1: the two allocator pools are page-aligned RW. */ ++static inline int g5p_ru_pass_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret = g5p_ru_pass_check_common(p, r, n); ++ ++ if (ret) ++ return ret; ++ if (((p->pool7_va | p->pool7_size | p->pool8_va | p->pool8_size) & ++ (G5P_GPU_PAGE - 1)) || ++ !g5p_ru_in(r, n, p->pool7_va, p->pool7_size, rw) || ++ !g5p_ru_in(r, n, p->pool8_va, p->pool8_size, rw)) ++ return -EINVAL; ++ return 0; ++} ++ ++#define G5P_RU_MAX_TABLE 65536U ++ ++/* TYPED_V2: the pools are the ISP scissor (8-byte entries) and depth-bias ++ * (12-byte) tables, READ and word aligned; the V1 pool fields stay zero. ++ * Flagged ZLS words are addresses into listed RW ranges; the rest is opaque. ++ */ ++static inline int g5p_ru_pass2_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_range *r, u32 n) ++{ ++ u32 rw = G5P_RU_ACCESS_READ | G5P_RU_ACCESS_WRITE; ++ int ret; ++ ++ if (p->pool7_va || p->pool7_size || p->pool8_va || p->pool8_size) ++ return -EINVAL; ++ ret = g5p_ru_pass_check_common(p, r, n); ++ if (ret) ++ return ret; ++ if (!e->scissor_count || e->scissor_count > G5P_RU_MAX_TABLE || ++ (e->scissor_va & 3) || ++ !g5p_ru_in(r, n, e->scissor_va, e->scissor_count * 8ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if (!e->dbias_count || e->dbias_count > G5P_RU_MAX_TABLE || ++ (e->dbias_va & 3) || ++ !g5p_ru_in(r, n, e->dbias_va, e->dbias_count * 12ULL, ++ G5P_RU_ACCESS_READ)) ++ return -EINVAL; ++ if ((e->pixel_width != 2 && e->pixel_width != 4 && ++ e->pixel_width != 8) || ++ (e->bgobj_vals & ~0x3ffU) || (e->zls_mask & ~0x3fU)) ++ return -EINVAL; ++ if (!!e->occlusion_query_va != !!e->occlusion_query_size || ++ (e->occlusion_query_va && ++ ((e->occlusion_query_va & 0xf) || e->occlusion_query_size < sizeof(u32) || ++ !g5p_ru_in(r, n, e->occlusion_query_va, e->occlusion_query_size, rw)))) ++ return -EINVAL; ++ for (u32 i = 0; i < 6; i++) ++ if ((e->zls_mask & (1U << i)) && ++ ((e->zls[i] >> 40) || ++ !g5p_ru_in(r, n, e->zls[i] & ~0xfULL, 16, rw))) ++ return -EINVAL; ++ return 0; ++} ++ ++/* TYPED_V3: the TA PDS program (W+0x88 -> R0+0x470) is an explicit reference ++ * and the VDM stream length is a separate field that never reaches W. ++ * Contract and evidence: docs/gpu/experiments/typed-v3-kernel-implementation.md. ++ * Everything here is rejected, never masked. Structural rules need only the ++ * resolved ranges; the two content rules take bytes the caller read through ++ * the BOs' own pages (g5p_ru_pass3_content()). The rules are a subset of what ++ * the userspace lane accepts (tools/mesa-a9 a9c_ctx_lint(), a9scene): nothing ++ * here is stricter than a client that passes those lints. ++ * -EINVAL stale vdm_control, malformed program/stream/descriptor fields, ++ * no terminator, or (V1 dialect) the V1 rules fail ++ * -ERANGE outside [pds_base, pds_base + code_window) or past hw_va_limit ++ * -ENOENT no listed range wholly covers the program, stream or task code ++ * -EACCES missing READ, or the program/task code range's BO lacks EXEC ++ * -ENODATA the first 16 program bytes are all zero ++ */ ++#define G5P_RU_PROGRAM_ALIGN 16U ++#define G5P_RU_PROGRAM_MIN 16U ++ ++struct g5p_ru_v3_limits { ++ u64 pds_base, code_window; ++ u64 hw_va_limit; /* 0: no statement */ ++}; ++ ++/* Where the content rules read: indexes into the ranges array. */ ++struct g5p_ru_v3_loc { ++ u64 program_off; /* the exact W+0x88 value */ ++ u32 program_range, vdm_range; ++ u32 pixel_width; /* the pass's output width, 2, 4 or 8, either dialect */ ++ bool dialect2; /* ext was non-zero: V2 3D words; else the V1 dialect */ ++}; ++ ++/* TYPED_V3 carries the V2 ext struct. An all-zero ext selects the V1 dialect ++ * (pools in base, no tables, V1 3D words): the pass tools/mesa-a9 records ++ * today, plus the explicit program. Any non-zero ext field selects V2. ++ */ ++static inline bool g5p_ru_pass2_zero(const struct g5p_ru_pass2 *e) ++{ ++ u64 any = e->scissor_va | e->dbias_va | e->background_texture | ++ e->occlusion_query_va | e->occlusion_query_size; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(e->zls); i++) ++ any |= e->zls[i]; ++ any |= e->scissor_count | e->dbias_count | e->bgobj_depth | ++ e->bgobj_vals | e->pixel_width | e->zls_mask; ++ return !any; ++} ++ ++/* First listed range that wholly contains [va, va + size). */ ++static inline bool g5p_ru_cover(const struct g5p_ru_range *r, u32 n, u64 va, ++ u64 size, u32 *idx) ++{ ++ if (!size || va > ~0ULL - size) ++ return false; ++ for (u32 i = 0; i < n; i++) ++ if (va >= r[i].va && va - r[i].va <= r[i].size && ++ size <= r[i].size - (va - r[i].va)) { ++ *idx = i; ++ return true; ++ } ++ return false; ++} ++ ++/* [va, va + size) ends at or below the limit; zero limit makes no statement. */ ++static inline bool g5p_ru_below(u64 limit, u64 va, u64 size) ++{ ++ return !limit || (size <= limit && va <= limit - size); ++} ++ ++/* One PDS task reference of a store/resume descriptor: a 32-bit offset from ++ * pds_base, 16-byte aligned like every captured value and W+0x88's mask, ++ * non-zero, inside the code window and the hardware limit, and starting in a ++ * listed READ range (`code`: whose BO also carries EXEC). ++ */ ++static inline int g5p_ru_v3_ref(const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, u32 off, ++ bool code) ++{ ++ u64 va; ++ u32 i; ++ ++ if (!off || (off % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (lim->code_window < G5P_RU_PROGRAM_MIN || ++ off > lim->code_window - G5P_RU_PROGRAM_MIN || ++ lim->pds_base > ~0ULL - off) ++ return -ERANGE; ++ va = lim->pds_base + off; ++ if (!g5p_ru_below(lim->hw_va_limit, va, G5P_RU_PROGRAM_MIN)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, va, G5P_RU_PROGRAM_MIN, &i)) ++ return -ENOENT; ++ if (!(r[i].access & G5P_RU_ACCESS_READ) || ++ (code && !(r[i].access & G5P_RU_ACCESS_EXEC))) ++ return -EACCES; ++ return 0; ++} ++ ++/* The TYPED_V3 pixel output width. The V1 dialect takes it from the V3 field ++ * (0 = 2, the V1 dialect exactly), the V2 dialect from ext, where the V3 field ++ * must stay zero. A width above 2 is several colour targets, and a partial ++ * render must then resume with the pass's own EOT: a one-target resume EOT ++ * would store only the first target. The EOT itself is not decoded. ++ */ ++static inline int g5p_ru_pass3_width(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ bool dialect2, u32 *width) ++{ ++ u32 w = v->pixel_width; ++ ++ if (v->pad || (w && w != 2 && w != 4 && w != 8) || (dialect2 && w)) ++ return -EINVAL; ++ if (dialect2) ++ w = e->pixel_width; /* 2, 4 or 8: g5p_ru_pass2_check() */ ++ else if (!w) ++ w = 2; ++ if (w > 2 && memcmp(p->eot, p->eot_resume, sizeof(p->eot))) ++ return -EINVAL; ++ *width = w; ++ return 0; ++} ++ ++/* The V3 rules, then the TYPED_V1 rules (all-zero ext) or the TYPED_V2 rules ++ * (any ext field set; pools zero), then the output width. base.vdm_control is ++ * checked first. On success loc names the ranges to read, the dialect and the ++ * width. ++ */ ++static inline int g5p_ru_pass3_check(const struct g5p_ru_pass *p, ++ const struct g5p_ru_pass2 *e, ++ const struct g5p_ru_pass3 *v, ++ const struct g5p_ru_v3_limits *lim, ++ const struct g5p_ru_range *r, u32 n, ++ struct g5p_ru_v3_loc *loc) ++{ ++ u32 pi, vi; ++ u64 off; ++ int ret; ++ ++ if (p->vdm_control) ++ return -EINVAL; /* a stream length here is the r36/r88 defect */ ++ if (v->ta_program_size < G5P_RU_PROGRAM_MIN || ++ v->ta_program_size > DRM_G5P_RENDER_MAX_PROGRAM_BYTES || ++ (v->ta_program_size % G5P_RU_PROGRAM_ALIGN) || ++ (v->ta_program_va % G5P_RU_PROGRAM_ALIGN)) ++ return -EINVAL; ++ if (p->vdm_size < 4 || p->vdm_size > G5P_RU_MAX_VDM || ++ ((p->vdm_va | p->vdm_size) & 3) || v->vdm_end < 4 || ++ v->vdm_end > p->vdm_size || (v->vdm_end & 3)) ++ return -EINVAL; ++ if (v->ta_program_va < lim->pds_base) ++ return -ERANGE; ++ off = v->ta_program_va - lim->pds_base; ++ if (lim->code_window < v->ta_program_size || ++ off > lim->code_window - v->ta_program_size) ++ return -ERANGE; ++ if (!g5p_ru_below(lim->hw_va_limit, v->ta_program_va, ++ v->ta_program_size) || ++ !g5p_ru_below(lim->hw_va_limit, p->vdm_va, p->vdm_size)) ++ return -ERANGE; ++ /* Every listed range, not only the two above: on A10 a buffer moved past ++ * the limit stalls the TA without a fault, and all buffers moved together. ++ */ ++ for (u32 i = 0; i < n; i++) ++ if (!g5p_ru_below(lim->hw_va_limit, r[i].va, r[i].size)) ++ return -ERANGE; ++ if (!g5p_ru_cover(r, n, v->ta_program_va, v->ta_program_size, &pi)) ++ return -ENOENT; ++ if (!(r[pi].access & G5P_RU_ACCESS_READ) || ++ !(r[pi].access & G5P_RU_ACCESS_EXEC)) ++ return -EACCES; ++ if (!g5p_ru_cover(r, n, p->vdm_va, p->vdm_size, &vi)) ++ return -ENOENT; ++ if (!(r[vi].access & G5P_RU_ACCESS_READ)) ++ return -EACCES; ++ /* Context store [0] / load [1]: code at word 2 (READ and EXEC), STREAM_OUT2 ++ * at word 5 (READ). Resume tasks: word 2 (READ only: R0+0xf060/0xf070, ++ * named stream-out by the userspace lane and aux program by r92 2.4, so ++ * its role is unresolved and EXEC is not demanded). Words 0, 1, 3 and 4 ++ * stay opaque. The spacing (store - program = 0x280, resume - store = ++ * 0x340) and a program that equals a task entry (r63 flew the store ++ * program; the role is not detectable structurally) are not enforced. ++ */ ++ for (u32 i = 0; i < 2; i++) { ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][2], true); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->context_pds[i][5], ++ false); ++ if (!ret) ++ ret = g5p_ru_v3_ref(lim, r, n, p->resume_pds[i][2], ++ false); ++ if (ret) ++ return ret; ++ } ++ loc->dialect2 = !g5p_ru_pass2_zero(e); ++ ret = loc->dialect2 ? g5p_ru_pass2_check(p, e, r, n) : ++ g5p_ru_pass_check(p, r, n); ++ if (!ret) ++ ret = g5p_ru_pass3_width(p, e, v, loc->dialect2, ++ &loc->pixel_width); ++ if (ret) ++ return ret; ++ loc->program_off = off; ++ loc->program_range = pi; ++ loc->vdm_range = vi; ++ return 0; ++} ++ ++/* The first 16 program bytes are not all zero (r88 published 0x370, which ++ * pointed at zeros) and the last stream dword is the terminator. `head` is ++ * G5P_RU_PROGRAM_MIN bytes, `tail` the 4 stream bytes at vdm_va + vdm_end - 4. ++ */ ++static inline int g5p_ru_pass3_content(const u8 *head, const u8 *tail) ++{ ++ u8 any = 0; ++ u32 word = tail[0] | tail[1] << 8 | tail[2] << 16 | (u32)tail[3] << 24; ++ ++ for (u32 i = 0; i < G5P_RU_PROGRAM_MIN; i++) ++ any |= head[i]; ++ if (!any) ++ return -ENODATA; ++ if (word != DRM_G5P_RENDER_STREAM_TERMINATE) ++ return -EINVAL; ++ return 0; ++} ++ ++/* AGXRenderTargetMemoryG5 output sanity for userspace geometry: exact one ++ * sample and the sizes the kernel will allocate. Opaque fields unchecked. ++ */ ++static inline int g5p_ru_geometry_check(const struct g5p_ru_rt *rt, u32 width, ++ u32 height, ++ struct g5p_ru_rt_sizes *sizes) ++{ ++ if (!width || !height || width > 16384 || height > 16384) ++ return -EINVAL; ++ if (rt->tiles > ((width + 31) / 32) * ((height + 31) / 32)) ++ return -EINVAL; ++ return g5p_ru_rt_sizes(rt, sizes); ++} ++ ++#endif +diff --git a/drivers/gpu/drm/apple-g5p/runtime-idle.inc b/drivers/gpu/drm/apple-g5p/runtime-idle.inc +new file mode 100644 +index 000000000..a83d99ad3 +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/runtime-idle.inc +@@ -0,0 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++static int g5p_idle_open(struct g5p_drm *d) ++{ ++ atomic_inc(&d->open_files); ++ return 0; ++} ++ ++static void g5p_idle_work(struct work_struct *work) ++{ ++} ++ ++static void g5p_idle_queue(struct g5p_drm *d) ++{ ++} diff --git a/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch b/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch new file mode 100644 index 00000000..b26eba8a --- /dev/null +++ b/kernel/patches/drivers/drm/0014-drm-expose-the-native-GEM-exporter-CPU-cache-contrac.patch @@ -0,0 +1,147 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:11:46 +0000 +Subject: [PATCH] drm: expose the native GEM exporter CPU cache contract + +[Orchard: from brkosebay/linux 78775ea3b9e; retain Orchard ADBE +implementation, adding its exporter contract only.] + +G5P PRIME imports require an immutable native cache mapping contract. +Unknown and foreign exporters stay conservative. + +Assisted-by: LLM +--- + drivers/gpu/drm/adbe/adbe_drv.c | 25 +++++++++++++++++++++++++ + drivers/gpu/drm/drm_prime.c | 19 +++++++++++++++++++ + include/drm/drm_gem.h | 8 ++++++++ + include/drm/drm_gem_dma_helper.h | 8 ++++++++ + include/drm/drm_prime.h | 2 ++ + 5 files changed, 62 insertions(+) + +diff --git a/drivers/gpu/drm/adbe/adbe_drv.c b/drivers/gpu/drm/adbe/adbe_drv.c +index 2508e6531..24c76fef9 100644 +--- a/drivers/gpu/drm/adbe/adbe_drv.c ++++ b/drivers/gpu/drm/adbe/adbe_drv.c +@@ -653,11 +653,36 @@ static int adbe_dumb_create(struct drm_file *file, struct drm_device *drm, + return drm_gem_dma_dumb_create_internal(file, drm, args); + } + ++/* Native allocations: the DMA helper's defaults plus the uncached-export ++ * contract. Imported objects never make that promise. ++ */ ++static const struct drm_gem_object_funcs adbe_gem_funcs = { ++ .free = drm_gem_dma_object_free, ++ .print_info = drm_gem_dma_object_print_info, ++ .get_sg_table = drm_gem_dma_object_get_sg_table, ++ .vmap = drm_gem_dma_object_vmap, ++ .mmap = drm_gem_dma_object_mmap, ++ .vm_ops = &drm_gem_dma_vm_ops, ++ .is_cpu_uncached = drm_gem_dma_object_is_cpu_uncached, ++}; ++ ++static struct drm_gem_object *adbe_gem_create_object(struct drm_device *drm, ++ size_t size) ++{ ++ struct drm_gem_dma_object *dma_obj = kzalloc_obj(*dma_obj); ++ ++ if (!dma_obj) ++ return ERR_PTR(-ENOMEM); ++ dma_obj->base.funcs = &adbe_gem_funcs; ++ return &dma_obj->base; ++} ++ + DEFINE_DRM_GEM_DMA_FOPS(adbe_fops); + + static const struct drm_driver adbe_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .fops = &adbe_fops, ++ .gem_create_object = adbe_gem_create_object, + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(adbe_dumb_create), + DRM_FBDEV_DMA_DRIVER_OPS, + .name = "adbe", +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index 9b44c78cd..70110173d 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -835,6 +835,25 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { + .vunmap = drm_gem_dmabuf_vunmap, + }; + ++/** ++ * drm_gem_dma_buf_is_cpu_uncached - query a trusted native GEM exporter ++ * @dma_buf: exported buffer whose supported CPU aliases are queried ++ * ++ * Returns true only for the explicit native exporter contract. Non-GEM, ++ * re-exported foreign objects and unspecified exporters remain conservative. ++ */ ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf) ++{ ++ struct drm_gem_object *obj; ++ ++ if (!dma_buf || dma_buf->ops != &drm_gem_prime_dmabuf_ops) ++ return false; ++ obj = dma_buf->priv; ++ return !drm_gem_is_imported(obj) && obj->funcs && ++ obj->funcs->is_cpu_uncached && obj->funcs->is_cpu_uncached(obj); ++} ++EXPORT_SYMBOL_GPL(drm_gem_dma_buf_is_cpu_uncached); ++ + /** + * drm_prime_pages_to_sg - converts a page array into an sg list + * @dev: DRM device +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 885244e37..048193377 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -73,6 +73,14 @@ enum drm_gem_object_status { + * struct drm_gem_object_funcs - GEM object functions + */ + struct drm_gem_object_funcs { ++ /** ++ * @is_cpu_uncached: Optional exporter contract. True only when every ++ * supported CPU mapping (kernel and userspace) is uncached or WC and ++ * no cacheable alias exists. Native allocation properties must be ++ * immutable; imported memory must not inherit this promise. Default ++ * NULL is conservative. This does not remove DMA ordering barriers. ++ */ ++ bool (*is_cpu_uncached)(struct drm_gem_object *obj); + /** + * @free: + * +diff --git a/include/drm/drm_gem_dma_helper.h b/include/drm/drm_gem_dma_helper.h +index f2678e7ec..481007c13 100644 +--- a/include/drm/drm_gem_dma_helper.h ++++ b/include/drm/drm_gem_dma_helper.h +@@ -4,6 +4,7 @@ + + #include + #include ++#include + #include + + struct drm_mode_create_dumb; +@@ -55,6 +56,13 @@ extern const struct vm_operations_struct drm_gem_dma_vm_ops; + * This function wraps drm_gem_dma_free_object(). Drivers that employ the DMA helpers + * should use it as their &drm_gem_object_funcs.free handler. + */ ++static inline bool drm_gem_dma_object_is_cpu_uncached(struct drm_gem_object *obj) ++{ ++ return !drm_gem_is_imported(obj) && ++ !to_drm_gem_dma_obj(obj)->map_noncoherent && ++ !dev_is_dma_coherent(drm_dev_dma_dev(obj->dev)); ++} ++ + static inline void drm_gem_dma_object_free(struct drm_gem_object *obj) + { + struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj); +diff --git a/include/drm/drm_prime.h b/include/drm/drm_prime.h +index f50f862f0..5de957f7d 100644 +--- a/include/drm/drm_prime.h ++++ b/include/drm/drm_prime.h +@@ -116,4 +116,6 @@ int drm_prime_sg_to_page_array(struct sg_table *sgt, struct page **pages, + int drm_prime_sg_to_dma_addr_array(struct sg_table *sgt, dma_addr_t *addrs, + int max_pages); + ++bool drm_gem_dma_buf_is_cpu_uncached(struct dma_buf *dma_buf); ++ + #endif /* __DRM_PRIME_H__ */ diff --git a/kernel/patches/drivers/drm/0015-drm-apple-g5p-account-per-job-GPU-and-host-durations.patch b/kernel/patches/drivers/drm/0015-drm-apple-g5p-account-per-job-GPU-and-host-durations.patch new file mode 100644 index 00000000..a49b9b63 --- /dev/null +++ b/kernel/patches/drivers/drm/0015-drm-apple-g5p-account-per-job-GPU-and-host-durations.patch @@ -0,0 +1,411 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:46 +0000 +Subject: [PATCH] drm/apple-g5p: account per-job GPU and host durations + +[Orchard: from brkosebay/linux 9102dab8891e83f71f36ce9aa0d49b79849ab8de; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/compute-user-layout.h | 1 + + drivers/gpu/apple-g5p/render-user-jobtime.h | 52 +++++++++ + drivers/gpu/apple-g5p/render-user-layout.h | 2 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 116 +++++++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 5 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 37 +++++- + drivers/gpu/drm/apple-g5p/drm-render.inc | 22 ++++ + 7 files changed, 234 insertions(+), 1 deletion(-) + create mode 100644 drivers/gpu/apple-g5p/render-user-jobtime.h + +diff --git a/drivers/gpu/apple-g5p/compute-user-layout.h b/drivers/gpu/apple-g5p/compute-user-layout.h +index 78b459832..bf8382fc8 100644 +--- a/drivers/gpu/apple-g5p/compute-user-layout.h ++++ b/drivers/gpu/apple-g5p/compute-user-layout.h +@@ -142,6 +142,7 @@ static inline void g5p_cu_job(u8 *p, u32 meta, u32 global, u32 slot, u32 seq, + u8 *w = p + G5P_CU_W; + + memset(w, 0, G5P_CU_W_BYTES); ++ g5p_put64(w + 0x278, 0); + g5p_put32(w, G5P_CU_TAG); + g5p_put32(w + 8, slot); + g5p_put32(w + 0xc, meta + G5P_CU_DEP); +diff --git a/drivers/gpu/apple-g5p/render-user-jobtime.h b/drivers/gpu/apple-g5p/render-user-jobtime.h +new file mode 100644 +index 000000000..39fd70488 +--- /dev/null ++++ b/drivers/gpu/apple-g5p/render-user-jobtime.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef G5P_RENDER_USER_JOBTIME_H ++#define G5P_RENDER_USER_JOBTIME_H ++#define G5P_JT_RING 256U ++#define G5P_JT_TICKS_PER_US 24U ++struct g5p_jt { ++ u64 id; ++ u32 ctx, seq; ++ s32 ret; ++ u32 flags; ++ u64 submit_ns, accept_ns, exec_ns; ++ u64 ta_pub_ns, ta_ev_ns, ta_done_ns; ++ u64 td_pub_ns, td_ev_ns, td_done_ns, signal_ns; ++ u64 ta_ticks, td_ticks; ++ u32 ta_samples, td_samples; ++ u64 touch_ns, bell_ns; ++}; ++struct g5p_jt_stats { ++ u64 jobs, failed, joint, ta_ticks, td_ticks; ++ u64 ioctl_ns, queue_ns, prep_ns, ta_wait_ns, gap_ns, td_wait_ns; ++ u64 tail_ns, touch_ns, bell_ns, samples; ++ u64 ta_irq_ns, td_irq_ns, ta_decide_ns, td_decide_ns, since_ns; ++}; ++static inline u64 g5p_jt_delta(u64 a, u64 b) ++{ ++ return a && b && b >= a ? b - a : 0; ++} ++static inline void g5p_jt_fold(struct g5p_jt_stats *s, const struct g5p_jt *j) ++{ ++ s->jobs++; ++ s->failed += j->ret != 0; ++ s->joint += !!(j->flags & 4); ++ if (j->ta_ticks != ~0ULL) ++ s->ta_ticks += j->ta_ticks; ++ if (j->td_ticks != ~0ULL) ++ s->td_ticks += j->td_ticks; ++ s->ioctl_ns += g5p_jt_delta(j->submit_ns, j->accept_ns); ++ s->queue_ns += g5p_jt_delta(j->accept_ns, j->exec_ns); ++ s->prep_ns += g5p_jt_delta(j->exec_ns, j->ta_pub_ns); ++ s->ta_wait_ns += g5p_jt_delta(j->ta_pub_ns, j->ta_done_ns); ++ s->gap_ns += g5p_jt_delta(j->ta_done_ns, j->td_pub_ns); ++ s->td_wait_ns += g5p_jt_delta(j->td_pub_ns, j->td_done_ns); ++ s->tail_ns += g5p_jt_delta(j->td_done_ns, j->signal_ns); ++ s->ta_irq_ns += g5p_jt_delta(j->ta_pub_ns, j->ta_ev_ns); ++ s->td_irq_ns += g5p_jt_delta(j->td_pub_ns, j->td_ev_ns); ++ s->ta_decide_ns += g5p_jt_delta(j->ta_ev_ns, j->ta_done_ns); ++ s->td_decide_ns += g5p_jt_delta(j->td_ev_ns, j->td_done_ns); ++ s->touch_ns += j->touch_ns; ++ s->bell_ns += j->bell_ns; ++ s->samples += (u64)j->ta_samples + j->td_samples; ++} ++#endif +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +index 822709ed8..77f704102 100644 +--- a/drivers/gpu/apple-g5p/render-user-layout.h ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -478,6 +478,8 @@ static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, + u32 cursor = sku, starts, finishes; + + memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); ++ g5p_put32(work + (td ? 0x8ac : 0x550), 0); ++ g5p_put32(work + (td ? 0x8b0 : 0x554), 0); + memset(p + sku, 0, td ? 0x180 : 0xc0); + g5p_put32(work, td ? 1 : 0); + g5p_put32(work + 4, slot); +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index f2fb4bf2d..cf4248bc4 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -15,6 +15,8 @@ + #include + #include + #include ++#include ++#include "../../apple-g5p/render-user-jobtime.h" + #include + #include + #include +@@ -106,11 +108,19 @@ struct g5p_file { + u64 job_id; + }; + ++static bool g5p_jt_log; ++module_param_named(job_time_log, g5p_jt_log, bool, 0600); ++ + struct g5p_drm { + struct drm_device drm; + struct g5p_diag *core; + /* Protects cached IRQ event/status fields; never held for hardware IO. */ + spinlock_t status_lock; ++ struct g5p_jt jt_ring[G5P_JT_RING]; ++ u32 jt_head, jt_count; ++ struct g5p_jt_stats jt_stats; ++ /* Core lock: only samples within the current execute are charged. */ ++ struct g5p_jt *jt_active; + u32 last_event[11], notifications[4], first_fault[11]; + u64 event_serial, fault_serial; + struct delayed_work idle_work; +@@ -166,6 +176,109 @@ static inline struct g5p_drm *to_g5p(struct drm_device *drm) + return container_of(drm, struct g5p_drm, drm); + } + ++static s64 g5p_jt_us(u64 a, u64 b) ++{ ++ return a && b ? div_s64((s64)b - (s64)a, 1000) : -1; ++} ++static void g5p_jt_line(char *buf, size_t size, const struct g5p_jt *j) ++{ ++ scnprintf( ++ buf, size, ++ "G5P-JT job=%llu ctx=%u seq=%u ret=%d f=%x sub=%llu q=%llu prep=%llu ta=%lld/%lld/%lld gap=%lld td=%lld/%lld/%lld tail=%llu total=%llu gta_us=%lld gtd_us=%lld ns=%u/%u touch_us=%llu bell_us=%llu\n", ++ j->id, j->ctx, j->seq, j->ret, j->flags, ++ g5p_jt_delta(j->submit_ns, j->accept_ns) / 1000, ++ g5p_jt_delta(j->accept_ns, j->exec_ns) / 1000, ++ g5p_jt_delta(j->exec_ns, j->ta_pub_ns) / 1000, ++ g5p_jt_us(j->ta_pub_ns, j->ta_ev_ns), ++ g5p_jt_us(j->ta_ev_ns, j->ta_done_ns), ++ g5p_jt_us(j->ta_pub_ns, j->ta_done_ns), ++ g5p_jt_us(j->ta_done_ns, j->td_pub_ns), ++ g5p_jt_us(j->td_pub_ns, j->td_ev_ns), ++ g5p_jt_us(j->td_ev_ns, j->td_done_ns), ++ g5p_jt_us(j->td_pub_ns, j->td_done_ns), ++ g5p_jt_delta(j->td_done_ns, j->signal_ns) / 1000, ++ g5p_jt_delta(j->submit_ns, j->signal_ns) / 1000, ++ j->ta_ticks == ~0ULL ? -1LL : (s64)(j->ta_ticks / 24), ++ j->td_ticks == ~0ULL ? -1LL : (s64)(j->td_ticks / 24), ++ j->ta_samples, j->td_samples, j->touch_ns / 1000, ++ j->bell_ns / 1000); ++} ++static int g5p_jobtimes_show(struct seq_file *m, void *unused) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct g5p_drm *d = to_g5p(entry->dev); ++ struct g5p_jt *snapshot; ++ unsigned long flags; ++ u32 count, head; ++ char line[768]; ++ ++ snapshot = kvmalloc_array(G5P_JT_RING, sizeof(*snapshot), GFP_KERNEL); ++ if (!snapshot) ++ return -ENOMEM; ++ spin_lock_irqsave(&d->status_lock, flags); ++ count = d->jt_count; ++ head = d->jt_head; ++ memcpy(snapshot, d->jt_ring, sizeof(d->jt_ring)); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ for (u32 i = 0; i < count; i++) { ++ g5p_jt_line(line, sizeof(line), ++ &snapshot[(head - count + i) % G5P_JT_RING]); ++ seq_puts(m, line); ++ } ++ kvfree(snapshot); ++ return 0; ++} ++static int g5p_gpu_stats_show(struct seq_file *m, void *unused) ++{ ++ struct g5p_drm *d = m->private; ++ struct g5p_jt_stats s; ++ unsigned long flags; ++ u64 now; ++ spin_lock_irqsave(&d->status_lock, flags); ++ s = d->jt_stats; ++ now = ktime_get_ns(); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ seq_printf( ++ m, ++ "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu\n", ++ now, s.since_ns, s.jobs, s.failed, s.ta_ticks, s.td_ticks, ++ s.ioctl_ns, s.queue_ns, s.prep_ns, s.ta_wait_ns, s.gap_ns, ++ s.td_wait_ns, s.tail_ns, s.touch_ns, s.bell_ns, s.samples, ++ s.ta_irq_ns, s.td_irq_ns, s.ta_decide_ns, s.td_decide_ns, ++ s.joint); ++ return 0; ++} ++static int g5p_gpu_stats_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, g5p_gpu_stats_show, inode->i_private); ++} ++static ssize_t g5p_gpu_stats_write(struct file *file, const char __user *buf, ++ size_t len, loff_t *pos) ++{ ++ struct seq_file *m = file->private_data; ++ struct g5p_drm *d = m->private; ++ unsigned long flags; ++ spin_lock_irqsave(&d->status_lock, flags); ++ memset(&d->jt_stats, 0, sizeof(d->jt_stats)); ++ d->jt_stats.since_ns = ktime_get_ns(); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ return len; ++} ++static const struct file_operations g5p_gpu_stats_fops = { ++ .owner = THIS_MODULE, ++ .open = g5p_gpu_stats_open, ++ .read = seq_read, ++ .write = g5p_gpu_stats_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++static void g5p_jt_debugfs_init(struct drm_minor *minor) ++{ ++ struct g5p_drm *d = to_g5p(minor->dev); ++ debugfs_create_file("g5p_gpu_stats", 0600, d->drm.debugfs_root, d, ++ &g5p_gpu_stats_fops); ++} ++ + static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) + { + return container_of(obj, struct g5p_bo, shmem.base); +@@ -535,6 +648,7 @@ static const struct drm_ioctl_desc g5p_ioctls[] = { + DEFINE_DRM_GEM_FOPS(g5p_drm_fops); + + static const struct drm_driver g5p_drm_driver = { ++ .debugfs_init = g5p_jt_debugfs_init, + .driver_features = DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | + DRIVER_SYNCOBJ_TIMELINE, + .open = g5p_drm_open, +@@ -644,6 +758,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + d->core = g; + d->fence_context = dma_fence_context_alloc(1); + spin_lock_init(&d->status_lock); ++ d->jt_stats.since_ns = ktime_get_ns(); + init_waitqueue_head(&d->ev_wq); + atomic_set(&d->accepted, 0); + atomic64_set(&d->job_sequence, 0); +@@ -665,6 +780,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + atomic_set(&d->open_files, 0); + atomic_set(&d->r8_queued, 0); + drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); ++ drm_debugfs_add_file(&d->drm, "g5p_jobtimes", g5p_jobtimes_show, NULL); + drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); + INIT_DELAYED_WORK(&d->idle_work, g5p_idle_work); + ret = drm_dev_register(&d->drm, 0); +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +index dcbe34c0b..559546358 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -393,6 +393,11 @@ out: + if (ret < 0 && ret != -ECANCELED && v->touched && !v->lost && + !g5p_ru_term_failed(&g5p_rv3_term)) + g5p_rv3_terminal(g, g5p_ru_term_from_ret(ret), "cdm"); ++ if (!ret) { ++ u8 *p = g5p_cu_page(b) + G5P_CU_W + 0x278; ++ dma_rmb(); ++ job->jt.ta_ticks = g5p_le32(p) | ((u64)g5p_le32(p + 4) << 32); ++ } + g5p_cu_receipt(g, r, job, ret); + return ret; + } +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 1d0a0a76d..0fd86d8e4 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -317,7 +317,18 @@ static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, + *pm = g5p_rv3_pm_cpu(b, r); + u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); + +- g5p_rv3_fw_touch(b); ++ { ++ struct g5p_jt *jt = r->ctx->gpu->jt_active; ++ u64 start = ktime_get_ns(); ++ g5p_rv3_fw_touch(b); ++ if (jt) { ++ jt->touch_ns += ktime_get_ns() - start; ++ if (e) ++ jt->td_samples++; ++ else ++ jt->ta_samples++; ++ } ++ } + memset(s, 0, sizeof(*s)); + dma_rmb(); + s->ns = ktime_get_ns(); +@@ -2501,6 +2512,8 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + int err = -EIO, ret, why, loss; + u32 seq, res; + ++ job->jt.exec_ns = ktime_get_ns(); ++ job->jt.flags = (first ? 1 : 0) | (!bound_before ? 2 : 0); + if (job->compute) + return g5p_cu_execute(d, r, job); + if (r->stage_done) +@@ -2746,6 +2759,28 @@ out: + v->ta_event_ns = g5p_rv3_event_disarm(d, 0); + if (v->phase >= G5P_RV3_TD_PUBLISH) + v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ job->jt.ta_pub_ns = v->ta_pub_ns; ++ job->jt.td_pub_ns = v->td_pub_ns; ++ job->jt.ta_ev_ns = v->ta_event_ns; ++ job->jt.td_ev_ns = v->td_event_ns; ++ job->jt.ta_done_ns = v->ta.end_ns; ++ job->jt.td_done_ns = v->td.end_ns; ++ for (u32 e = 0; e < 2; e++) ++ job->jt.bell_ns += ++ g5p_jt_delta(v->wk_start_ns[e], v->wk_done_ns[e]); ++ if (!ret) { ++ dma_rmb(); ++ for (u32 e = 0; e < 2; e++) { ++ u8 *p = g5p_ru_page(b, e) + ++ g5p_ru_w_at(g5p_rv3_js(rd, seq)) + ++ (e ? 0x8ac : 0x550); ++ u64 ticks = g5p_le32(p) | ((u64)g5p_le32(p + 4) << 32); ++ if (e) ++ job->jt.td_ticks = ticks; ++ else ++ job->jt.ta_ticks = ticks; ++ } ++ } + g5p_rv3_wake_end(d, r, v); + /* Any other failure after the hardware was touched (attach, publish, drain, protocol): the same + * criterion g5p_render_run() quarantines the context by. A stage stop (-ECANCELED) is not one. +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +index bd86d7dff..851a6a115 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -358,6 +358,7 @@ struct g5p_rjob { + struct g5p_cu_regs cu; + struct g5p_rv3 rv3; + u64 submit_ns; /* latency123: the submit ioctl */ ++ struct g5p_jt jt; + }; + + static u8 *g5p_ru_page(struct g5p_boot *b, u32 engine) +@@ -1027,7 +1028,9 @@ static void g5p_render_run(struct g5p_rjob *job) + */ + g5p_render_cache(d, job, false); + WRITE_ONCE(d->rdev->executing, true); ++ d->jt_active = &job->jt; + ret = g5p_rv3_execute(d, r, job); ++ d->jt_active = NULL; + /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ + smp_store_release(&d->rdev->executing, false); + hardware = job->rv3.touched || job->rv3.bind_failed; +@@ -1072,6 +1075,20 @@ static void g5p_render_run(struct g5p_rjob *job) + atomic_dec(&r->queued); + atomic_dec(&d->r8_queued); + dma_fence_signal(&job->fence->base); ++ job->jt.signal_ns = ktime_get_ns(); ++ job->jt.ret = ret; ++ job->jt.seq = job->rv3.seq; ++ spin_lock_irqsave(&d->status_lock, flags); ++ d->jt_ring[d->jt_head++ % G5P_JT_RING] = job->jt; ++ if (d->jt_count < G5P_JT_RING) ++ d->jt_count++; ++ g5p_jt_fold(&d->jt_stats, &job->jt); ++ spin_unlock_irqrestore(&d->status_lock, flags); ++ if (g5p_jt_log) { ++ char line[768]; ++ g5p_jt_line(line, sizeof(line), &job->jt); ++ dev_info(g->dev, "%s", line); ++ } + /* A job that may still run on the hardware (a bad context) keeps what it might use. */ + if (!READ_ONCE(job->ctx->bad)) + g5p_render_reap(r, job->fence->base.seqno, false); +@@ -2045,6 +2062,8 @@ static int g5p_render_submit_admitted(struct drm_device *drm, + job = kzalloc(sizeof(*job), GFP_KERNEL); + if (job) { + job->submit_ns = ktime_get_ns(); ++ job->jt.submit_ns = job->submit_ns; ++ job->jt.ta_ticks = job->jt.td_ticks = ~0ULL; + job->objs = kvcalloc(res_count, sizeof(*job->objs), GFP_KERNEL); + } + if (!ranges || !job || !job->objs) { +@@ -2196,6 +2215,9 @@ static int g5p_render_submit_admitted(struct drm_device *drm, + job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); + job->setup_error = g5p_render_implicit(job, r->fence_context); + drm_dev_get(drm); ++ job->jt.id = job->id; ++ job->jt.ctx = context; ++ job->jt.accept_ns = ktime_get_ns(); + g5p_rsched_push(d, r, job); + mutex_unlock(&d->render_submit_lock); + job = NULL; diff --git a/kernel/patches/drivers/drm/0016-drm-apple-g5p-reduce-serial-render-job-overhead.patch b/kernel/patches/drivers/drm/0016-drm-apple-g5p-reduce-serial-render-job-overhead.patch new file mode 100644 index 00000000..ab02642c --- /dev/null +++ b/kernel/patches/drivers/drm/0016-drm-apple-g5p-reduce-serial-render-job-overhead.patch @@ -0,0 +1,379 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:46 +0000 +Subject: [PATCH] drm/apple-g5p: reduce serial render job overhead + +[Orchard: from brkosebay/linux 9a0a283e51dabf45af303fbae67a43568b2681bc; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-evwait.h | 2 +- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 2 +- + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 2 +- + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 2 +- + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 1 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 155 ++++++++++++++----- + 6 files changed, 122 insertions(+), 42 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +index ccb7d2c8f..5a14b32db 100644 +--- a/drivers/gpu/apple-g5p/render-user-evwait.h ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -17,7 +17,7 @@ + #define G5P_RENDER_USER_EVWAIT_H + + #define G5P_EW_POST_US \ +- 100U /* after the phase's own event: completion is imminent */ ++ 50U /* after the phase's own event: completion is imminent */ + #define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ + + enum g5p_ew_act { +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index cf4248bc4..46231b477 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -762,7 +762,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + init_waitqueue_head(&d->ev_wq); + atomic_set(&d->accepted, 0); + atomic64_set(&d->job_sequence, 0); +- d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", 0); ++ d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", WQ_HIGHPRI); + if (IS_ERR_OR_NULL(d->render_wq)) { + ret = d->render_wq ? PTR_ERR(d->render_wq) : -ENOMEM; + goto err_pm; +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +index 559546358..d09e56c37 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -60,7 +60,7 @@ static void g5p_cu_sample(struct g5p_diag *g, u32 seq, u32 target, + u8 *p = g5p_cu_page(b), *global = g5p_ru_global(b, G5P_CU_ENGINE); + u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); + +- g5p_rv3_fw_touch(b); ++ g5p_rv3_fw_touch(b, NULL); + memset(s, 0, sizeof(*s)); + dma_rmb(); + s->ns = ktime_get_ns(); +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +index 018e83a5b..1d3c42192 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-latency123.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -55,7 +55,7 @@ MODULE_PARM_DESC( + render_user_event_wake, + "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " + "until the next firmware event (the IRQ thread's decode wakes the waiter), at " +- "most N us (100..20000) as the safety net, and poll every 100 us once the " ++ "most N us (100..20000) as the safety net, and poll every 50 us once the " + "phase's own completion event is latched; read live"); + + /* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +index ac4dd841e..22a63accf 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.h ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -48,6 +48,7 @@ struct g5p_rv3 { + u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], + wk_t1_pub[2], wk_t1_end[2]; + u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool joint; /* latched at execute entry; attach stays serial */ + bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ + /* The bind of this job's context published its DC3 pair and then failed: that context is + * lost, the device and the contexts already bound are not. +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 0fd86d8e4..761b3015f 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -249,7 +249,10 @@ static int g5p_rv3_errno(struct g5p_diag *g, int ret) + * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU + * has no sampler, so it costs nothing then. + */ +-static void g5p_rv3_fw_touch(struct g5p_boot *b) ++static const bool render_user_joint; ++static const unsigned int render_user_fw_ready_mode = 2; ++ ++static void g5p_rv3_fw_touch(struct g5p_boot *b, const struct g5p_render *render) + { + struct g5p_dma_page *const regions[] = { b->root, b->shared }; + u32 counts[] = { ARRAY_SIZE(b->root), ARRAY_SIZE(b->shared) }; +@@ -259,10 +262,13 @@ static void g5p_rv3_fw_touch(struct g5p_boot *b) + for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) + (void)READ_ONCE( + *(const u32 *)(regions[r][i].cpu + at)); +- /* perf60 stage 4: the per-context PM sets are firmware pages like the shared ones */ +- for (u32 i = 0; i < b->pm_sets * G5P_PMSET_PAGES; i++) +- for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) +- (void)READ_ONCE(*(const u32 *)(b->pm_set[i].cpu + at)); ++ /* DC callers retain the full sweep. Root and shared are always whole. */ ++ for (u32 set = 0; set < b->pm_sets; set++) { ++ for (u32 i = 0; i < G5P_PMSET_PAGES; i++) ++ for (u32 at = 0; at < G5P_GPU_PAGE; at += 64) ++ (void)READ_ONCE(*(const u32 *)( ++ b->pm_set[set * G5P_PMSET_PAGES + i].cpu + at)); ++ } + } + + /* perf60 bell gates, A/B knobs for the multi-context 3D fault on the bell path (render_user_wake=2 only; +@@ -309,7 +315,7 @@ static u32 g5p_rv3_js(const struct g5p_rdev *rd, u32 seq) + + + /* RAM only. */ +-static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++static void g5p_rv3_sample_ram(struct g5p_diag *g, struct g5p_render *r, u32 e, + u32 seq, struct g5p_rv3_ev *s) + { + struct g5p_boot *b = g->boot; +@@ -317,18 +323,6 @@ static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, + *pm = g5p_rv3_pm_cpu(b, r); + u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); + +- { +- struct g5p_jt *jt = r->ctx->gpu->jt_active; +- u64 start = ktime_get_ns(); +- g5p_rv3_fw_touch(b); +- if (jt) { +- jt->touch_ns += ktime_get_ns() - start; +- if (e) +- jt->td_samples++; +- else +- jt->ta_samples++; +- } +- } + memset(s, 0, sizeof(*s)); + dma_rmb(); + s->ns = ktime_get_ns(); +@@ -356,6 +350,25 @@ static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, + READ_ONCE(r->ctx->gpu->ta_hit_ns)); + } + ++/* Full qualified visibility sweep for fallback, observation and DC-independent callers. */ ++static void g5p_rv3_sample(struct g5p_diag *g, struct g5p_render *r, u32 e, ++ u32 seq, struct g5p_rv3_ev *s) ++{ ++ struct g5p_jt *jt = r->ctx->gpu->jt_active; ++ u64 start = ktime_get_ns(); ++ ++ g5p_rv3_fw_touch(g->boot, r); ++ if (jt) ++ jt->touch_ns += ktime_get_ns() - start; ++ g5p_rv3_sample_ram(g, r, e, seq, s); ++ if (jt) { ++ if (e) ++ jt->td_samples++; ++ else ++ jt->ta_samples++; ++ } ++} ++ + /* TA: one step of render-user-retire.h's TA rules when ta is given, else the flown r74b + * predicate. 3D: one step of its 3D rules when rt is given, else the r96 predicate. + */ +@@ -373,6 +386,52 @@ static bool g5p_rv3_done(u32 e, const struct g5p_rv3_ev *s, + return s->consumed && s->idle_ok && s->retired && (e || s->pm_ok); + } + ++/* Test on copies: probing must not advance the real coherent-run clocks, ++ * count a second sample, or change the authoritative decision state. A matching ++ * event is mandatory even where the legacy idle-clear/hold rule could finish. ++ */ ++static bool g5p_rv3_probe_done(u32 e, const struct g5p_rv3_ev *s, ++ const struct g5p_ru_retire *rt, const struct g5p_ru_ta *ta) ++{ ++ struct g5p_ru_retire rt_copy; ++ struct g5p_ru_ta ta_copy; ++ ++ if (!s->event || s->failed || s->proto) ++ return false; ++ if (rt) ++ rt_copy = *rt; ++ if (ta) ++ ta_copy = *ta; ++ return g5p_rv3_done(e, s, rt ? &rt_copy : NULL, ta ? &ta_copy : NULL); ++} ++ ++/* This changes only scan timing; live-PM coverage is controlled separately. ++ * Pending/stale/invalid samples always run the selected sweep and resample. TA retains every sweep to keep its ++ * PM hand-off unchanged. Even a pre-completed 3D retains its first full sweep. ++ */ ++static void g5p_rv3_wait_sample(struct g5p_diag *g, struct g5p_render *r, ++ u32 e, u32 seq, struct g5p_rv3_ev *s, ++ const struct g5p_ru_retire *rt, const struct g5p_ru_ta *ta, ++ bool sampled, bool drain_failed) ++{ ++ u32 mode = READ_ONCE(render_user_fw_ready_mode); ++ bool ready = false; ++ ++ if ((mode == 1 || mode == 2) && !drain_failed) { ++ g5p_rv3_sample_ram(g, r, e, seq, s); ++ ready = g5p_rv3_probe_done(e, s, rt, ta); ++ ++ if (mode == 2 && e == 1 && sampled && ready) { ++ struct g5p_jt *jt = r->ctx->gpu->jt_active; ++ ++ if (jt) ++ jt->td_samples++; ++ return; ++ } ++ } ++ g5p_rv3_sample(g, r, e, seq, s); ++} ++ + /* The TA clauses in the bits of render-user-retire.h, for the receipt. */ + static u32 g5p_rv3_ta_unsat(const struct g5p_rv3_ev *s) + { +@@ -513,7 +572,7 @@ static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, + struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) + { + struct g5p_drm *d = r->ctx->gpu; +- bool closing = false, slow = false; ++ bool closing = false, slow = false, sampled = false; + + for (;;) { + int drain = g5p_boot_drain(g); +@@ -521,7 +580,8 @@ static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, + d->event_serial); /* after the drain, before the sample */ + bool fault, silent, done; + +- g5p_rv3_sample(g, r, e, seq, s); ++ g5p_rv3_wait_sample(g, r, e, seq, s, rt, ta, sampled, drain != 0); ++ sampled = true; + /* latency101: a silent event stream during the phase is a dead firmware (never off). */ + silent = + g5p_ru_wait_grace(w, s->ns, READ_ONCE(g->boot->events)); +@@ -978,7 +1038,7 @@ static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) + return ret; + if (READ_ONCE(b->failed)) + return -EIO; +- g5p_rv3_fw_touch(b); ++ g5p_rv3_fw_touch(b, NULL); + dma_rmb(); + switch (g5p_ru_bind_step(READ_ONCE(ctl[0]), READ_ONCE(ctl[4]), + target, expired)) { +@@ -1401,11 +1461,12 @@ static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, + * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) + */ + static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, +- const u32 *ring, u32 start, u32 previous, u32 seq) ++ const u32 *ring, u32 start, u32 previous, u32 seq, bool joint) + { + const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); + +- return (u32)!r->retired[0] | (u32)!r->consumed[0] << 1 | ++ return (u32)(!joint && !r->retired[0]) | ++ (u32)(!joint && !r->consumed[0]) << 1 | + (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | + (u32)(READ_ONCE(ring[0]) != start) << 4 | + (u32)(g5p_le32(p + G5P_WORK_CTL) != previous) << 5 | +@@ -1419,7 +1480,7 @@ static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, + << 10; + } + +-/* The 3D after the TA retired: record, WORK_CTL, producer, and NO doorbell. */ ++/* 3D publication: tag4 orders it after TA, including joint publication. */ + static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, + struct g5p_rv3 *v, u32 seq) + { +@@ -1433,7 +1494,7 @@ static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, + /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one + * fails, and the refusal names the clauses and the raw values it read. + */ +- why = g5p_rv3_td_why(r, p, ring, start, previous, seq); ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq, v->joint); + if (why) { + dev_emerg( + g->dev, +@@ -1615,12 +1676,12 @@ static void g5p_rv3_receipt(struct g5p_diag *g, struct g5p_render *r, + return; + g5p_rv3_receipt_printk( + g, ret, +- "RENDER-V3 end seq=%u ret=%d phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " ++ "RENDER-V3 end seq=%u ret=%d joint=%u phase=%s ta=%s/%lld td=%s/%lld bound_ms=%u " + "cap_ms=%u ta_consumed_ms=%lld td_consumed_ms=%lld late_gap=%u/%u " + "late_after=%u/%u grace=%u bells=%d ta_ring=%u/%u td_ring=%u/%u " + "ta_stamp=%08x/%08x td_stamp=%08x/%08x pm80=%08x r0_after_ta_publish=bell-only " + "end=v3\n", +- v->seq, ret, ++ v->seq, ret, v->joint, + g5p_rv3_phase_names[min_t(u32, v->phase, G5P_RV3_PHASES - 1)], + g5p_rv3_results[v->ta.result], + v->ta.end_ns ? +@@ -2513,7 +2574,9 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + u32 seq, res; + + job->jt.exec_ns = ktime_get_ns(); +- job->jt.flags = (first ? 1 : 0) | (!bound_before ? 2 : 0); ++ v->joint = !job->compute && !first && READ_ONCE(render_user_joint); ++ job->jt.flags = (first ? 1 : 0) | (!bound_before ? 2 : 0) | ++ (v->joint ? BIT(2) : 0); + if (job->compute) + return g5p_cu_execute(d, r, job); + if (r->stage_done) +@@ -2613,10 +2676,22 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + r->jobs++; + v->phase = G5P_RV3_TA_PUBLISH; + g5p_rv3_event_arm(d, 0, 0); ++ if (v->joint) ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); + g5p_rv3_wake_mark(d, r, v, 0); ++ if (v->joint) ++ g5p_rv3_wake_mark(d, r, v, 1); + ret = g5p_rv3_publish_ta(g, r, v, seq); + if (ret) + goto out; ++ if (v->joint) { ++ v->phase = G5P_RV3_TD_PUBLISH; ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample(g, r, 1, seq, &v->last[1]); ++ goto out; ++ } ++ } + v->phase = G5P_RV3_TA_WAIT; + if (r->stall_job && r->jobs == r->stall_job && !first) { + /* render_user_test_stall_job: the host gives up, the firmware runs on */ +@@ -2683,15 +2758,17 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, + g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); + ; +- v->phase = G5P_RV3_TD_PUBLISH; +- g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); +- g5p_rv3_wake_mark(d, r, v, 1); +- ret = g5p_rv3_publish_3d(g, r, v, seq); +- if (ret) { +- g5p_rv3_sample( +- g, r, 1, seq, +- &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ +- goto out; ++ if (!v->joint) { ++ v->phase = G5P_RV3_TD_PUBLISH; ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) { ++ g5p_rv3_sample( ++ g, r, 1, seq, ++ &v->last[1]); /* RAM only: the receipt prints the real 3D ring and stamps */ ++ goto out; ++ } + } + v->phase = G5P_RV3_TD_WAIT; + g5p_rv3_wait_start(g, r, &v->td, v->td_pub_ns, v->job_deadline_ns); +@@ -2757,7 +2834,7 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + out: + if (v->phase >= G5P_RV3_TA_PUBLISH) + v->ta_event_ns = g5p_rv3_event_disarm(d, 0); +- if (v->phase >= G5P_RV3_TD_PUBLISH) ++ if (v->joint || v->phase >= G5P_RV3_TD_PUBLISH) + v->td_event_ns = g5p_rv3_event_disarm(d, 1); + job->jt.ta_pub_ns = v->ta_pub_ns; + job->jt.td_pub_ns = v->td_pub_ns; +@@ -2872,6 +2949,8 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + if (act != G5P_RU_DRAIN_PUBLISH_TD) + break; + r->retired[0] = true; ++ if (r->submitted[1]) ++ continue; + g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); + armed = true; + if (g5p_rv3_publish_3d(g, r, v, s->seq)) { diff --git a/kernel/patches/drivers/drm/0017-drm-apple-g5p-allow-killable-VM-binds-and-tune-post-.patch b/kernel/patches/drivers/drm/0017-drm-apple-g5p-allow-killable-VM-binds-and-tune-post-.patch new file mode 100644 index 00000000..ef9e0d71 --- /dev/null +++ b/kernel/patches/drivers/drm/0017-drm-apple-g5p-allow-killable-VM-binds-and-tune-post-.patch @@ -0,0 +1,147 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:46 +0000 +Subject: [PATCH] drm/apple-g5p: allow killable VM binds and tune post-event + polling + +[Orchard: from brkosebay/linux 9f3f6272edeaea964aa638a6c8bac8cee516a72c; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-evwait.h | 12 +++++------- + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 3 ++- + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 14 +++++++++----- + drivers/gpu/drm/apple-g5p/drm-vm.inc | 8 ++++++-- + 4 files changed, 22 insertions(+), 15 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-evwait.h b/drivers/gpu/apple-g5p/render-user-evwait.h +index 5a14b32db..dc4a8d5c6 100644 +--- a/drivers/gpu/apple-g5p/render-user-evwait.h ++++ b/drivers/gpu/apple-g5p/render-user-evwait.h +@@ -11,22 +11,20 @@ + * without an event; the predicate stays level-triggered on every sample, as before. + * Once the phase's own completion event is latched (s->event) but the RAM rules still wait + * (idle words, retire stamp, PM receipt land a little after the event), the pause is a short +- * G5P_EW_POST_US poll: there is no further event to wait for. ++ * post_us poll: there is no further event to wait for. + */ + #ifndef G5P_RENDER_USER_EVWAIT_H + #define G5P_RENDER_USER_EVWAIT_H + +-#define G5P_EW_POST_US \ +- 50U /* after the phase's own event: completion is imminent */ + #define G5P_EW_MAX_US 20000U /* safety timeout ceiling */ + + enum g5p_ew_act { + G5P_EW_LEGACY, /* render_user_event_wake 0: usleep_range(1000, 2000), release8 */ +- G5P_EW_SHORT, /* the phase's event seen: sleep G5P_EW_POST_US */ ++ G5P_EW_SHORT, /* the phase's event seen: sleep post_us */ + G5P_EW_EVENT, /* sleep until any event since `seen`, at most *wait_us */ + }; + +-static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, ++static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, u32 post_us, bool phase_event, + u32 *wait_us) + { + if (!knob_us) { +@@ -34,10 +32,10 @@ static inline enum g5p_ew_act g5p_ew_plan(u32 knob_us, bool phase_event, + return G5P_EW_LEGACY; + } + if (phase_event) { +- *wait_us = G5P_EW_POST_US; ++ *wait_us = post_us; + return G5P_EW_SHORT; + } +- *wait_us = knob_us < G5P_EW_POST_US ? G5P_EW_POST_US : ++ *wait_us = knob_us < post_us ? post_us : + knob_us > G5P_EW_MAX_US ? G5P_EW_MAX_US : + knob_us; + return G5P_EW_EVENT; +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +index d09e56c37..2873ca73c 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -95,7 +95,8 @@ static void g5p_cu_pause(struct g5p_drm *d, u64 seen) + { + u32 wait_us; + +- if (g5p_ew_plan(READ_ONCE(render_user_event_wake), false, &wait_us) == ++ if (g5p_ew_plan(READ_ONCE(render_user_event_wake), 50U, false, ++ &wait_us) == + G5P_EW_LEGACY) { + usleep_range(1000, 2000); + return; +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +index 1d3c42192..a08b0d422 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-latency123.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -45,7 +45,7 @@ static struct { + u32 pmgr[2]; + bool seen[2], idle[2], probed; + /* render_user_event_wake: pauses ended by an event, the timeout, a post-event poll, legacy */ +- u32 ew[4]; ++ u32 ew[4], post_us; + u64 ew_ns; + } g5p_l123; + +@@ -55,8 +55,11 @@ MODULE_PARM_DESC( + render_user_event_wake, + "perf60 default0 (release8: sleep 1-2 ms between completion samples); N: sleep " + "until the next firmware event (the IRQ thread's decode wakes the waiter), at " +- "most N us (100..20000) as the safety net, and poll every 50 us once the " +- "phase's own completion event is latched; read live"); ++ "most N us (post-event interval..20000) as the safety net, and poll at " ++ "render_user_post_event_us once the phase's own completion event is latched; read live"); ++ ++/* Sample once per job so a live update cannot change an in-flight wait. */ ++static const unsigned int render_user_post_event_us = 50; + + /* A word of a firmware object; objects may straddle the region's 16 KiB pages. */ + static u8 *g5p_l123_word(struct g5p_boot *b, u32 slot, u32 field) +@@ -135,6 +138,7 @@ static void g5p_l123_job_reset(struct g5p_diag *g) + g5p_l123.probed = false; + memset(g5p_l123.ew, 0, sizeof(g5p_l123.ew)); + g5p_l123.ew_ns = 0; ++ g5p_l123.post_us = clamp(READ_ONCE(render_user_post_event_us), 10U, 100U); + g5p_l123.seen[0] = false; + g5p_l123.seen[1] = false; + g5p_l123.us = (0); +@@ -256,8 +260,8 @@ static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, + g5p_l123_evict(g->boot); + g5p_l123.next_ns = now + (u64)g5p_l123.us * NSEC_PER_USEC; + } +- act = g5p_ew_plan(READ_ONCE(render_user_event_wake), s->event, +- &wait_us); ++ act = g5p_ew_plan(READ_ONCE(render_user_event_wake), g5p_l123.post_us, ++ s->event, &wait_us); + t0 = ktime_get_ns(); + if (act == G5P_EW_LEGACY) { + usleep_range(1000, 2000); +diff --git a/drivers/gpu/drm/apple-g5p/drm-vm.inc b/drivers/gpu/drm/apple-g5p/drm-vm.inc +index 644554156..b244922d5 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-vm.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-vm.inc +@@ -768,7 +768,8 @@ static int g5p_vm_bind(struct drm_device *drm, void *data, + (a->op != DRM_G5P_VM_MAP && a->op != DRM_G5P_VM_UNMAP) || + (a->op == DRM_G5P_VM_UNMAP && (a->bo || a->offset))) + return -EINVAL; +- mutex_lock(&f->lock); ++ if (mutex_lock_killable(&f->lock)) ++ return -EINTR; + vm = xa_load(&f->vms, a->vm); + if (!vm) + goto out; +@@ -799,7 +800,10 @@ static int g5p_vm_bind(struct drm_device *drm, void *data, + * compute still uses the global mutex. + */ + if (!vm->resident->render) { +- mutex_lock(&to_g5p(drm)->core->lock); ++ if (mutex_lock_killable(&to_g5p(drm)->core->lock)) { ++ ret = -EINTR; ++ goto out; ++ } + locked = true; + } + resident = true; diff --git a/kernel/patches/drivers/drm/0018-drm-apple-g5p-account-joint-latency-and-capture-RAM-.patch b/kernel/patches/drivers/drm/0018-drm-apple-g5p-account-joint-latency-and-capture-RAM-.patch new file mode 100644 index 00000000..06e53b56 --- /dev/null +++ b/kernel/patches/drivers/drm/0018-drm-apple-g5p-account-joint-latency-and-capture-RAM-.patch @@ -0,0 +1,538 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:46 +0000 +Subject: [PATCH] drm/apple-g5p: account joint latency and capture RAM phase + stamps + +[Orchard: from brkosebay/linux 851c6df2e4eb0822ea041e53543ba7ffcdf7f60c; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/boot.inc | 12 +- + drivers/gpu/apple-g5p/render-user-jobtime.h | 28 ++++- + drivers/gpu/apple-g5p/render-user-layout.h | 11 +- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 118 ++++++++++++++++++-- + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 2 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 41 +++++-- + drivers/gpu/drm/apple-g5p/drm-render.inc | 4 +- + 7 files changed, 193 insertions(+), 23 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +index 6a62f2eb0..13267238b 100644 +--- a/drivers/gpu/apple-g5p/boot.inc ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -145,6 +145,8 @@ struct g5p_dma_page { + }; + + struct g5p_boot { ++ u64 hard_cyc; ++ struct task_struct *irq_task; + u32 r27_variant, r27_mode_writes, r46_reset, r50_mode, r50_writes; + + bool table_clean, preinstall_context, compute_boot_vm; +@@ -865,6 +867,13 @@ out: + return ret; + } + ++static irqreturn_t g5p_boot_hardirq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ WRITE_ONCE(g->boot->hard_cyc, arch_timer_read_counter()); ++ return IRQ_WAKE_THREAD; ++} ++ + static irqreturn_t g5p_boot_irq(int irq, void *data) + { + struct g5p_diag *g = data; +@@ -872,6 +881,7 @@ static irqreturn_t g5p_boot_irq(int irq, void *data) + u32 n = atomic_inc_return(&b->irqs); + int ret = 0; + ++ WRITE_ONCE(b->irq_task, current); + if (b->r73b) + return g5p_r73b_irq(g, irq, n); + if (n > g5p_transport_event_limit()) { +@@ -911,7 +921,7 @@ static int g5p_boot_irq_setup(struct g5p_diag *g) + b->irq = platform_get_irq(to_platform_device(g->dev), 4); + if (b->irq < 0) + return b->irq; +- ret = devm_request_threaded_irq(g->dev, b->irq, NULL, g5p_boot_irq, ++ ret = devm_request_threaded_irq(g->dev, b->irq, g5p_boot_hardirq, g5p_boot_irq, + IRQF_ONESHOT, "g5p-meta", g); + if (ret) + return ret; +diff --git a/drivers/gpu/apple-g5p/render-user-jobtime.h b/drivers/gpu/apple-g5p/render-user-jobtime.h +index 39fd70488..7eacc4b40 100644 +--- a/drivers/gpu/apple-g5p/render-user-jobtime.h ++++ b/drivers/gpu/apple-g5p/render-user-jobtime.h +@@ -14,19 +14,43 @@ struct g5p_jt { + u64 ta_ticks, td_ticks; + u32 ta_samples, td_samples; + u64 touch_ns, bell_ns; ++ u64 abs[2][2]; ++ u32 wp[2][2]; ++ u64 bell_cyc[2], hit_cyc[2], hard_cyc[2], exec_cyc, signal_cyc; ++ u64 prev_signal_cyc; ++ u8 src[2]; ++ bool abs_enabled; + }; + struct g5p_jt_stats { + u64 jobs, failed, joint, ta_ticks, td_ticks; + u64 ioctl_ns, queue_ns, prep_ns, ta_wait_ns, gap_ns, td_wait_ns; + u64 tail_ns, touch_ns, bell_ns, samples; ++ s64 ovh_ns; + u64 ta_irq_ns, td_irq_ns, ta_decide_ns, td_decide_ns, since_ns; + }; + static inline u64 g5p_jt_delta(u64 a, u64 b) + { + return a && b && b >= a ? b - a : 0; + } ++/* Joint 3D waiting starts only once both publish and TA completion occurred. */ ++static inline u64 g5p_jt_td_start(const struct g5p_jt *j) ++{ ++ return j->flags & 4 ? max(j->td_pub_ns, j->ta_ev_ns) : j->td_pub_ns; ++} ++static inline bool g5p_jt_has_ovh(const struct g5p_jt *j) ++{ ++ return j->ta_pub_ns && j->td_ev_ns && ++ j->ta_ticks != ~0ULL && j->td_ticks != ~0ULL; ++} ++static inline s64 g5p_jt_ovh_ns(const struct g5p_jt *j) ++{ ++ return (s64)j->td_ev_ns - (s64)j->ta_pub_ns - ++ div_s64((s64)(j->ta_ticks + j->td_ticks) * 1000, 24); ++} + static inline void g5p_jt_fold(struct g5p_jt_stats *s, const struct g5p_jt *j) + { ++ if (g5p_jt_has_ovh(j)) ++ s->ovh_ns += g5p_jt_ovh_ns(j); + s->jobs++; + s->failed += j->ret != 0; + s->joint += !!(j->flags & 4); +@@ -39,10 +63,10 @@ static inline void g5p_jt_fold(struct g5p_jt_stats *s, const struct g5p_jt *j) + s->prep_ns += g5p_jt_delta(j->exec_ns, j->ta_pub_ns); + s->ta_wait_ns += g5p_jt_delta(j->ta_pub_ns, j->ta_done_ns); + s->gap_ns += g5p_jt_delta(j->ta_done_ns, j->td_pub_ns); +- s->td_wait_ns += g5p_jt_delta(j->td_pub_ns, j->td_done_ns); ++ s->td_wait_ns += g5p_jt_delta(g5p_jt_td_start(j), j->td_done_ns); + s->tail_ns += g5p_jt_delta(j->td_done_ns, j->signal_ns); + s->ta_irq_ns += g5p_jt_delta(j->ta_pub_ns, j->ta_ev_ns); +- s->td_irq_ns += g5p_jt_delta(j->td_pub_ns, j->td_ev_ns); ++ s->td_irq_ns += g5p_jt_delta(g5p_jt_td_start(j), j->td_ev_ns); + s->ta_decide_ns += g5p_jt_delta(j->ta_ev_ns, j->ta_done_ns); + s->td_decide_ns += g5p_jt_delta(j->td_ev_ns, j->td_done_ns); + s->touch_ns += j->touch_ns; +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +index 77f704102..624a56a11 100644 +--- a/drivers/gpu/apple-g5p/render-user-layout.h ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -438,6 +438,7 @@ static inline u32 g5p_ru_endpoint(u32 seq) + * control, pointer ring, TQ, stamps and dependency object stay shared. Slot 1 clears TQ/STAMP/DEP + * (0x1800..0x1b70) and the r25 holder (G5P_R25_HEAD, 0x3000). + */ ++#define G5P_RU_ABS 0x2800U + #define G5P_RU_SLOT1 0x1c00U + #define G5P_RU_SLOT1_END \ + (G5P_RU_SLOT1 + G5P_RU_CMD_BYTES + G5P_RU_3D_W_BYTES + 0x180U) +@@ -468,7 +469,7 @@ static inline u32 g5p_ru_sku_at(u32 js, bool td) + static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, + bool td, u32 seq, const u8 *image, + u32 ta_finish, bool zero_timestamps, u32 js, +- u32 pmid) ++ u32 pmid, bool abs) + { + u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; + u32 finish = meta + G5P_R22_STAMP + 4; +@@ -480,6 +481,14 @@ static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, + memcpy(work, image, td ? G5P_RU_3D_W_BYTES : G5P_RU_TA_W_BYTES); + g5p_put32(work + (td ? 0x8ac : 0x550), 0); + g5p_put32(work + (td ? 0x8b0 : 0x554), 0); ++ BUILD_BUG_ON(G5P_RU_SLOT1_END > G5P_RU_ABS); ++ BUILD_BUG_ON(G5P_RU_ABS + 32 > G5P_R25_HEAD); ++ if (abs) { ++ g5p_put32(work + (td ? 0x8b4 : 0x558), meta + G5P_RU_ABS + 16 * js); ++ g5p_put32(work + (td ? 0x8b8 : 0x55c), meta + G5P_RU_ABS + 16 * js + 8); ++ g5p_put64(p + G5P_RU_ABS + 16 * js, 0); ++ g5p_put64(p + G5P_RU_ABS + 16 * js + 8, 0); ++ } + memset(p + sku, 0, td ? 0x180 : 0xc0); + g5p_put32(work, td ? 1 : 0); + g5p_put32(work + 4, slot); +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 46231b477..1e285cdd5 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include "../../apple-g5p/render-user-jobtime.h" + #include +@@ -118,6 +119,7 @@ struct g5p_drm { + spinlock_t status_lock; + struct g5p_jt jt_ring[G5P_JT_RING]; + u32 jt_head, jt_count; ++ u64 jt_last_signal_cyc; + struct g5p_jt_stats jt_stats; + /* Core lock: only samples within the current execute are charged. */ + struct g5p_jt *jt_active; +@@ -134,10 +136,12 @@ struct g5p_drm { + * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. + */ + u32 td_want; +- u64 td_hit_ns; ++ u64 td_hit_ns, td_hit_cyc, td_hard_cyc; ++ u8 td_src; + /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ + bool ta_want; +- u64 ta_hit_ns; ++ u64 ta_hit_ns, ta_hit_cyc, ta_hard_cyc; ++ u8 ta_src; + /* wake110: type-1 completion events decoded per engine (word 1 bit 0 TA, bit 1 3D) since + * load, and the decode time of the first one since the executor last cleared it. + */ +@@ -180,11 +184,44 @@ static s64 g5p_jt_us(u64 a, u64 b) + { + return a && b ? div_s64((s64)b - (s64)a, 1000) : -1; + } ++static s64 g5p_jt_cyc_us(u64 a, u64 b) ++{ ++ return a && b ? div_s64((s64)b - (s64)a, 24) : -1; ++} ++static void g5p_jt_abs_line(char *buf, size_t size, const struct g5p_jt *j) ++{ ++ u64 ta0 = j->abs[0][0], ta1 = j->abs[0][1]; ++ u64 td0 = j->abs[1][0], td1 = j->abs[1][1]; ++ size_t n = strlen(buf); ++ ++ /* ts1 may chase the pointer word already cleared by ts0. No retry. */ ++ if (!ta1 && ta0 && j->ta_ticks != ~0ULL) ++ ta1 = ta0 + j->ta_ticks; ++ if (!td1 && td0 && j->td_ticks != ~0ULL) ++ td1 = td0 + j->td_ticks; ++ if (j->abs_enabled) ++ n += scnprintf(buf + n, size - n, ++ " abs=1 P=%lld TA=%lld X=%lld G=%lld TD=%lld Y=%lld", ++ g5p_jt_cyc_us(j->bell_cyc[0], ta0), g5p_jt_cyc_us(ta0, ta1), ++ g5p_jt_cyc_us(ta1, j->hit_cyc[0]), ++ g5p_jt_cyc_us(j->flags & 4 ? ta1 : j->bell_cyc[1], td0), ++ g5p_jt_cyc_us(td0, td1), g5p_jt_cyc_us(td1, j->hit_cyc[1])); ++ n += scnprintf(buf + n, size - n, " irqth=%lld/%lld src=%u/%u pgap=%lld", ++ j->src[0] == 1 ? g5p_jt_cyc_us(j->hard_cyc[0], j->hit_cyc[0]) : -1LL, ++ j->src[1] == 1 ? g5p_jt_cyc_us(j->hard_cyc[1], j->hit_cyc[1]) : -1LL, ++ j->src[0], j->src[1], g5p_jt_cyc_us(j->prev_signal_cyc, j->exec_cyc)); ++ if (j->abs_enabled) ++ n += scnprintf(buf + n, size - n, ++ " wp=%08x,%08x/%08x,%08x raw=%llx,%llx,%llx,%llx", ++ j->wp[0][0], j->wp[0][1], j->wp[1][0], j->wp[1][1], ++ j->bell_cyc[0], ta0, ta1, j->hit_cyc[0]); ++ scnprintf(buf + n, size - n, "\n"); ++} + static void g5p_jt_line(char *buf, size_t size, const struct g5p_jt *j) + { + scnprintf( + buf, size, +- "G5P-JT job=%llu ctx=%u seq=%u ret=%d f=%x sub=%llu q=%llu prep=%llu ta=%lld/%lld/%lld gap=%lld td=%lld/%lld/%lld tail=%llu total=%llu gta_us=%lld gtd_us=%lld ns=%u/%u touch_us=%llu bell_us=%llu\n", ++ "G5P-JT job=%llu ctx=%u seq=%u ret=%d f=%x sub=%llu q=%llu prep=%llu ta=%lld/%lld/%lld gap=%lld td=%lld/%lld/%lld tail=%llu total=%llu gta_us=%lld gtd_us=%lld ns=%u/%u touch_us=%llu bell_us=%llu ovh=%lld", + j->id, j->ctx, j->seq, j->ret, j->flags, + g5p_jt_delta(j->submit_ns, j->accept_ns) / 1000, + g5p_jt_delta(j->accept_ns, j->exec_ns) / 1000, +@@ -193,15 +230,18 @@ static void g5p_jt_line(char *buf, size_t size, const struct g5p_jt *j) + g5p_jt_us(j->ta_ev_ns, j->ta_done_ns), + g5p_jt_us(j->ta_pub_ns, j->ta_done_ns), + g5p_jt_us(j->ta_done_ns, j->td_pub_ns), +- g5p_jt_us(j->td_pub_ns, j->td_ev_ns), ++ g5p_jt_us(g5p_jt_td_start(j), j->td_ev_ns), + g5p_jt_us(j->td_ev_ns, j->td_done_ns), +- g5p_jt_us(j->td_pub_ns, j->td_done_ns), ++ g5p_jt_us(g5p_jt_td_start(j), j->td_done_ns), + g5p_jt_delta(j->td_done_ns, j->signal_ns) / 1000, + g5p_jt_delta(j->submit_ns, j->signal_ns) / 1000, + j->ta_ticks == ~0ULL ? -1LL : (s64)(j->ta_ticks / 24), + j->td_ticks == ~0ULL ? -1LL : (s64)(j->td_ticks / 24), + j->ta_samples, j->td_samples, j->touch_ns / 1000, +- j->bell_ns / 1000); ++ j->bell_ns / 1000, ++ g5p_jt_has_ovh(j) ? g5p_jt_us(j->ta_pub_ns, j->td_ev_ns) - ++ (s64)(j->ta_ticks / 24) - (s64)(j->td_ticks / 24) : -1LL); ++ g5p_jt_abs_line(buf, size, j); + } + static int g5p_jobtimes_show(struct seq_file *m, void *unused) + { +@@ -210,7 +250,7 @@ static int g5p_jobtimes_show(struct seq_file *m, void *unused) + struct g5p_jt *snapshot; + unsigned long flags; + u32 count, head; +- char line[768]; ++ char line[1024]; + + snapshot = kvmalloc_array(G5P_JT_RING, sizeof(*snapshot), GFP_KERNEL); + if (!snapshot) +@@ -240,12 +280,12 @@ static int g5p_gpu_stats_show(struct seq_file *m, void *unused) + spin_unlock_irqrestore(&d->status_lock, flags); + seq_printf( + m, +- "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu\n", ++ "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld\n", + now, s.since_ns, s.jobs, s.failed, s.ta_ticks, s.td_ticks, + s.ioctl_ns, s.queue_ns, s.prep_ns, s.ta_wait_ns, s.gap_ns, + s.td_wait_ns, s.tail_ns, s.touch_ns, s.bell_ns, s.samples, + s.ta_irq_ns, s.td_irq_ns, s.ta_decide_ns, s.td_decide_ns, +- s.joint); ++ s.joint, s.ovh_ns); + return 0; + } + static int g5p_gpu_stats_open(struct inode *inode, struct file *file) +@@ -272,6 +312,53 @@ static const struct file_operations g5p_gpu_stats_fops = { + .llseek = seq_lseek, + .release = single_release, + }; ++static int g5p_fwclock_cmp(const void *a, const void *b) ++{ ++ s64 x = *(const s64 *)a, y = *(const s64 *)b; ++ return (x > y) - (x < y); ++} ++static int g5p_fwclock_show(struct seq_file *m, void *unused) ++{ ++ struct drm_debugfs_entry *entry = m->private; ++ struct g5p_boot *b = to_g5p(entry->dev)->core->boot; ++ u32 hist[201] = {}, n = 0, pol; ++ u64 prev = 0, last = 0; ++ s64 *offsets; ++ ++ if (!b || !b->root[0].cpu) ++ return -ENODEV; ++ offsets = kvmalloc_array(2000, sizeof(*offsets), GFP_KERNEL); ++ if (!offsets) ++ return -ENOMEM; ++ pol = b->layout.offset[0x4a]; ++ seq_puts(m, "# change points: host_cyc fw_now(+918) fw_prev(+8f0)\n"); ++ for (u32 i = 0; i < 2000; i++) { ++ u64 now, old, host; ++ dma_rmb(); ++ now = g5p_dc_root_word(b, pol + 0x918) | ++ (u64)g5p_dc_root_word(b, pol + 0x91c) << 32; ++ old = g5p_dc_root_word(b, pol + 0x8f0) | ++ (u64)g5p_dc_root_word(b, pol + 0x8f4) << 32; ++ host = arch_timer_read_counter(); ++ if (i && now != prev) { ++ if (last) ++ hist[min_t(u64, (host - last) / 240, 200)]++; ++ last = host; ++ offsets[n++] = (s64)host - (s64)now; ++ seq_printf(m, "%llu %llu %llu\n", host, now, old); ++ } ++ prev = now; ++ udelay(10); ++ } ++ seq_puts(m, "# governor cadence: bucket_us count (2000 means >=2000)\n"); ++ for (u32 i = 0; i < ARRAY_SIZE(hist); i++) ++ seq_printf(m, "%u %u\n", i * 10, hist[i]); ++ sort(offsets, n, sizeof(*offsets), g5p_fwclock_cmp, NULL); ++ seq_printf(m, "changes=%u median_host_minus_fw_cyc=%lld\n", n, ++ n ? (offsets[n / 2] + offsets[(n - 1) / 2]) / 2 : -1LL); ++ kvfree(offsets); ++ return 0; ++} + static void g5p_jt_debugfs_init(struct drm_minor *minor) + { + struct g5p_drm *d = to_g5p(minor->dev); +@@ -513,10 +600,18 @@ static void g5p_drm_event(void *cookie, const u32 words[11]) + for (u32 i = 0; i < 4; i++) + d->notifications[i] |= words[i + 1]; + if (d->td_want && !d->td_hit_ns && +- g5p_ru_rt_event_match(words, d->td_want & 0xff)) ++ g5p_ru_rt_event_match(words, d->td_want & 0xff)) { + WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); +- if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) ++ d->td_hit_cyc = arch_timer_read_counter(); ++ d->td_hard_cyc = READ_ONCE(d->core->boot->hard_cyc); ++ d->td_src = current == READ_ONCE(d->core->boot->irq_task) ? 1 : 2; ++ } ++ if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) { + WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); ++ d->ta_hit_cyc = arch_timer_read_counter(); ++ d->ta_hard_cyc = READ_ONCE(d->core->boot->hard_cyc); ++ d->ta_src = current == READ_ONCE(d->core->boot->irq_task) ? 1 : 2; ++ } + for (u32 e = 0; words[0] == 1 && e < 2; e++) { + if (!(words[1] & BIT(e))) + continue; +@@ -781,6 +876,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + atomic_set(&d->r8_queued, 0); + drm_debugfs_add_file(&d->drm, "g5p_bos", g5p_bos_show, NULL); + drm_debugfs_add_file(&d->drm, "g5p_jobtimes", g5p_jobtimes_show, NULL); ++ drm_debugfs_add_file(&d->drm, "g5p_fwclock", g5p_fwclock_show, NULL); + drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); + INIT_DELAYED_WORK(&d->idle_work, g5p_idle_work); + ret = drm_dev_register(&d->drm, 0); +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +index 22a63accf..67870e050 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.h ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -48,6 +48,8 @@ struct g5p_rv3 { + u32 wk_did[2], wk_power[2], wk_fence[2], wk_gpc[2], wk_cons[2], + wk_t1_pub[2], wk_t1_end[2]; + u64 wk_start_ns[2], wk_done_ns[2], wk_first_ns[2]; ++ bool abs; ++ u64 bell_cyc[2]; + bool joint; /* latched at execute entry; attach stays serial */ + bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ + /* The bind of this job's context published its DC3 pair and then failed: that context is +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 761b3015f..20c350eb4 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -51,6 +51,8 @@ + * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. + */ + ++static const bool render_user_abs_stamps; ++ + static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { + "idle", "attach", "bind", "ta-publish", + "ta-wait", "3d-publish", "3d-wait", "done", +@@ -455,15 +457,19 @@ static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) + if (e) { + d->td_want = G5P_RV3_TD_WANT | endpoint; + WRITE_ONCE(d->td_hit_ns, 0); ++ d->td_hit_cyc = d->td_hard_cyc = 0; ++ d->td_src = 0; + } else { + d->ta_want = true; + WRITE_ONCE(d->ta_hit_ns, 0); ++ d->ta_hit_cyc = d->ta_hard_cyc = 0; ++ d->ta_src = 0; + } + spin_unlock_irqrestore(&d->status_lock, flags); + } + + /* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ +-static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e, struct g5p_jt *jt) + { + unsigned long flags; + u64 hit; +@@ -476,6 +482,11 @@ static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e) + d->ta_want = false; + hit = d->ta_hit_ns; + } ++ if (jt) { ++ jt->hit_cyc[e] = e ? d->td_hit_cyc : d->ta_hit_cyc; ++ jt->hard_cyc[e] = e ? d->td_hard_cyc : d->ta_hard_cyc; ++ jt->src[e] = e ? d->td_src : d->ta_src; ++ } + spin_unlock_irqrestore(&d->status_lock, flags); + return hit; + } +@@ -852,6 +863,7 @@ static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, + }; + v->wk_cons[e] = in.cons; + v->wk_power[e] = in.power; ++ v->bell_cyc[e] = arch_timer_read_counter(); + v->wk_did[e] = g5p_wk_decide(&in, &io, &v->wk_fence[e], &v->wk_gpc[e]); + v->wk_done_ns[e] = ktime_get_ns(); + spin_unlock_irqrestore(&b->event_lock, flags); +@@ -2574,6 +2586,10 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + u32 seq, res; + + job->jt.exec_ns = ktime_get_ns(); ++ job->jt.exec_cyc = arch_timer_read_counter(); ++ job->jt.prev_signal_cyc = d->jt_last_signal_cyc; ++ v->abs = READ_ONCE(render_user_abs_stamps); ++ job->jt.abs_enabled = v->abs; + v->joint = !job->compute && !first && READ_ONCE(render_user_joint); + job->jt.flags = (first ? 1 : 0) | (!bound_before ? 2 : 0) | + (v->joint ? BIT(2) : 0); +@@ -2664,7 +2680,7 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + g5p_rv3_pm_meta(r), r->slot[e], e, seq, + e ? job->td_w : job->ta_w, + g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, +- g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r)); ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r), v->abs); + r->submitted[e] = false; + r->consumed[e] = false; + r->retired[e] = false; +@@ -2833,9 +2849,10 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + ret = 0; + out: + if (v->phase >= G5P_RV3_TA_PUBLISH) +- v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0, &job->jt); + if (v->joint || v->phase >= G5P_RV3_TD_PUBLISH) +- v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1, &job->jt); ++ memcpy(job->jt.bell_cyc, v->bell_cyc, sizeof(v->bell_cyc)); + job->jt.ta_pub_ns = v->ta_pub_ns; + job->jt.td_pub_ns = v->td_pub_ns; + job->jt.ta_ev_ns = v->ta_event_ns; +@@ -2852,6 +2869,16 @@ out: + g5p_ru_w_at(g5p_rv3_js(rd, seq)) + + (e ? 0x8ac : 0x550); + u64 ticks = g5p_le32(p) | ((u64)g5p_le32(p + 4) << 32); ++ if (v->abs) { ++ u32 js = g5p_rv3_js(rd, seq); ++ u8 *page = g5p_ru_page(b, e); ++ u8 *w = page + g5p_ru_w_at(js); ++ for (u32 k = 0; k < 2; k++) { ++ job->jt.abs[e][k] = (u64)g5p_le32(page + G5P_RU_ABS + 16 * js + 8 * k) | ++ (u64)g5p_le32(page + G5P_RU_ABS + 16 * js + 8 * k + 4) << 32; ++ job->jt.wp[e][k] = g5p_le32(w + (e ? 0x8b4 : 0x558) + 4 * k); ++ } ++ } + if (e) + job->jt.td_ticks = ticks; + else +@@ -2934,7 +2961,7 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], + NULL, &v->tr, &v->last[0], + &v->ta_consumed_ns, &err); +- v->ta_event_ns = g5p_rv3_event_disarm(d, 0); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0, NULL); + } else { + if (!armed) + g5p_rv3_event_arm(d, 1, +@@ -2943,7 +2970,7 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], + &v->rt, NULL, &v->last[1], + &v->td_consumed_ns, &err); +- v->td_event_ns = g5p_rv3_event_disarm(d, 1); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1, NULL); + } + act = g5p_ru_drain_step(&st, res); + if (act != G5P_RU_DRAIN_PUBLISH_TD) +@@ -2954,7 +2981,7 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); + armed = true; + if (g5p_rv3_publish_3d(g, r, v, s->seq)) { +- g5p_rv3_event_disarm(d, 1); ++ g5p_rv3_event_disarm(d, 1, NULL); + res = G5P_RU_WAIT_FAULT; + act = g5p_ru_drain_step(&st, res); + break; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +index 851a6a115..e4291417d 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -1076,6 +1076,8 @@ static void g5p_render_run(struct g5p_rjob *job) + atomic_dec(&d->r8_queued); + dma_fence_signal(&job->fence->base); + job->jt.signal_ns = ktime_get_ns(); ++ job->jt.signal_cyc = arch_timer_read_counter(); ++ d->jt_last_signal_cyc = job->jt.signal_cyc; + job->jt.ret = ret; + job->jt.seq = job->rv3.seq; + spin_lock_irqsave(&d->status_lock, flags); +@@ -1085,7 +1087,7 @@ static void g5p_render_run(struct g5p_rjob *job) + g5p_jt_fold(&d->jt_stats, &job->jt); + spin_unlock_irqrestore(&d->status_lock, flags); + if (g5p_jt_log) { +- char line[768]; ++ char line[1024]; + g5p_jt_line(line, sizeof(line), &job->jt); + dev_info(g->dev, "%s", line); + } diff --git a/kernel/patches/drivers/drm/0019-drm-apple-g5p-pipeline-render-jobs-two-deep-render-u.patch b/kernel/patches/drivers/drm/0019-drm-apple-g5p-pipeline-render-jobs-two-deep-render-u.patch new file mode 100644 index 00000000..bf156476 --- /dev/null +++ b/kernel/patches/drivers/drm/0019-drm-apple-g5p-pipeline-render-jobs-two-deep-render-u.patch @@ -0,0 +1,1193 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:46 +0000 +Subject: [PATCH] drm/apple-g5p: pipeline render jobs two deep + (render_user_pipeline_depth) + +[Orchard: from brkosebay/linux 2be3fbe5f3c5908b604dd29002b01d1acf1b99ba; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-jobtime.h | 3 +- + drivers/gpu/apple-g5p/render-user-layout.h | 38 ++ + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 30 +- + .../gpu/drm/apple-g5p/drm-render-sched.inc | 16 +- + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 12 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 556 ++++++++++++++++-- + drivers/gpu/drm/apple-g5p/drm-render.inc | 96 ++- + 7 files changed, 697 insertions(+), 54 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-jobtime.h b/drivers/gpu/apple-g5p/render-user-jobtime.h +index 7eacc4b40..61e2de97f 100644 +--- a/drivers/gpu/apple-g5p/render-user-jobtime.h ++++ b/drivers/gpu/apple-g5p/render-user-jobtime.h +@@ -22,7 +22,7 @@ struct g5p_jt { + bool abs_enabled; + }; + struct g5p_jt_stats { +- u64 jobs, failed, joint, ta_ticks, td_ticks; ++ u64 jobs, failed, joint, ahead, ta_ticks, td_ticks; + u64 ioctl_ns, queue_ns, prep_ns, ta_wait_ns, gap_ns, td_wait_ns; + u64 tail_ns, touch_ns, bell_ns, samples; + s64 ovh_ns; +@@ -54,6 +54,7 @@ static inline void g5p_jt_fold(struct g5p_jt_stats *s, const struct g5p_jt *j) + s->jobs++; + s->failed += j->ret != 0; + s->joint += !!(j->flags & 4); ++ s->ahead += !!(j->flags & 8); /* F5: started while the previous job's 3D ran */ + if (j->ta_ticks != ~0ULL) + s->ta_ticks += j->ta_ticks; + if (j->td_ticks != ~0ULL) +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +index 624a56a11..675e9fbe6 100644 +--- a/drivers/gpu/apple-g5p/render-user-layout.h ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -18,6 +18,11 @@ + */ + #define G5P_RU_PRIVATE_VA G5P_R22_AUX + #define G5P_RU_PRIVATE_SIZE 0x1000000ULL ++/* F5: the second PM set's aux window, at the top of the private window (render targets are ++ * allocated from its bottom, so their VAs do not move). ++ */ ++#define G5P_RU_AUX2_VA \ ++ (G5P_RU_PRIVATE_VA + G5P_RU_PRIVATE_SIZE - 0x20000ULL) + #define G5P_RU_USER_VA_MIN 0x40000000ULL + /* Catalogue entry of the firmware GiB, borrowed from the boot tables. */ + #define G5P_RU_FIRMWARE_CI ((G5P_IMAGE_VA >> 30) & 1023) +@@ -642,6 +647,39 @@ static inline bool g5p_ru_retired_at(const u8 *p, u32 meta, bool td, u32 seq, + return true; + } + ++/* F5 (render_user_pipeline_depth 2): retirement of job `seq` while the later job `latest` ++ * (seq + 1) is already published on the same channel. The firmware's tag4 dependency test is ++ * signed `(required - *stamp) <= 0` (META 0x80001a38 / 0x80000a54; kext 6c8a808 builds the ++ * record against one stamp word per identity), so a stamp only moves forward and job seq is ++ * done once both stamp words are at or past seq. The W pointers and W sequence of seq's own ++ * slot must still be ours, and every channel watermark must lie between seq's endpoint and ++ * latest's (latest's entries may already be consumed). latest == seq: g5p_ru_retired_at(). ++ */ ++static inline bool g5p_ru_retired_pl(const u8 *p, u32 meta, bool td, u32 seq, ++ u32 js, u32 latest) ++{ ++ const u8 *w = p + g5p_ru_w_at(js); ++ u32 endpoint = g5p_ru_endpoint(seq); ++ u32 span = (2 * (latest - seq)) % G5P_WORK_SLOTS; ++ ++ if (latest == seq) ++ return g5p_ru_retired_at(p, meta, td, seq, js); ++ if (g5p_le32(w + (td ? 0x888 : 0x52c)) != meta + G5P_R22_STAMP || ++ g5p_le32(w + (td ? 0x88c : 0x530)) != meta + G5P_R22_STAMP + 4 || ++ g5p_le32(w + (td ? 0x890 : 0x534)) != seq || ++ (s32)(g5p_le32(p + G5P_R22_STAMP) - seq) < 0 || ++ (s32)(g5p_le32(p + G5P_R22_STAMP + 4) - seq) < 0) ++ return false; ++ for (u32 off = 0; off <= 0x40; off += 0x10) { ++ u32 ctl = g5p_le32(p + G5P_WORK_CTL + off); ++ ++ if (ctl >= G5P_WORK_SLOTS || ++ (ctl + G5P_WORK_SLOTS - endpoint) % G5P_WORK_SLOTS > span) ++ return false; ++ } ++ return true; ++} ++ + /* A stamp beyond the published target, or a watermark not on the even + * two-entry grid, is a protocol error rather than "not yet retired". + */ +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 1e285cdd5..f9ed3aad0 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -132,12 +132,14 @@ struct g5p_drm { + u32 autoboot_phase; + /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ + wait_queue_head_t ev_wq; +- /* TYPED_V3 3D completion-event latch (render-user-retire.h): G5P_RV3_TD_WANT | endpoint +- * while a 3D phase waits, else 0; td_hit_ns: decode time of the first matching event. ++ /* TYPED_V3 3D completion-event latches (render-user-retire.h): G5P_RV3_TD_WANT | endpoint ++ * while a 3D phase is published, else 0; td_hit_ns: decode time of the first matching event. ++ * Two, indexed by the job's sequence parity (g5p_rv3_td_latch()): with ++ * render_user_pipeline_depth 2 job N+1's 3D is armed while job N's 3D still waits. + */ +- u32 td_want; +- u64 td_hit_ns, td_hit_cyc, td_hard_cyc; +- u8 td_src; ++ u32 td_want[2]; ++ u64 td_hit_ns[2], td_hit_cyc[2], td_hard_cyc[2]; ++ u8 td_src[2]; + /* The TA's latch: armed while a TA phase is published; ta_hit_ns as td_hit_ns. */ + bool ta_want; + u64 ta_hit_ns, ta_hit_cyc, ta_hard_cyc; +@@ -280,12 +282,12 @@ static int g5p_gpu_stats_show(struct seq_file *m, void *unused) + spin_unlock_irqrestore(&d->status_lock, flags); + seq_printf( + m, +- "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld\n", ++ "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld ahead=%llu\n", + now, s.since_ns, s.jobs, s.failed, s.ta_ticks, s.td_ticks, + s.ioctl_ns, s.queue_ns, s.prep_ns, s.ta_wait_ns, s.gap_ns, + s.td_wait_ns, s.tail_ns, s.touch_ns, s.bell_ns, s.samples, + s.ta_irq_ns, s.td_irq_ns, s.ta_decide_ns, s.td_decide_ns, +- s.joint, s.ovh_ns); ++ s.joint, s.ovh_ns, s.ahead); + return 0; + } + static int g5p_gpu_stats_open(struct inode *inode, struct file *file) +@@ -599,12 +601,14 @@ static void g5p_drm_event(void *cookie, const u32 words[11]) + if (words[0] == 1) + for (u32 i = 0; i < 4; i++) + d->notifications[i] |= words[i + 1]; +- if (d->td_want && !d->td_hit_ns && +- g5p_ru_rt_event_match(words, d->td_want & 0xff)) { +- WRITE_ONCE(d->td_hit_ns, ktime_get_ns()); +- d->td_hit_cyc = arch_timer_read_counter(); +- d->td_hard_cyc = READ_ONCE(d->core->boot->hard_cyc); +- d->td_src = current == READ_ONCE(d->core->boot->irq_task) ? 1 : 2; ++ for (u32 k = 0; k < 2; k++) { ++ if (!d->td_want[k] || d->td_hit_ns[k] || ++ !g5p_ru_rt_event_match(words, d->td_want[k] & 0xff)) ++ continue; ++ WRITE_ONCE(d->td_hit_ns[k], ktime_get_ns()); ++ d->td_hit_cyc[k] = arch_timer_read_counter(); ++ d->td_hard_cyc[k] = READ_ONCE(d->core->boot->hard_cyc); ++ d->td_src[k] = current == READ_ONCE(d->core->boot->irq_task) ? 1 : 2; + } + if (d->ta_want && !d->ta_hit_ns && g5p_ru_ta_event_match(words)) { + WRITE_ONCE(d->ta_hit_ns, ktime_get_ns()); +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +index 21dc64d89..3b3ce4b17 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -162,6 +162,19 @@ static void g5p_rsched_work(struct work_struct *work) + bool has_wake; + int what; + ++ /* F5: the job the previous one started ahead runs first, before any drain, recovery ++ * or pick: it is on the hardware (or fails at once with its lost context). ++ */ ++ if (rd->ahead) { ++ struct g5p_rjob *next = rd->ahead; ++ ++ rd->ahead = NULL; ++ g5p_render_run(next); ++ /* not while the core lock is kept for a further ahead job */ ++ if (!rd->ahead) ++ g5p_render_close_deferred(d); ++ continue; ++ } + /* A lost job blocks the channel: hold every job until the drain is decided, and + * until a recovery the drain or the executor asked for has run. + */ +@@ -186,7 +199,8 @@ static void g5p_rsched_work(struct work_struct *work) + if (what == G5P_RS_JOB) { + g5p_render_run( + container_of(node, struct g5p_rjob, node)); +- g5p_render_close_deferred(d); ++ if (!rd->ahead) ++ g5p_render_close_deferred(d); + continue; + } + r = container_of(e, struct g5p_render, entity); +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +index 67870e050..db2af8af7 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.h ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -51,6 +51,18 @@ struct g5p_rv3 { + bool abs; + u64 bell_cyc[2]; + bool joint; /* latched at execute entry; attach stays serial */ ++ /* F5 (render_user_pipeline_depth 2, latched at execute entry): this job may start its ++ * context's next job once its own TA completed (g5p_rv3_ahead()). ++ */ ++ bool depth2; ++ /* F5: this job was prepared and published by the previous job's executor, while that ++ * job's 3D ran (core lock held throughout); ahead_ret is that start's result. Its own ++ * execute resumes at the TA wait. ++ */ ++ bool ahead; ++ int ahead_ret; ++ bool ta_disarmed; /* F5: the TA latch was disarmed at the TA decision */ ++ bool pm2; /* F5: the job's PM is its context's second set */ + bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ + /* The bind of this job's context published its DC3 pair and then failed: that context is + * lost, the device and the contexts already bound are not. +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 20c350eb4..18b33b2e5 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -53,6 +53,32 @@ + + static const bool render_user_abs_stamps; + ++/* ++ * F5, depth-2 job pipelining (docs/gpu/lane-a2-ta-done-latency-spec.md §4.3; latched per job at ++ * execute entry). 1: today's serial executor. 2: once job N's TA completed, and while its 3D ++ * runs, the executor prepares and publishes (TA and 3D, joint) the next job of the *same* context ++ * if it is queued with every input signalled, then waits and retires N, and the worker resumes ++ * N+1 at its TA wait. Apple's model: iOS kicks TA and 3D of each submission without waiting for ++ * the previous one's retire (AGXChannelQueue_submitBlockInvoke cbb948) and takes a ParamBuffer ++ * per prepare (6cb9afc). The 3D's tag4 wait on the shared TA-finish stamp compares signed ++ * (required - *stamp) <= 0 (firmware META 0x80001a38 / 0x80000a54, kext 6c8a808), so TA N+1 ++ * writing N+1 into that word never blocks N's 3D. Never across contexts: the core lock stays ++ * held from N's TA publish to N+1's 3D retire (Henrik's 0004: TA and 3D of one job adjacent, ++ * no other context's phase between them). ++ */ ++static const uint render_user_pipeline_depth = 1; ++ ++/* F5: consecutive ahead starts before the executor lets one job end with the core lock released, ++ * so a VM bind or an idle step waiting for that lock is never starved by a busy context. ++ */ ++#define G5P_RV3_AHEAD_CHAIN 8U ++ ++/* drm-render.inc / drm-render-sched.inc, included after this file */ ++static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, ++ bool to_cpu); ++static bool g5p_rsched_ready(struct g5p_rs_job *node, void *cookie, ++ unsigned long *deadline); ++ + static const char *const g5p_rv3_phase_names[G5P_RV3_PHASES] = { + "idle", "attach", "bind", "ta-publish", + "ta-wait", "3d-publish", "3d-wait", "done", +@@ -101,6 +127,40 @@ static struct g5p_dma_page *g5p_rv3_pm_aux(struct g5p_rdev *rd, + return r->pm_aux_mapped ? r->pm_aux : rd->aux; + } + ++/* F5: the PM of one job (pm2: its context's second set), for its records and its waits. */ ++static u8 *g5p_rv3_pm_cpu_of(struct g5p_boot *b, const struct g5p_render *r, ++ bool pm2) ++{ ++ if (pm2 && r->pm2_set >= 0) ++ return b->pm_set[r->pm2_set * G5P_PMSET_PAGES + G5P_PMSET_CONTROL] ++ .cpu; ++ return g5p_rv3_pm_cpu(b, r); ++} ++ ++static u32 g5p_rv3_pm_meta_of(const struct g5p_render *r, bool pm2) ++{ ++ if (pm2 && r->pm2_set >= 0) ++ return g5p_meta_address( ++ g5p_pmset_va(r->pm2_set, G5P_PMSET_CONTROL)); ++ return g5p_rv3_pm_meta(r); ++} ++ ++static u32 g5p_rv3_pm_id_of(const struct g5p_rdev *rd, ++ const struct g5p_render *r, bool pm2) ++{ ++ if (pm2 && r->pm2_set >= 0) ++ return rd->pm_ctx >= 2 ? g5p_pmset_id(r->pm2_set) : 0; ++ return g5p_rv3_pm_id(rd, r); ++} ++ ++/* F5: the 3D event latch of job seq (g5p_drm.td_want[]): its endpoint's parity, so jobs N and ++ * N + 1 never share one. ++ */ ++static u32 g5p_rv3_td_latch(u32 endpoint) ++{ ++ return (endpoint >> 1) & 1; ++} ++ + /* The boot tuple the V3 path is proven under (r96 params.txt), minus the instrumentation: + * r73b/r73c/r73e/r73f, r74 with the raised IRQ cap, r74f, r74g, r87b self-drain; no r74h, + * no r80..r96 (their IRQ banks read registers), no r75/r79 firmware control, and no fixed +@@ -321,10 +381,23 @@ static void g5p_rv3_sample_ram(struct g5p_diag *g, struct g5p_render *r, u32 e, + u32 seq, struct g5p_rv3_ev *s) + { + struct g5p_boot *b = g->boot; ++ struct g5p_rdev *rd = r->ctx->gpu->rdev; ++ u32 js = g5p_rv3_js(rd, seq); + u8 *p = g5p_ru_page(b, e), *global = g5p_ru_global(b, e), +- *pm = g5p_rv3_pm_cpu(b, r); ++ *pm = g5p_rv3_pm_cpu_of(b, r, r->js_pm2[js]); + u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); ++ /* F5: the device's latest published job; seq + 1 when the next job was started ahead. ++ * Then this job's ring entry is the one before the producer, its stamps may already be ++ * past seq, and its watermarks may already be on the next job's entries, once that job's ++ * phase on this engine is published (r->submitted[e] is the next job's). Otherwise ++ * latest == seq: exactly the serial predicates. ++ */ ++ u32 latest = rd->sequence, target = r->target[e]; + ++ if (latest - seq == 1 && r->submitted[e]) ++ target = (target - 1) & 255; ++ else ++ latest = seq; + memset(s, 0, sizeof(*s)); + dma_rmb(); + s->ns = ktime_get_ns(); +@@ -339,16 +412,18 @@ static void g5p_rv3_sample_ram(struct g5p_diag *g, struct g5p_render *r, u32 e, + s->pm80 = g5p_le32(pm + 0x80); + s->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); + s->proto = s->prod != r->target[e] || s->cons > 255 || +- !g5p_ru_stamp_sane(p, seq); +- s->consumed = s->cons == r->target[e]; ++ !g5p_ru_stamp_sane(p, latest); ++ /* consumed: the consumer reached this job's entry (serial: == the producer) */ ++ s->consumed = s->cons <= 255 && ++ ((s->cons - target) & 255) <= ((s->prod - target) & 255); + /* The predicates of the flown r74b (TA) and r96 (3D) waits; the 3D's PM serial is a receipt. */ + s->idle_ok = !s->idle[0] && !s->idle[1] && !s->idle[2] && + (!e || (!s->idle[4] && !s->idle[5])); +- s->retired = g5p_ru_retired_at(p, g5p_ru_meta(e), e, seq, +- g5p_rv3_js(r->ctx->gpu->rdev, seq)); +- s->pm_ok = e ? s->pm80 == r->ctx->gpu->rdev->pm_serial + 1 : ++ s->retired = g5p_ru_retired_pl(p, g5p_ru_meta(e), e, seq, js, latest); ++ s->pm_ok = e ? s->pm80 == rd->pm_serial + 1 : + g5p_r22_pm_saved(pm + 0x80); +- s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns) : ++ s->event = !!(e ? READ_ONCE(r->ctx->gpu->td_hit_ns[g5p_rv3_td_latch( ++ g5p_ru_endpoint(seq))]) : + READ_ONCE(r->ctx->gpu->ta_hit_ns)); + } + +@@ -453,12 +528,14 @@ static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) + unsigned long flags; + + BUILD_BUG_ON(G5P_RU_RT_GRID != G5P_WORK_SLOTS); ++ u32 k = g5p_rv3_td_latch(endpoint); ++ + spin_lock_irqsave(&d->status_lock, flags); + if (e) { +- d->td_want = G5P_RV3_TD_WANT | endpoint; +- WRITE_ONCE(d->td_hit_ns, 0); +- d->td_hit_cyc = d->td_hard_cyc = 0; +- d->td_src = 0; ++ d->td_want[k] = G5P_RV3_TD_WANT | endpoint; ++ WRITE_ONCE(d->td_hit_ns[k], 0); ++ d->td_hit_cyc[k] = d->td_hard_cyc[k] = 0; ++ d->td_src[k] = 0; + } else { + d->ta_want = true; + WRITE_ONCE(d->ta_hit_ns, 0); +@@ -468,24 +545,28 @@ static void g5p_rv3_event_arm(struct g5p_drm *d, u32 e, u32 endpoint) + spin_unlock_irqrestore(&d->status_lock, flags); + } + +-/* Disarm engine e's latch and return the decode time of the matching event (0 = none). */ +-static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e, struct g5p_jt *jt) ++/* Disarm engine e's latch (3D: job seq's) and return the decode time of the matching event ++ * (0 = none). ++ */ ++static u64 g5p_rv3_event_disarm(struct g5p_drm *d, u32 e, u32 seq, ++ struct g5p_jt *jt) + { ++ u32 k = g5p_rv3_td_latch(g5p_ru_endpoint(seq)); + unsigned long flags; + u64 hit; + + spin_lock_irqsave(&d->status_lock, flags); + if (e) { +- d->td_want = 0; +- hit = d->td_hit_ns; ++ d->td_want[k] = 0; ++ hit = d->td_hit_ns[k]; + } else { + d->ta_want = false; + hit = d->ta_hit_ns; + } + if (jt) { +- jt->hit_cyc[e] = e ? d->td_hit_cyc : d->ta_hit_cyc; +- jt->hard_cyc[e] = e ? d->td_hard_cyc : d->ta_hard_cyc; +- jt->src[e] = e ? d->td_src : d->ta_src; ++ jt->hit_cyc[e] = e ? d->td_hit_cyc[k] : d->ta_hit_cyc; ++ jt->hard_cyc[e] = e ? d->td_hard_cyc[k] : d->ta_hard_cyc; ++ jt->src[e] = e ? d->td_src[k] : d->ta_src; + } + spin_unlock_irqrestore(&d->status_lock, flags); + return hit; +@@ -628,7 +709,8 @@ static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, + rt ? rt->unsat : + ta ? ta->unsat : + 0, +- e ? READ_ONCE(d->td_hit_ns) : ++ e ? READ_ONCE(d->td_hit_ns[g5p_rv3_td_latch( ++ g5p_ru_endpoint(seq))]) : + READ_ONCE(d->ta_hit_ns)); + } + if (w->expired) { +@@ -1471,12 +1553,40 @@ static int g5p_rv3_publish_ta(struct g5p_diag *g, struct g5p_render *r, + * 0 TA not retired 1 TA not consumed 2 3D already submitted 3 producer > 255 4 consumer != producer + * 5 WORK_CTL 6 WORK_CTL+0x40 7 command tag != 4 8 dependency word 9 dependency value + * 10 3D stamp words != seq - 1 (job N leaves N; 0 before the first) ++ * F5 (ahead: job seq is published while job seq - 1's 3D runs): clause 5 admits WORK_CTL anywhere ++ * on the previous job's own entries (not yet consumed, half or wholly), clause 10 the stamps at ++ * seq - 2 or seq - 1 (g5p_rv3_ahead_why() checked both before anything was staged). + */ ++static bool g5p_rv3_ctl_behind(u32 ctl, u32 previous) ++{ ++ return ctl < G5P_WORK_SLOTS && ++ (previous + G5P_WORK_SLOTS - ctl) % G5P_WORK_SLOTS <= 2; ++} ++ ++static bool g5p_rv3_stamp_behind(u32 stamp, u32 seq) ++{ ++ return stamp == seq - 1 || stamp == seq - 2; ++} ++ + static u32 g5p_rv3_td_why(const struct g5p_render *r, const u8 *p, +- const u32 *ring, u32 start, u32 previous, u32 seq, bool joint) ++ const u32 *ring, u32 start, u32 previous, u32 seq, bool joint, ++ bool ahead) + { + const u8 *cmd = p + g5p_ru_cmd_at(g5p_rv3_js(r->ctx->gpu->rdev, seq)); + ++ if (ahead) ++ return (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | ++ (u32)(READ_ONCE(ring[0]) != start) << 4 | ++ (u32)!g5p_rv3_ctl_behind(g5p_le32(p + G5P_WORK_CTL), ++ previous) << 5 | ++ (u32)(g5p_le32(p + G5P_WORK_CTL + 0x40) != previous) << 6 | ++ (u32)(g5p_le32(cmd) != 4) << 7 | ++ (u32)(g5p_le32(cmd + 4) != ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4) << 8 | ++ (u32)(g5p_le32(cmd + 8) != seq) << 9 | ++ (u32)(!g5p_rv3_stamp_behind(g5p_le32(p + G5P_R22_STAMP), seq) || ++ !g5p_rv3_stamp_behind(g5p_le32(p + G5P_R22_STAMP + 4), ++ seq)) << 10; + return (u32)(!joint && !r->retired[0]) | + (u32)(!joint && !r->consumed[0]) << 1 | + (u32)r->submitted[1] << 2 | (u32)(start > 255) << 3 | +@@ -1506,7 +1616,8 @@ static int g5p_rv3_publish_3d(struct g5p_diag *g, struct g5p_render *r, + /* RAM preconditions of r96's admit (g5p_rv3_td_why()). Nothing has been written when one + * fails, and the refusal names the clauses and the raw values it read. + */ +- why = g5p_rv3_td_why(r, p, ring, start, previous, seq, v->joint); ++ why = g5p_rv3_td_why(r, p, ring, start, previous, seq, v->joint, ++ v->ahead); + if (why) { + dev_emerg( + g->dev, +@@ -1862,14 +1973,22 @@ static int g5p_rv3_lose(struct g5p_drm *d, struct g5p_render *r, + s->engine = e; + s->drain_ms = (45000); + s->lost_ns = ktime_get_ns(); ++ /* F5: the context's next job, already published behind this one (its TA at least), is ++ * on the channel too: the drain waits for it after this one. Its own run fails at once ++ * (the context is lost). ++ */ ++ if (d->rdev->ahead && d->rdev->ahead->ctx == r->ctx && ++ d->rdev->sequence == seq + 1 && r->submitted[0]) ++ s->ahead_seq = seq + 1; + kref_get(&r->ctx->ref); + d->rdev->stuck = s; + dev_emerg( + d->core->dev, +- "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u: context " +- "lost, device kept, channel blocked end=v3\n", ++ "RENDER-V3 lost seq=%u engine=%u slots=%u/%u phase_ms=%lld drain_ms=%u ahead=%u: " ++ "context lost, device kept, channel blocked end=v3\n", + seq, e, r->slot[0], r->slot[1], +- g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms); ++ g5p_rv3_us(w->start_ns, w->end_ns) / 1000, s->drain_ms, ++ s->ahead_seq); + return 0; + } + +@@ -2209,6 +2328,110 @@ static void g5p_rv3_pmctx_take(struct g5p_drm *d, struct g5p_render *r, u32 seq) + rd->pm_set_next = (set + 1) % b->pm_sets; + } + ++/* F5: the context's second PM set (render_user_pipeline_depth 2), built for this epoch at the ++ * device's block count the way g5p_rv3_pmctx_build() builds the first: taken from the boot sets ++ * with its own pool at the context's first pipelined job, rebuilt (RAM only) when the epoch or ++ * the device pool changed. Its aux pages are the context's own, mapped at G5P_RU_AUX2_VA at ++ * creation, so no live VM is edited. Core lock held; the job in flight uses the other set. ++ * false: the context stays serial (no set or no memory; not retried). ++ */ ++static bool g5p_rv3_pm2_ready(struct g5p_drm *d, struct g5p_render *r, ++ u32 seq) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_dma_page *page; ++ u64 *physical = NULL; ++ bool fresh = r->pm2_set < 0; ++ int set = r->pm2_set, ret = 0; ++ ++ if (r->pm2_failed || !r->aux2_mapped || r->pm_set < 0) ++ return false; ++ if (!fresh && r->pm2_epoch == rd->recoveries + 1 && ++ r->pm2_blocks >= rd->blocks) ++ return true; ++ if (fresh) ++ set = g5p_pmset_pick(rd->pm_set_used, b->pm_sets, ++ rd->pm_set_next); ++ if (set < 0) { ++ r->pm2_failed = true; ++ dev_warn( ++ g->dev, ++ "RENDER-V3 pm2 seq=%u slots=%u/%u set=none: no free PM set, the context stays " ++ "serial end=v3\n", ++ seq, r->slot[0], r->slot[1]); ++ return false; ++ } ++ page = &b->pm_set[set * G5P_PMSET_PAGES]; ++ mutex_lock(&rd->lock); ++ if (rd->blocks > r->pm2_blocks) { ++ u32 blocks = rd->blocks; ++ struct g5p_dma_page *pool = ++ kvcalloc(blocks * 8, sizeof(*pool), GFP_KERNEL); ++ ++ ret = pool ? 0 : -ENOMEM; ++ if (pool && r->pm2_pool) ++ memcpy(pool, r->pm2_pool, ++ r->pm2_blocks * 8 * sizeof(*pool)); ++ for (u32 i = r->pm2_blocks * 8; !ret && i < blocks * 8; i++) ++ ret = g5p_vm_alloc_page(d->drm.dev, &pool[i]); ++ if (ret && pool) { ++ for (u32 i = r->pm2_blocks * 8; i < blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &pool[i]); ++ kvfree(pool); ++ } else if (!ret) { ++ kvfree(r->pm2_pool); ++ r->pm2_pool = pool; ++ r->pm2_blocks = blocks; ++ } ++ } ++ if (!ret) { ++ physical = kvcalloc(r->pm2_blocks * 8, sizeof(*physical), ++ GFP_KERNEL); ++ ret = physical ? 0 : -ENOMEM; ++ } ++ for (u32 i = 0; !ret && i < r->pm2_blocks * 8; i++) { ++ physical[i] = r->pm2_pool[i].dma; ++ memset(r->pm2_pool[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm2_pool[i]); ++ } ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ memset(r->pm2_aux[i].cpu, 0, G5P_GPU_PAGE); ++ g5p_table_clean_page(g, &r->pm2_aux[i]); ++ } ++ if (!ret) ++ ret = g5p_ru_pm(page[G5P_PMSET_CONTROL].cpu, ++ page[G5P_PMSET_LIST].cpu, ++ page[G5P_PMSET_FREE].cpu, ++ g5p_pmset_va(set, G5P_PMSET_LIST), ++ g5p_pmset_va(set, G5P_PMSET_FREE), physical, ++ r->pm2_blocks, G5P_RU_AUX2_VA); ++ kvfree(physical); ++ if (!ret) { ++ for (u32 k = 0; k < G5P_PMSET_PAGES; k++) ++ g5p_table_clean_page(g, &page[k]); ++ dma_wmb(); ++ } ++ mutex_unlock(&rd->lock); ++ dev_info( ++ g->dev, ++ "RENDER-V3 pm2 seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u ret=%d end=v3\n", ++ seq, r->slot[0], r->slot[1], fresh ? "new" : "rebuild", set, ++ rd->pm_ctx >= 2 ? g5p_pmset_id(set) : 0, r->pm2_blocks, ret); ++ if (ret) { ++ r->pm2_failed = true; ++ return false; ++ } ++ if (fresh) { ++ r->pm2_set = set; ++ rd->pm_set_used[set] = true; ++ rd->pm_set_next = (set + 1) % b->pm_sets; ++ } ++ r->pm2_epoch = rd->recoveries + 1; ++ return true; ++} ++ + /* perf60 stage 4 diagnostics (render_user_fault_read): whose page each PM lane in S points at + * after the TA saved them: the device pool or aux, a set page, a context's pool or aux. + */ +@@ -2235,6 +2458,17 @@ static void g5p_rv3_pmctx_release(struct g5p_drm *d, struct g5p_render *r) + r->pm_blocks = 0; + for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) + g5p_ru_free_page(d->drm.dev, &r->pm_aux[i]); ++ /* F5: the second set the same way */ ++ if (r->pm2_set >= 0) ++ rd->pm_set_used[r->pm2_set] = false; ++ r->pm2_set = -1; ++ for (u32 i = 0; r->pm2_pool && i < r->pm2_blocks * 8; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm2_pool[i]); ++ kvfree(r->pm2_pool); ++ r->pm2_pool = NULL; ++ r->pm2_blocks = 0; ++ for (u32 i = 0; i < G5P_RU_AUX_PAGES; i++) ++ g5p_ru_free_page(d->drm.dev, &r->pm2_aux[i]); + } + + #define G5P_RV3_PM_S 0x80U +@@ -2573,6 +2807,212 @@ static void g5p_oc_work(struct work_struct *work) + + + ++/* F5: a depth-2 job's region headers in slice js of its target's main ring (TA.W28, 3D.W20; ++ * 6cb3d68 sizes the ring for sixteen slices, the r25 RT variant uses the next one the same way), ++ * so consecutive jobs alternate slices and a job started ahead never writes what the previous ++ * job's 3D reads. The secondary buffer is the TA's alone (the 3D W has no pointer to it). RAM ++ * edit of the job's own W images, before they are staged. ++ */ ++static void g5p_rv3_rt_slice(struct g5p_rjob *job, u32 js) ++{ ++ u64 ta = g5p_le32(job->ta_w + 0x28) | ++ (u64)g5p_le32(job->ta_w + 0x2c) << 32; ++ u64 td = g5p_le32(job->td_w + 0x20) | ++ (u64)g5p_le32(job->td_w + 0x24) << 32; ++ ++ if (!js) ++ return; ++ g5p_put64(job->ta_w + 0x28, ta + job->rt_slice); ++ g5p_put64(job->td_w + 0x20, td + job->rt_slice); ++} ++ ++/* F5: why job `cur` (its TA complete, its 3D published and running) may not start its context's ++ * next job now. RAM reads only; 0 = it may. Never printed per job: counted in ahead_skipped. ++ */ ++static u32 g5p_rv3_ahead_why(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *cur) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &cur->rv3; ++ struct g5p_ru_v3_state st = g5p_rv3_state(g, r); ++ u32 seq = v->seq, prev = g5p_ru_endpoint(seq); ++ u32 *ta_ring = (u32 *)g5p_boot_shared(b, 0x60), ++ *td_ring = (u32 *)g5p_boot_shared(b, 0x64); ++ u8 *ta = g5p_ru_page(b, 0), *td = g5p_ru_page(b, 1); ++ u32 ta_start, td_start; ++ ++ if (!v->depth2 || !v->joint || v->ahead_ret) ++ return 1; ++ if (rd->job_slots != 2 || !rd->pm_ctx || r->pm_set < 0 || ++ r->pm2_failed || !r->aux2_mapped) ++ return 2; ++ if (!r->bound || r->stage_done || r->stall_job || r->fault_job || ++ r->touch_ms) ++ return 3; ++ if (rd->ahead || rd->stuck || READ_ONCE(rd->recovering) || ++ READ_ONCE(cur->ctx->bad)) ++ return 4; ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || ++ g5p_ru_v3_admit(&st, true)) ++ return 5; ++ if (rd->ahead_chain >= G5P_RV3_AHEAD_CHAIN) ++ return 6; ++ dma_rmb(); ++ /* the TA channel at rest on cur's entries (g5p_rv3_publish_ta()'s own admission) */ ++ ta_start = READ_ONCE(ta_ring[4]); ++ if (ta_start > 255 || READ_ONCE(ta_ring[0]) != ta_start || ++ ta_start != r->target[0] || ++ g5p_le32(ta + G5P_WORK_CTL) != prev || ++ g5p_le32(ta + G5P_WORK_CTL + 0x40) != prev) ++ return 7; ++ /* the 3D channel on cur's entries, its notification consumed, cur not overtaken */ ++ td_start = READ_ONCE(td_ring[4]); ++ if (td_start > 255 || READ_ONCE(td_ring[0]) != td_start || ++ td_start != r->target[1] || ++ g5p_le32(td + G5P_WORK_CTL + 0x40) != prev || ++ !g5p_rv3_ctl_behind(g5p_le32(td + G5P_WORK_CTL), prev) || ++ !g5p_rv3_stamp_behind(g5p_le32(td + G5P_R22_STAMP), seq + 1) || ++ !g5p_rv3_stamp_behind(g5p_le32(td + G5P_R22_STAMP + 4), seq + 1)) ++ return 8; ++ return 0; ++} ++ ++/* F5: the head job of r's entity if it may run now (a typed render job, every input signalled, ++ * no error). pop: taken off the entity the way g5p_rs_pick() takes a picked job. An input not ++ * yet signalled gets the scheduler's callback, as at a pick. NULL: none. ++ */ ++static struct g5p_rjob *g5p_rv3_ahead_next(struct g5p_drm *d, ++ struct g5p_render *r, bool pop) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_rs_entity *e = &r->entity; ++ struct g5p_rjob *next = NULL; ++ unsigned long flags, deadline = 0; ++ ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ if (e->in_run && !list_empty(&e->jobs)) { ++ struct g5p_rs_job *head = ++ list_first_entry(&e->jobs, struct g5p_rs_job, link); ++ struct g5p_rjob *j = container_of(head, struct g5p_rjob, node); ++ ++ if (j->v3 && !j->compute && !j->setup_error && !j->dep_error && ++ g5p_rsched_ready(head, d, &deadline) && !j->dep_error) ++ next = j; ++ if (next && pop) { ++ list_del_init(&head->link); ++ if (list_empty(&e->jobs) && !e->close_pending) { ++ list_del_init(&e->link); ++ e->in_run = false; ++ } ++ } ++ } ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ return next; ++} ++ ++/* F5: start `job`, the next job of context r, while job `cur`'s 3D runs (core lock held, cur's TA ++ * complete): g5p_rv3_execute()'s preparation and joint publication for a bound context's job, on ++ * the other W slot, the other PM set and the other slice of its target's ring. Its result is ++ * job->rv3.ahead_ret; the worker runs it next (rd->ahead) and its execute resumes at the TA wait. ++ */ ++static void g5p_rv3_ahead_start(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *cur, struct g5p_rjob *job) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b = g->boot; ++ struct g5p_rv3 *v = &job->rv3; ++ u32 seq = rd->sequence + 1, js = g5p_rv3_js(rd, seq); ++ bool pm2 = !r->js_pm2[g5p_rv3_js(rd, cur->rv3.seq)]; ++ int ret; ++ ++ job->jt.exec_ns = ktime_get_ns(); ++ job->jt.exec_cyc = arch_timer_read_counter(); ++ job->jt.prev_signal_cyc = d->jt_last_signal_cyc; ++ v->abs = READ_ONCE(render_user_abs_stamps); ++ job->jt.abs_enabled = v->abs; ++ v->joint = true; ++ v->depth2 = READ_ONCE(render_user_pipeline_depth) == 2; /* latched per job */ ++ v->ahead = true; ++ v->pm2 = pm2; ++ job->jt.flags = BIT(2) | BIT(3); ++ rd->ahead = job; ++ rd->ahead_started++; ++ rd->ahead_chain++; ++ /* before the hardware may read them, as g5p_render_run() does for a picked job */ ++ job->cache.enabled = READ_ONCE(render_user_cache_receipts); ++ job->cache.policy = READ_ONCE(render_user_cache_private_wc); ++ job->cache.import_policy = READ_ONCE(render_user_cache_import_wc); ++ g5p_render_cache(d, job, false); ++ v->touched = true; ++ ret = pm2 ? 0 : g5p_rv3_pmctx_job(d, r, seq); ++ if (ret) ++ goto out; ++ g5p_rv3_pmsnap(d, r, seq, G5P_PMSNAP_START); ++ rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu_of(b, r, pm2) + 0x80); ++ r->js_pm2[js] = pm2; ++ g5p_rv3_rt_slice(job, js); ++ for (u32 e = 0; e < 2; e++) { ++ g5p_ru_job_at( ++ g5p_ru_page(b, e), g5p_ru_meta(e), ++ g5p_meta_address(g5p_object_va(&b->layout, 0x51 + e)), ++ g5p_rv3_pm_meta_of(r, pm2), r->slot[e], e, seq, ++ e ? job->td_w : job->ta_w, ++ g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, js, ++ g5p_rv3_pm_id_of(rd, r, pm2), v->abs); ++ r->submitted[e] = false; ++ r->consumed[e] = false; ++ r->retired[e] = false; ++ } ++ g5p_l123_job_prepare(d, job); ++ g5p_render_clean_job(g); ++ rd->sequence = seq; ++ v->seq = seq; ++ r->jobs++; ++ v->phase = G5P_RV3_TA_PUBLISH; ++ g5p_rv3_event_arm(d, 0, 0); ++ g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); ++ g5p_rv3_wake_mark(d, r, v, 0); ++ g5p_rv3_wake_mark(d, r, v, 1); ++ ret = g5p_rv3_publish_ta(g, r, v, seq); ++ if (ret) ++ goto out; ++ v->phase = G5P_RV3_TD_PUBLISH; ++ ret = g5p_rv3_publish_3d(g, r, v, seq); ++ if (ret) ++ g5p_rv3_sample(g, r, 1, seq, &v->last[1]); ++out: ++ v->ahead_ret = ret; ++} ++ ++/* F5: after job `cur`'s TA completed, start its context's next job if it may (core lock held). ++ * The second PM set is made ready (taken and built at the context's first pipelined job) only ++ * when there is a next job to use it. ++ */ ++static void g5p_rv3_ahead(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *cur) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ bool pm2 = !r->js_pm2[g5p_rv3_js(rd, cur->rv3.seq)]; ++ struct g5p_rjob *next; ++ ++ if (g5p_rv3_ahead_why(d, r, cur) || !g5p_rv3_ahead_next(d, r, false) || ++ (pm2 && !g5p_rv3_pm2_ready(d, r, cur->rv3.seq + 1))) { ++ rd->ahead_skipped++; ++ return; ++ } ++ next = g5p_rv3_ahead_next(d, r, true); ++ if (!next) { ++ rd->ahead_skipped++; ++ return; ++ } ++ g5p_rv3_ahead_start(d, r, cur, next); ++} ++ + static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + struct g5p_rjob *job) + { +@@ -2585,12 +3025,26 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + int err = -EIO, ret, why, loss; + u32 seq, res; + ++ if (v->ahead) { ++ /* F5: prepared and published by the previous job's executor, which kept the core ++ * lock (g5p_rv3_ahead_start()): wait for it from its TA on, or end with that start's ++ * result. ++ */ ++ seq = v->seq; ++ if (v->ahead_ret) { ++ ret = v->ahead_ret; ++ goto out; ++ } ++ g5p_l123_job_reset(g); ++ goto ta_wait; ++ } + job->jt.exec_ns = ktime_get_ns(); + job->jt.exec_cyc = arch_timer_read_counter(); + job->jt.prev_signal_cyc = d->jt_last_signal_cyc; + v->abs = READ_ONCE(render_user_abs_stamps); + job->jt.abs_enabled = v->abs; + v->joint = !job->compute && !first && READ_ONCE(render_user_joint); ++ v->depth2 = v->joint && READ_ONCE(render_user_pipeline_depth) == 2; + job->jt.flags = (first ? 1 : 0) | (!bound_before ? 2 : 0) | + (v->joint ? BIT(2) : 0); + if (job->compute) +@@ -2673,6 +3127,13 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + rd->pm_serial = g5p_le32(g5p_rv3_pm_cpu(b, r) + 0x80); + if (g5p_ru_fault_due(r->fault_job, r->jobs, first)) + g5p_rv3_test_fault(g, r, job, seq); ++ /* F5: a job started with nothing in flight uses the first PM set; with depth 2 its region ++ * headers go to its W slot's slice of the target ring, so the job started behind it uses ++ * the other set and slice. ++ */ ++ r->js_pm2[g5p_rv3_js(rd, seq)] = false; ++ if (v->depth2) ++ g5p_rv3_rt_slice(job, g5p_rv3_js(rd, seq)); + for (u32 e = 0; e < 2; e++) { + g5p_ru_job_at( + g5p_ru_page(b, e), g5p_ru_meta(e), +@@ -2708,6 +3169,7 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + goto out; + } + } ++ta_wait: + v->phase = G5P_RV3_TA_WAIT; + if (r->stall_job && r->jobs == r->stall_job && !first) { + /* render_user_test_stall_job: the host gives up, the firmware runs on */ +@@ -2773,7 +3235,14 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + if (first) + g5p_r74_mark(g, G5P_R74_TA_RAM_COMPLETE, + g5p_le32(g5p_ru_page(b, 0) + G5P_R22_STAMP + 4)); +- ; ++ if (v->depth2) { ++ /* F5: the TA latch is the next job's from here; then start that job while this ++ * job's 3D (published with the TA) runs. ++ */ ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0, seq, &job->jt); ++ v->ta_disarmed = true; ++ g5p_rv3_ahead(d, r, job); ++ } + if (!v->joint) { + v->phase = G5P_RV3_TD_PUBLISH; + g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(seq)); +@@ -2792,7 +3261,9 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + g5p_ru_rt_start(&v->rt, r->settle_ms); + res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, + &v->last[1], &v->td_consumed_ns, &err); +- r->consumed[1] = v->td_consumed_ns != 0; ++ /* F5: once a later job is prepared, the context's flags are that job's */ ++ if (rd->sequence == seq) ++ r->consumed[1] = v->td_consumed_ns != 0; + /* mmio_selftest bit 2 (rung 5, DFU-risk on a park tuple): one licensed ticket at the 3D + * completion decision; always: the idle clock of bit 3. + */ +@@ -2842,16 +3313,17 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + v->last[1].idle[4], v->last[1].idle[5], + g5p_ru_rt_ms(v->td_pub_ns, v->rt.coherent_ns), + g5p_ru_rt_ms(v->td_pub_ns, v->td.end_ns)); +- r->retired[1] = true; ++ if (rd->sequence == seq) ++ r->retired[1] = true; + v->phase = G5P_RV3_DONE; + rd->recover_streak = + 0; /* a job finished: a later fault starts a new streak */ + ret = 0; + out: +- if (v->phase >= G5P_RV3_TA_PUBLISH) +- v->ta_event_ns = g5p_rv3_event_disarm(d, 0, &job->jt); ++ if (v->phase >= G5P_RV3_TA_PUBLISH && !v->ta_disarmed) ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0, seq, &job->jt); + if (v->joint || v->phase >= G5P_RV3_TD_PUBLISH) +- v->td_event_ns = g5p_rv3_event_disarm(d, 1, &job->jt); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1, seq, &job->jt); + memcpy(job->jt.bell_cyc, v->bell_cyc, sizeof(v->bell_cyc)); + job->jt.ta_pub_ns = v->ta_pub_ns; + job->jt.td_pub_ns = v->td_pub_ns; +@@ -2909,6 +3381,9 @@ out: + /* perf60 bell gates: the next job's gap and context switch are measured from here. */ + d->bell_prev_end_ns = ktime_get_ns(); + d->bell_prev_ctx = r->ctx; ++ /* F5: a job that started none ends the chain (its run releases the core lock) */ ++ if (!rd->ahead) ++ rd->ahead_chain = 0; + return ret; + } + +@@ -2946,6 +3421,7 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + msleep(5); + } + WRITE_ONCE(rd->executing, true); ++again: + for (;;) { + u32 e = st == G5P_RU_DRAIN_TA ? 0 : 1; + u64 now = ktime_get_ns(); +@@ -2961,7 +3437,7 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], + NULL, &v->tr, &v->last[0], + &v->ta_consumed_ns, &err); +- v->ta_event_ns = g5p_rv3_event_disarm(d, 0, NULL); ++ v->ta_event_ns = g5p_rv3_event_disarm(d, 0, s->seq, NULL); + } else { + if (!armed) + g5p_rv3_event_arm(d, 1, +@@ -2970,7 +3446,7 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], + &v->rt, NULL, &v->last[1], + &v->td_consumed_ns, &err); +- v->td_event_ns = g5p_rv3_event_disarm(d, 1, NULL); ++ v->td_event_ns = g5p_rv3_event_disarm(d, 1, s->seq, NULL); + } + act = g5p_ru_drain_step(&st, res); + if (act != G5P_RU_DRAIN_PUBLISH_TD) +@@ -2981,12 +3457,26 @@ static bool g5p_rv3_drain(struct g5p_drm *d) + g5p_rv3_event_arm(d, 1, g5p_ru_endpoint(s->seq)); + armed = true; + if (g5p_rv3_publish_3d(g, r, v, s->seq)) { +- g5p_rv3_event_disarm(d, 1, NULL); ++ g5p_rv3_event_disarm(d, 1, s->seq, NULL); + res = G5P_RU_WAIT_FAULT; + act = g5p_ru_drain_step(&st, res); + break; + } + } ++ if (act == G5P_RU_DRAIN_UNBLOCK && s->ahead_seq) { ++ /* F5: the next job, published behind the stuck one, drains the same way */ ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 drained seq=%u engine=%u slots=%u/%u: the job started behind it " ++ "(seq=%u) drains next end=v3\n", ++ s->seq, s->engine, r->slot[0], r->slot[1], s->ahead_seq); ++ s->seq = s->ahead_seq; ++ s->ahead_seq = 0; ++ st = G5P_RU_DRAIN_TA; ++ res = G5P_RU_WAIT_PENDING; ++ armed = false; ++ goto again; ++ } + if (act == G5P_RU_DRAIN_UNBLOCK) { + r->retired[1] = true; + r->lost_drained = true; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +index e4291417d..2cd6737d1 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -232,6 +232,16 @@ struct g5p_rdev { + u32 recover_streak; /* recoveries since a job completed (core lock) */ + struct list_head + contexts; /* every render context (lock), for the recovery */ ++ /* F5 (render_user_pipeline_depth 2): the job its context's previous job started ahead ++ * (prepared and published while that job's 3D ran). The worker runs it next, before any ++ * pick, drain or recovery; ahead_held: the previous run kept the core lock for it, so the ++ * lock stays held from one job's TA publish to the next job's 3D retire (Henrik 0004: no ++ * other context's phase is published in between). Worker only. ++ */ ++ struct g5p_rjob *ahead; ++ bool ahead_held; ++ u32 ahead_chain; /* consecutive ahead starts (G5P_RV3_AHEAD_CHAIN) */ ++ u64 ahead_started, ahead_skipped; + }; + + /* render-user-fault.h's ruling window on the device state: a failure noted while a TYPED_V3 job or +@@ -306,6 +316,20 @@ struct g5p_render { + struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; + bool pm_aux_mapped; + bool pm_first; /* it brought the epoch up (attach): the device aux is its own */ ++ /* F5 (render_user_pipeline_depth 2): the second PM set, the parameter buffer of a job ++ * started while the previous job's 3D still reads the first (iOS takes a ParamBuffer ID ++ * per prepare, 6cb9afc). Its aux pages are allocated and mapped at aux2_node (the top of ++ * the private window) at creation; the set and its pool are taken at the first pipelined ++ * job and kept for the context's life, like the first. js_pm2[js]: the job in W slot js ++ * uses it. pm2_failed: no set or memory; the context stays serial. ++ */ ++ int pm2_set; ++ u32 pm2_blocks, pm2_epoch; ++ struct g5p_dma_page *pm2_pool; ++ struct g5p_dma_page pm2_aux[G5P_RU_AUX_PAGES]; ++ struct drm_mm_node aux2_node; ++ bool aux2_mapped, pm2_failed; ++ bool js_pm2[2]; + bool close_defer_run; /* render_user_close_defer: its deferred close is running */ + }; + +@@ -316,6 +340,7 @@ struct g5p_rstuck { + struct g5p_render *r; + struct g5p_rv3 v; + u32 seq, engine, drain_ms; ++ u32 ahead_seq; /* F5: the context's next job, already published behind it; 0 = none */ + u64 lost_ns; + }; + +@@ -358,6 +383,11 @@ struct g5p_rjob { + struct g5p_cu_regs cu; + struct g5p_rv3 rv3; + u64 submit_ns; /* latency123: the submit ioctl */ ++ /* F5: the slice of its target's main ring (6cb3d68: sixteen slices). With ++ * render_user_pipeline_depth 2 a job's TA.W28/3D.W20 move js slices into the ring, so a ++ * job started ahead never writes the region headers the previous job's 3D reads. ++ */ ++ u64 rt_slice; + struct g5p_jt jt; + }; + +@@ -985,8 +1015,13 @@ static void g5p_render_run(struct g5p_rjob *job) + struct g5p_render *r = job->ctx->render; + unsigned long lock_deadline, flags; + bool hardware = false; ++ /* F5: a job started ahead is on the hardware already, under the core lock the previous ++ * run kept for it (rdev->ahead_held). ++ */ ++ bool ahead = job->rv3.ahead, held = ahead && d->rdev->ahead_held; + int ret; + ++ d->rdev->ahead_held = false; + job->cache.enabled = READ_ONCE(render_user_cache_receipts); + job->cache.policy = READ_ONCE(render_user_cache_private_wc); + job->cache.import_policy = READ_ONCE(render_user_cache_import_wc); +@@ -999,13 +1034,31 @@ static void g5p_render_run(struct g5p_rjob *job) + * that fails here, without waiting for its inputs and without competing for the command + * lock. + */ +- if (g5p_r8_admit_closed(&g5p_r8_state.admit)) ++ if (ahead) { ++ /* Admitted at its start. Its fate is now its context's: a lost previous job lost the ++ * context, and the drain or the recovery covers this job's hardware too; a failed ++ * device fails it. Otherwise it is waited for whatever admission says now, because ++ * the firmware holds its memory. ++ */ ++ hardware = true; ++ ret = 0; ++ if (job->ctx->bad) { ++ job->rv3.lost = true; ++ ret = -EIO; ++ } else if (READ_ONCE(g->boot->failed)) { ++ ret = -EIO; ++ } ++ if (ret && held) { ++ mutex_unlock(&g->lock); ++ held = false; ++ } ++ } else if (g5p_r8_admit_closed(&g5p_r8_state.admit)) + ret = -ESHUTDOWN; + else + ret = job->setup_error ?: job->dep_error; +- if (!ret && (job->ctx->bad || READ_ONCE(g->boot->failed))) ++ if (!ret && !ahead && (job->ctx->bad || READ_ONCE(g->boot->failed))) + ret = -EIO; +- if (!ret) { ++ if (!ret && !held) { + /* Bounded: never queue forever behind a command stuck in MMIO. */ + lock_deadline = jiffies + 2 * HZ; + while (!mutex_trylock(&g->lock)) { +@@ -1026,14 +1079,16 @@ static void g5p_render_run(struct g5p_rjob *job) + /* RAM-only from the DC3 attach on: no snapshot (it reads R0), no + * export, no register access of any kind after a timeout. + */ +- g5p_render_cache(d, job, false); ++ if (!ahead) /* an ahead job was cleaned at its start */ ++ g5p_render_cache(d, job, false); + WRITE_ONCE(d->rdev->executing, true); + d->jt_active = &job->jt; + ret = g5p_rv3_execute(d, r, job); + d->jt_active = NULL; + /* after `recovering`, if the executor set it (g5p_rdev_ruling()) */ + smp_store_release(&d->rdev->executing, false); +- hardware = job->rv3.touched || job->rv3.bind_failed; ++ hardware = ahead || job->rv3.touched || ++ job->rv3.bind_failed; + if (!ret || ret == -ECANCELED) + g5p_render_cache( + d, job, +@@ -1042,7 +1097,14 @@ static void g5p_render_run(struct g5p_rjob *job) + WRITE_ONCE(r->live_maps, true); + } + g->last_result = ret; +- mutex_unlock(&g->lock); ++ /* F5: this job completed and started its context's next one: the core lock stays ++ * held for it (the worker runs it next). A failed job releases the lock; its ++ * context is lost, and the next job fails without waiting. ++ */ ++ if (!ret && d->rdev->ahead) ++ d->rdev->ahead_held = true; ++ else ++ mutex_unlock(&g->lock); + } + spin_lock_irqsave(&d->status_lock, flags); + /* A failed device fails every later job; one context's lost job or failed bind does not. */ +@@ -1248,6 +1310,9 @@ static void g5p_render_unhook(struct g5p_context *c) + if (r->aux_mapped) + g5p_vm_clear(c->vm, dev, G5P_RU_PRIVATE_VA, + G5P_RU_AUX_PAGES * G5P_GPU_PAGE); ++ if (r->aux2_mapped) ++ g5p_vm_clear(c->vm, dev, G5P_RU_AUX2_VA, ++ G5P_RU_AUX_PAGES * G5P_GPU_PAGE); + for (u32 i = 0; i < 4; i++) { + struct drm_mm_node *n = i == 0 ? &r->low_node : + i == 1 ? &r->firmware_node : +@@ -1287,6 +1352,8 @@ static void g5p_render_release(struct g5p_context *c) + } + if (drm_mm_node_allocated(&r->aux_node)) + drm_mm_remove_node(&r->aux_node); ++ if (drm_mm_node_allocated(&r->aux2_node)) ++ drm_mm_remove_node(&r->aux2_node); + drm_mm_takedown(&r->private_mm); + spin_lock_irqsave(&d->core->slot_lock, flags); + for (u32 e = 0; e < 2; e++) +@@ -1368,6 +1435,7 @@ static int g5p_render_create_context(struct drm_device *drm, + return -ENOMEM; + } + r->pm_set = -1; ++ r->pm2_set = -1; + c->gpu = d; + c->render = r; + r->ctx = c; +@@ -1475,6 +1543,21 @@ static int g5p_render_create_context(struct drm_device *drm, + &d->rdev->aux[i]); + r->aux_mapped = true; + } ++ /* F5: the second PM set's own aux pages, mapped once here (file lock, before any job), so ++ * a pipelined job never edits a live VM. Unused while render_user_pipeline_depth is 1. ++ */ ++ r->aux2_node.start = G5P_RU_AUX2_VA; ++ r->aux2_node.size = G5P_RU_AUX_PAGES * G5P_GPU_PAGE; ++ if (!ret) ++ ret = drm_mm_reserve_node(&r->private_mm, &r->aux2_node); ++ for (u32 i = 0; !ret && i < G5P_RU_AUX_PAGES; i++) { ++ ret = g5p_vm_alloc_page(d->drm.dev, &r->pm2_aux[i]); ++ if (!ret) ++ ret = g5p_render_map_page(d, r, ++ G5P_RU_AUX2_VA + i * G5P_GPU_PAGE, ++ &r->pm2_aux[i]); ++ r->aux2_mapped = true; ++ } + if (ret) + goto err_unlock; + ret = xa_alloc(&f->contexts, &id, c, XA_LIMIT(1, G5P_MAX_CONTEXTS), +@@ -2170,6 +2253,7 @@ static int g5p_render_submit_admitted(struct drm_device *drm, + g5p_ru_rt_fill(job->ta_w, job->td_w, &rt->geometry, + rt->node.start + (r->v.rt == 1 ? rt->sizes.slice : 0), + rt->node.start + rt->sizes.main); ++ job->rt_slice = rt->sizes.slice; + { + g5p_ru_fill_msaa(job->ta_w, job->td_w, &msaa, &rt->geometry); + }; diff --git a/kernel/patches/drivers/drm/0020-drm-apple-g5p-honor-fragment-only-own-context-waits-.patch b/kernel/patches/drivers/drm/0020-drm-apple-g5p-honor-fragment-only-own-context-waits-.patch new file mode 100644 index 00000000..aacc5478 --- /dev/null +++ b/kernel/patches/drivers/drm/0020-drm-apple-g5p-honor-fragment-only-own-context-waits-.patch @@ -0,0 +1,286 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:47 +0000 +Subject: [PATCH] drm/apple-g5p: honor fragment-only own-context waits and + attribute ahead decisions + +[Orchard: from brkosebay/linux fd3bb22d5f627fdb6f88308d4b18c95a58a6f3dc; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-jobtime.h | 5 ++ + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 10 +-- + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 28 +++++++-- + drivers/gpu/drm/apple-g5p/drm-render.inc | 69 ++++++++++++++++++++- + include/uapi/drm/apple_g5p_drm.h | 7 +++ + 5 files changed, 105 insertions(+), 14 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-jobtime.h b/drivers/gpu/apple-g5p/render-user-jobtime.h +index 61e2de97f..96ae1d258 100644 +--- a/drivers/gpu/apple-g5p/render-user-jobtime.h ++++ b/drivers/gpu/apple-g5p/render-user-jobtime.h +@@ -19,9 +19,11 @@ struct g5p_jt { + u64 bell_cyc[2], hit_cyc[2], hard_cyc[2], exec_cyc, signal_cyc; + u64 prev_signal_cyc; + u8 src[2]; ++ u8 dep_own, dep_full, why; + bool abs_enabled; + }; + struct g5p_jt_stats { ++ u64 dep_own, dep_full, why[16]; + u64 jobs, failed, joint, ahead, ta_ticks, td_ticks; + u64 ioctl_ns, queue_ns, prep_ns, ta_wait_ns, gap_ns, td_wait_ns; + u64 tail_ns, touch_ns, bell_ns, samples; +@@ -52,6 +54,9 @@ static inline void g5p_jt_fold(struct g5p_jt_stats *s, const struct g5p_jt *j) + if (g5p_jt_has_ovh(j)) + s->ovh_ns += g5p_jt_ovh_ns(j); + s->jobs++; ++ s->dep_own += j->dep_own; ++ s->dep_full += j->dep_full; ++ if (j->why < 16) s->why[j->why]++; + s->failed += j->ret != 0; + s->joint += !!(j->flags & 4); + s->ahead += !!(j->flags & 8); /* F5: started while the previous job's 3D ran */ +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index f9ed3aad0..09fa937a9 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -208,10 +208,10 @@ static void g5p_jt_abs_line(char *buf, size_t size, const struct g5p_jt *j) + g5p_jt_cyc_us(ta1, j->hit_cyc[0]), + g5p_jt_cyc_us(j->flags & 4 ? ta1 : j->bell_cyc[1], td0), + g5p_jt_cyc_us(td0, td1), g5p_jt_cyc_us(td1, j->hit_cyc[1])); +- n += scnprintf(buf + n, size - n, " irqth=%lld/%lld src=%u/%u pgap=%lld", ++ n += scnprintf(buf + n, size - n, " irqth=%lld/%lld src=%u/%u pgap=%lld dep=%u/%u why=%u", + j->src[0] == 1 ? g5p_jt_cyc_us(j->hard_cyc[0], j->hit_cyc[0]) : -1LL, + j->src[1] == 1 ? g5p_jt_cyc_us(j->hard_cyc[1], j->hit_cyc[1]) : -1LL, +- j->src[0], j->src[1], g5p_jt_cyc_us(j->prev_signal_cyc, j->exec_cyc)); ++ j->src[0], j->src[1], g5p_jt_cyc_us(j->prev_signal_cyc, j->exec_cyc), j->dep_own, j->dep_full, j->why); + if (j->abs_enabled) + n += scnprintf(buf + n, size - n, + " wp=%08x,%08x/%08x,%08x raw=%llx,%llx,%llx,%llx", +@@ -282,12 +282,12 @@ static int g5p_gpu_stats_show(struct seq_file *m, void *unused) + spin_unlock_irqrestore(&d->status_lock, flags); + seq_printf( + m, +- "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld ahead=%llu\n", ++ "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld ahead=%llu dep_own=%llu dep_full=%llu aheadwhy=%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", + now, s.since_ns, s.jobs, s.failed, s.ta_ticks, s.td_ticks, + s.ioctl_ns, s.queue_ns, s.prep_ns, s.ta_wait_ns, s.gap_ns, + s.td_wait_ns, s.tail_ns, s.touch_ns, s.bell_ns, s.samples, + s.ta_irq_ns, s.td_irq_ns, s.ta_decide_ns, s.td_decide_ns, +- s.joint, s.ovh_ns, s.ahead); ++ s.joint, s.ovh_ns, s.ahead, s.dep_own, s.dep_full, s.why[0], s.why[1], s.why[2], s.why[3], s.why[4], s.why[5], s.why[6], s.why[7], s.why[8], s.why[9], s.why[10], s.why[11], s.why[12], s.why[13], s.why[14], s.why[15]); + return 0; + } + static int g5p_gpu_stats_open(struct inode *inode, struct file *file) +@@ -433,7 +433,7 @@ static int g5p_get_info(struct drm_device *drm, void *data, + ; + ; + { +- caps |= DRM_G5P_CAP_TYPED_RENDER_V3 | ++ caps |= DRM_G5P_CAP_SYNC_WAIT_3D | DRM_G5P_CAP_TYPED_RENDER_V3 | + DRM_G5P_CAP_RENDER_MULTI_CONTEXT | + DRM_G5P_CAP_RENDER_PIXEL_OUTPUT_WIDTH | + DRM_G5P_CAP_RENDER_OCCLUSION_QUERY; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 18b33b2e5..85cb08d38 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -1,3 +1,6 @@ ++/* F5b: honour DRM_G5P_SYNC_WAIT_3D (drop this context's own fences from a 3D-only input). A no-op ++ * at render_user_pipeline_depth 1: a job is picked only after its predecessor retired. */ ++static const bool render_user_ordered_waits = true; + /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + /* + * TYPED_V3 serial executor: the control flow that completed the natural TA and the +@@ -2886,7 +2889,7 @@ static u32 g5p_rv3_ahead_why(struct g5p_drm *d, struct g5p_render *r, + * yet signalled gets the scheduler's callback, as at a pick. NULL: none. + */ + static struct g5p_rjob *g5p_rv3_ahead_next(struct g5p_drm *d, +- struct g5p_render *r, bool pop) ++ struct g5p_render *r, bool pop, u32 *why) + { + struct g5p_rdev *rd = d->rdev; + struct g5p_rs_entity *e = &r->entity; +@@ -2894,14 +2897,19 @@ static struct g5p_rjob *g5p_rv3_ahead_next(struct g5p_drm *d, + unsigned long flags, deadline = 0; + + spin_lock_irqsave(&rd->sched_lock, flags); ++ if (why) *why = 10; + if (e->in_run && !list_empty(&e->jobs)) { + struct g5p_rs_job *head = + list_first_entry(&e->jobs, struct g5p_rs_job, link); + struct g5p_rjob *j = container_of(head, struct g5p_rjob, node); + +- if (j->v3 && !j->compute && !j->setup_error && !j->dep_error && +- g5p_rsched_ready(head, d, &deadline) && !j->dep_error) ++ if (!j->v3 || j->compute || j->setup_error || j->dep_error) { ++ if (why) *why = 11; ++ } else if (!g5p_rsched_ready(head, d, &deadline) || j->dep_error) { ++ if (why) *why = 12; ++ } else { + next = j; ++ } + if (next && pop) { + list_del_init(&head->link); + if (list_empty(&e->jobs) && !e->close_pending) { +@@ -3000,16 +3008,24 @@ static void g5p_rv3_ahead(struct g5p_drm *d, struct g5p_render *r, + bool pm2 = !r->js_pm2[g5p_rv3_js(rd, cur->rv3.seq)]; + struct g5p_rjob *next; + +- if (g5p_rv3_ahead_why(d, r, cur) || !g5p_rv3_ahead_next(d, r, false) || +- (pm2 && !g5p_rv3_pm2_ready(d, r, cur->rv3.seq + 1))) { ++ u32 why = 0; ++ int ret = g5p_rv3_ahead_why(d, r, cur); ++ if (ret) why = ret + 1; ++ else if (!g5p_rv3_ahead_next(d, r, false, &why)) { } ++ else if (pm2 && !g5p_rv3_pm2_ready(d, r, cur->rv3.seq + 1)) why = 13; ++ else why = 0; ++ if (why) { ++ cur->jt.why = why; + rd->ahead_skipped++; + return; + } +- next = g5p_rv3_ahead_next(d, r, true); ++ next = g5p_rv3_ahead_next(d, r, true, NULL); + if (!next) { ++ cur->jt.why = 14; + rd->ahead_skipped++; + return; + } ++ cur->jt.why = 1; + g5p_rv3_ahead_start(d, r, cur, next); + } + +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +index 2cd6737d1..d51dc945a 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -362,6 +362,7 @@ struct g5p_rjob { + struct g5p_fence *fence; + struct dma_fence *inputs[G5P_MAX_SYNCS - 1]; + u32 input_count; ++ u32 input_3d; + struct g5p_robj *objs; + u32 obj_count; + struct dma_fence **deps; /* implicit dependencies from dma_resv */ +@@ -988,6 +989,60 @@ static int g5p_render_implicit(struct g5p_rjob *job, u64 own_context) + } + + /* Free the mappings unmapped beside jobs up to fence seqno `seqno` (all: every one). */ ++/* F5b (render_user_ordered_waits): a 3D-only input (DRM_G5P_SYNC_WAIT_3D) needs none of this ++ * context's own earlier jobs: they are ahead of this job in the context's FIFO, and their 3D runs ++ * before this job's on the one 3D ring (TA N+1 may overlap 3D N, which a 3D-only wait allows). ++ * Each such input is replaced by its other unsignalled leaves; a leaf signalled with an error is ++ * kept, so the job still fails before the hardware. No room for the leaves: keep it whole. ++ * Called before the job is pushed (dep_next 0). Own-leaf errors after this point are covered by ++ * ctx->bad and the two-in-flight drain, not dep_error. dma_fence_get_status() takes the fence ++ * lock: fine here (process context, before the push); keep the locked variant. ++ */ ++static void g5p_render_order_inputs(struct g5p_rjob *job, u64 own) ++{ ++ struct dma_fence *keep[ARRAY_SIZE(job->inputs)]; ++ u32 n = 0; ++ ++ for (u32 i = 0; i < job->input_count; i++) { ++ struct dma_fence *f = job->inputs[i], *leaf; ++ struct dma_fence_unwrap it; ++ u32 own_n = 0, other = 0, room; ++ ++ if (!(job->input_3d & BIT(i))) { ++ keep[n++] = f; ++ continue; ++ } ++ dma_fence_unwrap_for_each(leaf, &it, f) { ++ if (dma_fence_is_signaled(leaf) && dma_fence_get_status(leaf) >= 0) ++ continue; ++ if (leaf->context == own && !dma_fence_is_signaled(leaf)) ++ own_n++; ++ else ++ other++; ++ } ++ room = ARRAY_SIZE(job->inputs) - n - (job->input_count - i - 1); ++ if (!own_n || other > room) { ++ if (own_n && job->jt.dep_full < 255) ++ job->jt.dep_full++; ++ keep[n++] = f; ++ continue; ++ } ++ dma_fence_unwrap_for_each(leaf, &it, f) { ++ if (dma_fence_is_signaled(leaf) && dma_fence_get_status(leaf) >= 0) ++ continue; ++ if (leaf->context == own && !dma_fence_is_signaled(leaf)) ++ continue; ++ if (n < ARRAY_SIZE(keep)) ++ keep[n++] = dma_fence_get(leaf); ++ } ++ job->jt.dep_own = min(255U, job->jt.dep_own + own_n); ++ dma_fence_put(f); ++ } ++ memcpy(job->inputs, keep, n * sizeof(*keep)); ++ job->input_count = n; ++ job->input_3d = 0; ++} ++ + static void g5p_render_reap(struct g5p_render *r, u64 seqno, bool all) + { + struct g5p_mapping *m, *t; +@@ -1714,9 +1769,12 @@ static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, + syncs[i].point == syncs[j].point && + syncs[i].flags == syncs[j].flags) + return -EINVAL; +- if (syncs[i].flags & ~DRM_G5P_SYNC_SIGNAL) ++ if (syncs[i].flags & ~(DRM_G5P_SYNC_SIGNAL | DRM_G5P_SYNC_WAIT_3D)) ++ return -EINVAL; ++ if ((syncs[i].flags & DRM_G5P_SYNC_SIGNAL) && ++ (syncs[i].flags & DRM_G5P_SYNC_WAIT_3D)) + return -EINVAL; +- if (syncs[i].flags) { ++ if (syncs[i].flags & DRM_G5P_SYNC_SIGNAL) { + if (*signal) + return -EINVAL; + *signal = drm_syncobj_find(file, syncs[i].handle); +@@ -1733,12 +1791,14 @@ static int g5p_render_syncs(struct drm_file *file, struct g5p_rjob *job, + &job->inputs[job->input_count]); + if (ret) + return ret; ++ if (syncs[i].flags & DRM_G5P_SYNC_WAIT_3D) ++ job->input_3d |= BIT(job->input_count); + job->input_count++; + } + if (!*signal) + return -EINVAL; + for (u32 i = 0; i < count; i++) +- if (!syncs[i].flags && syncs[i].handle == signal_handle && ++ if (!(syncs[i].flags & DRM_G5P_SYNC_SIGNAL) && syncs[i].handle == signal_handle && + (!*point || !syncs[i].point || syncs[i].point >= *point)) + return -EINVAL; + if (*point) { +@@ -2299,6 +2359,9 @@ static int g5p_render_submit_admitted(struct drm_device *drm, + } + *job_id = job->id; + job->deadline = jiffies + msecs_to_jiffies(G5P_RU_INPUT_MS); ++ if (job->input_3d && READ_ONCE(render_user_ordered_waits)) ++ g5p_render_order_inputs(job, r->fence_context); ++ job->input_3d = 0; + job->setup_error = g5p_render_implicit(job, r->fence_context); + drm_dev_get(drm); + job->jt.id = job->id; +diff --git a/include/uapi/drm/apple_g5p_drm.h b/include/uapi/drm/apple_g5p_drm.h +index 5e41ab533..174e5957e 100644 +--- a/include/uapi/drm/apple_g5p_drm.h ++++ b/include/uapi/drm/apple_g5p_drm.h +@@ -628,4 +628,11 @@ struct drm_g5p_submit_compute_typed { + DRM_IOWR(DRM_COMMAND_BASE + DRM_G5P_SUBMIT_COMPUTE_TYPED, \ + struct drm_g5p_submit_compute_typed) + ++/* An input that gates only the job's 3D (fragment) stage. The kernel drops the parts of it that ++ * are this render context's own earlier jobs (same-context 3D runs in queue order); every other ++ * fence still gates the whole job. Not with DRM_G5P_SYNC_SIGNAL. Needs DRM_G5P_CAP_SYNC_WAIT_3D. */ ++#define DRM_G5P_SYNC_WAIT_3D (1U << 1) ++/* DRM_G5P_CAP_SYNC_WAIT_3D - drm_g5p_sync.flags accepts DRM_G5P_SYNC_WAIT_3D */ ++#define DRM_G5P_CAP_SYNC_WAIT_3D (1ULL << 34) ++ + #endif diff --git a/kernel/patches/drivers/drm/0021-drm-apple-g5p-also-start-the-next-job-early-on-submi.patch b/kernel/patches/drivers/drm/0021-drm-apple-g5p-also-start-the-next-job-early-on-submi.patch new file mode 100644 index 00000000..1abc2c6e --- /dev/null +++ b/kernel/patches/drivers/drm/0021-drm-apple-g5p-also-start-the-next-job-early-on-submi.patch @@ -0,0 +1,404 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:47 +0000 +Subject: [PATCH] drm/apple-g5p: also start the next job early on submit + during the 3D wait + +[Orchard: from brkosebay/linux 7860353758439c13faae99d5a43c43e73ae0553b; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-jobtime.h | 7 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 20 ++- + drivers/gpu/drm/apple-g5p/drm-latency123.inc | 11 +- + .../gpu/drm/apple-g5p/drm-render-sched.inc | 2 + + drivers/gpu/drm/apple-g5p/drm-render-v3.h | 7 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 134 ++++++++++++++++-- + 6 files changed, 160 insertions(+), 21 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-jobtime.h b/drivers/gpu/apple-g5p/render-user-jobtime.h +index 96ae1d258..501da66fb 100644 +--- a/drivers/gpu/apple-g5p/render-user-jobtime.h ++++ b/drivers/gpu/apple-g5p/render-user-jobtime.h +@@ -20,10 +20,15 @@ struct g5p_jt { + u64 prev_signal_cyc; + u8 src[2]; + u8 dep_own, dep_full, why; ++ /* F5c: 1 = this job started its context's next job from its 3D wait (render_user_ahead_on_submit); ++ * why keeps the TA-decision reason. The started job has flags bit 4 (and bit 3, as F5). ++ */ ++ u8 late; + bool abs_enabled; + }; + struct g5p_jt_stats { + u64 dep_own, dep_full, why[16]; ++ u64 ahead_late, late_why[16]; /* F5c: late starts; by the starter's TA-decision why */ + u64 jobs, failed, joint, ahead, ta_ticks, td_ticks; + u64 ioctl_ns, queue_ns, prep_ns, ta_wait_ns, gap_ns, td_wait_ns; + u64 tail_ns, touch_ns, bell_ns, samples; +@@ -60,6 +65,8 @@ static inline void g5p_jt_fold(struct g5p_jt_stats *s, const struct g5p_jt *j) + s->failed += j->ret != 0; + s->joint += !!(j->flags & 4); + s->ahead += !!(j->flags & 8); /* F5: started while the previous job's 3D ran */ ++ s->ahead_late += !!(j->flags & 16); /* F5c: ... from that job's 3D wait (counted in ahead too) */ ++ if (j->late && j->why < 16) s->late_why[j->why]++; + if (j->ta_ticks != ~0ULL) + s->ta_ticks += j->ta_ticks; + if (j->td_ticks != ~0ULL) +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 09fa937a9..f184afb68 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -132,6 +132,13 @@ struct g5p_drm { + u32 autoboot_phase; + /* render_user_event_wake: the phase waiter sleeps here until event_serial moves */ + wait_queue_head_t ev_wq; ++ /* F5c (render_user_ahead_on_submit): while a depth-2 job's 3D wait may still start its ++ * context's next job (ahead_late_armed, written by the worker only), a push or a queued head's ++ * signalled input bumps ahead_poke and wakes ev_wq; the waiter's pause also ends on it. ++ * Never bumps event_serial (the firmware event count). ++ */ ++ atomic_t ahead_poke; ++ bool ahead_late_armed; + /* TYPED_V3 3D completion-event latches (render-user-retire.h): G5P_RV3_TD_WANT | endpoint + * while a 3D phase is published, else 0; td_hit_ns: decode time of the first matching event. + * Two, indexed by the job's sequence parity (g5p_rv3_td_latch()): with +@@ -208,10 +215,10 @@ static void g5p_jt_abs_line(char *buf, size_t size, const struct g5p_jt *j) + g5p_jt_cyc_us(ta1, j->hit_cyc[0]), + g5p_jt_cyc_us(j->flags & 4 ? ta1 : j->bell_cyc[1], td0), + g5p_jt_cyc_us(td0, td1), g5p_jt_cyc_us(td1, j->hit_cyc[1])); +- n += scnprintf(buf + n, size - n, " irqth=%lld/%lld src=%u/%u pgap=%lld dep=%u/%u why=%u", ++ n += scnprintf(buf + n, size - n, " irqth=%lld/%lld src=%u/%u pgap=%lld dep=%u/%u why=%u late=%u", + j->src[0] == 1 ? g5p_jt_cyc_us(j->hard_cyc[0], j->hit_cyc[0]) : -1LL, + j->src[1] == 1 ? g5p_jt_cyc_us(j->hard_cyc[1], j->hit_cyc[1]) : -1LL, +- j->src[0], j->src[1], g5p_jt_cyc_us(j->prev_signal_cyc, j->exec_cyc), j->dep_own, j->dep_full, j->why); ++ j->src[0], j->src[1], g5p_jt_cyc_us(j->prev_signal_cyc, j->exec_cyc), j->dep_own, j->dep_full, j->why, j->late); + if (j->abs_enabled) + n += scnprintf(buf + n, size - n, + " wp=%08x,%08x/%08x,%08x raw=%llx,%llx,%llx,%llx", +@@ -282,12 +289,16 @@ static int g5p_gpu_stats_show(struct seq_file *m, void *unused) + spin_unlock_irqrestore(&d->status_lock, flags); + seq_printf( + m, +- "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld ahead=%llu dep_own=%llu dep_full=%llu aheadwhy=%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", ++ "now_ns=%llu since_ns=%llu jobs=%llu failed=%llu ta_ticks=%llu td_ticks=%llu ioctl_ns=%llu queue_ns=%llu prep_ns=%llu ta_wait_ns=%llu gap_ns=%llu td_wait_ns=%llu tail_ns=%llu touch_ns=%llu bell_ns=%llu samples=%llu ta_irq_ns=%llu td_irq_ns=%llu ta_decide_ns=%llu td_decide_ns=%llu joint=%llu ovh_ns=%lld ahead=%llu dep_own=%llu dep_full=%llu aheadwhy=%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu" ++ " ahead_late=%llu latewhy=%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu,%llu\n", + now, s.since_ns, s.jobs, s.failed, s.ta_ticks, s.td_ticks, + s.ioctl_ns, s.queue_ns, s.prep_ns, s.ta_wait_ns, s.gap_ns, + s.td_wait_ns, s.tail_ns, s.touch_ns, s.bell_ns, s.samples, + s.ta_irq_ns, s.td_irq_ns, s.ta_decide_ns, s.td_decide_ns, +- s.joint, s.ovh_ns, s.ahead, s.dep_own, s.dep_full, s.why[0], s.why[1], s.why[2], s.why[3], s.why[4], s.why[5], s.why[6], s.why[7], s.why[8], s.why[9], s.why[10], s.why[11], s.why[12], s.why[13], s.why[14], s.why[15]); ++ s.joint, s.ovh_ns, s.ahead, s.dep_own, s.dep_full, s.why[0], s.why[1], s.why[2], s.why[3], s.why[4], s.why[5], s.why[6], s.why[7], s.why[8], s.why[9], s.why[10], s.why[11], s.why[12], s.why[13], s.why[14], s.why[15], ++ s.ahead_late, s.late_why[0], s.late_why[1], s.late_why[2], s.late_why[3], s.late_why[4], ++ s.late_why[5], s.late_why[6], s.late_why[7], s.late_why[8], s.late_why[9], s.late_why[10], ++ s.late_why[11], s.late_why[12], s.late_why[13], s.late_why[14], s.late_why[15]); + return 0; + } + static int g5p_gpu_stats_open(struct inode *inode, struct file *file) +@@ -859,6 +870,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + spin_lock_init(&d->status_lock); + d->jt_stats.since_ns = ktime_get_ns(); + init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->ahead_poke, 0); + atomic_set(&d->accepted, 0); + atomic64_set(&d->job_sequence, 0); + d->render_wq = drmm_alloc_ordered_workqueue(&d->drm, "g5p-render", WQ_HIGHPRI); +diff --git a/drivers/gpu/drm/apple-g5p/drm-latency123.inc b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +index a08b0d422..d0cd28e30 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-latency123.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-latency123.inc +@@ -233,10 +233,13 @@ static void g5p_l123_evict(struct g5p_boot *b) + } + + /* Replaces the waiter's sleep between two RAM samples of engine e. seen: d->event_serial as +- * read after the drain and before this sample (render-user-evwait.h). ++ * read after the drain and before this sample (render-user-evwait.h). late_seen (F5c, a depth-2 ++ * 3D wait that may still start the next job; else NULL): the event-wake sleep also ends when ++ * d->ahead_poke moves past it. + */ + static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, +- const struct g5p_rv3_ev *s, u64 seen) ++ const struct g5p_rv3_ev *s, u64 seen, ++ const int *late_seen) + { + u64 now = s->ns, t0; + enum g5p_ew_act act; +@@ -271,7 +274,9 @@ static void g5p_l123_pause(struct g5p_drm *d, struct g5p_diag *g, u32 e, + g5p_l123.ew[2]++; + } else if (wait_event_hrtimeout( + d->ev_wq, +- g5p_ew_woken(seen, READ_ONCE(d->event_serial)), ++ g5p_ew_woken(seen, READ_ONCE(d->event_serial)) || ++ (late_seen && ++ atomic_read(&d->ahead_poke) != *late_seen), + us_to_ktime(wait_us))) { + g5p_l123.ew[1]++; /* -ETIME: the safety net */ + } else { +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +index 3b3ce4b17..744bfc317 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -44,6 +44,7 @@ static void g5p_rjob_dep_cb(struct dma_fence *fence, struct dma_fence_cb *cb) + + WRITE_ONCE(job->cb_fired, true); + mod_delayed_work(job->gpu->render_wq, &job->gpu->rdev->sched_work, 0); ++ g5p_rv3_ahead_poke(job->gpu); /* F5c: a running 3D wait may start this head now */ + } + + /* Scheduler lock held: no callback of this job stays armed. */ +@@ -120,6 +121,7 @@ static void g5p_rsched_push(struct g5p_drm *d, struct g5p_render *r, + g5p_rs_push(&d->rdev->sched, &r->entity, &job->node); + spin_unlock_irqrestore(&d->rdev->sched_lock, flags); + g5p_rsched_kick(d); ++ g5p_rv3_ahead_poke(d); /* F5c: after the add; a running 3D wait may start it now */ + } + + /* The close item joins the context's entity behind its jobs; false if one is queued. */ +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.h b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +index db2af8af7..28450e320 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.h ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.h +@@ -62,6 +62,13 @@ struct g5p_rv3 { + bool ahead; + int ahead_ret; + bool ta_disarmed; /* F5: the TA latch was disarmed at the TA decision */ ++ /* F5c (render_user_ahead_on_submit): this job's 3D wait re-tries the early start of its ++ * context's next job when the device's ahead_poke moves past late_poke (a push, or a queued ++ * head's input signalled). Set at the TA decision, cleared once a job is started ahead or ++ * the 3D wait ends. Worker only. ++ */ ++ bool late; ++ int late_poke; + bool pm2; /* F5: the job's PM is its context's second set */ + bool touched; /* the DC3 pair was published: the firmware may reference the job's memory */ + /* The bind of this job's context published its DC3 pair and then failed: that context is +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 85cb08d38..0e308c96b 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -76,6 +76,54 @@ static const uint render_user_pipeline_depth = 1; + */ + #define G5P_RV3_AHEAD_CHAIN 8U + ++/* ++ * F5c: F5 decides once, at job N's TA completion; a job pushed (or whose inputs signal) while N's ++ * 3D still runs then waited for N's retire. With this set, N's 3D wait re-runs that same decision ++ * (g5p_rv3_ahead(), the worker, core lock held) whenever a push or a queued head's signalled input ++ * bumps d->ahead_poke, until a job is started or N's 3D is seen done. Apple kicks on submit ++ * (AGXChannelQueue_submitBlockInvoke). A no-op at render_user_pipeline_depth 1. ++ */ ++static const bool render_user_ahead_on_submit = true; ++ ++/* F5c: a job joined a FIFO, or a queued head's input signalled. Any context: the submit path, ++ * or a fence callback under the fence lock (possibly hard IRQ): a counter and a wake only, never ++ * the start itself. The full barrier orders the FIFO add (or the signal) before the armed look, ++ * against the executor's armed store before its FIFO look (g5p_rv3_late_arm()). ++ */ ++static void g5p_rv3_ahead_poke(struct g5p_drm *d) ++{ ++ smp_mb(); ++ if (!READ_ONCE(d->ahead_late_armed)) ++ return; ++ atomic_inc(&d->ahead_poke); ++ wake_up(&d->ev_wq); ++} ++ ++/* F5c, worker: job v's 3D wait may re-try the early start (render_user_ahead_on_submit). Before ++ * the TA-decision try, so that try is the first look after arming: no poke can be missed. ++ */ ++static void g5p_rv3_late_arm(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ if (!v->depth2 || !READ_ONCE(render_user_ahead_on_submit)) ++ return; ++ WRITE_ONCE(d->ahead_late_armed, true); ++ smp_mb(); ++ v->late_poke = atomic_read(&d->ahead_poke); ++ v->late = true; ++} ++ ++static void g5p_rv3_late_disarm(struct g5p_drm *d, struct g5p_rv3 *v) ++{ ++ if (!v->late) ++ return; ++ v->late = false; ++ WRITE_ONCE(d->ahead_late_armed, false); ++} ++ ++struct g5p_rjob; ++static void g5p_rv3_ahead_late(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *cur, const struct g5p_rv3_ev *s); ++ + /* drm-render.inc / drm-render-sched.inc, included after this file */ + static void g5p_render_cache(struct g5p_drm *d, struct g5p_rjob *job, + bool to_cpu); +@@ -660,11 +708,14 @@ static void g5p_rv3_np_end(struct g5p_diag *g, u32 e, u32 seq, u32 result) + * touch instant is taken after a step that left the phase undecided, before the sleep (off: never). + * The predicate is level-triggered: every sample, including the one right after each sleep and the + * closing one at the deadline, re-reads the whole RAM state (rt: the 3D, ta: the TA). ++ * late (F5c, the executor's own 3D wait of a depth-2 job; else NULL): after a sample that left the ++ * phase undecided, re-try the early start of the context's next job if poked (g5p_rv3_ahead_late()). + */ + static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, + u32 seq, struct g5p_ru_wait *w, struct g5p_ru_touch *t, + struct g5p_ru_retire *rt, struct g5p_ru_ta *ta, +- struct g5p_rv3_ev *s, u64 *consumed_ns, int *err) ++ struct g5p_rv3_ev *s, u64 *consumed_ns, int *err, ++ struct g5p_rjob *late) + { + struct g5p_drm *d = r->ctx->gpu; + bool closing = false, slow = false, sampled = false; +@@ -722,7 +773,11 @@ static u32 g5p_rv3_wait(struct g5p_diag *g, struct g5p_render *r, u32 e, + } + if (g5p_ru_touch_due(t, s->ns, w->result, w->expired)) + g5p_rv3_touch(g, t, s->ns); +- g5p_l123_pause(d, g, e, s, seen); ++ if (late && !fault) ++ g5p_rv3_ahead_late(d, r, late, s); ++ g5p_l123_pause(d, g, e, s, seen, ++ late && late->rv3.late ? &late->rv3.late_poke : ++ NULL); + } + } + +@@ -2999,10 +3054,12 @@ out: + + /* F5: after job `cur`'s TA completed, start its context's next job if it may (core lock held). + * The second PM set is made ready (taken and built at the context's first pipelined job) only +- * when there is a next job to use it. ++ * when there is a next job to use it. late (F5c): the same decision re-run from cur's 3D wait ++ * (g5p_rv3_ahead_late()); it keeps cur's TA-decision why and F5's skip count, and marks a start ++ * (cur->jt.late, the started job's flags bit 4). + */ + static void g5p_rv3_ahead(struct g5p_drm *d, struct g5p_render *r, +- struct g5p_rjob *cur) ++ struct g5p_rjob *cur, bool late) + { + struct g5p_rdev *rd = d->rdev; + bool pm2 = !r->js_pm2[g5p_rv3_js(rd, cur->rv3.seq)]; +@@ -3015,18 +3072,61 @@ static void g5p_rv3_ahead(struct g5p_drm *d, struct g5p_render *r, + else if (pm2 && !g5p_rv3_pm2_ready(d, r, cur->rv3.seq + 1)) why = 13; + else why = 0; + if (why) { +- cur->jt.why = why; +- rd->ahead_skipped++; ++ if (!late) { ++ cur->jt.why = why; ++ rd->ahead_skipped++; ++ } + return; + } + next = g5p_rv3_ahead_next(d, r, true, NULL); + if (!next) { +- cur->jt.why = 14; +- rd->ahead_skipped++; ++ if (!late) { ++ cur->jt.why = 14; ++ rd->ahead_skipped++; ++ } + return; + } +- cur->jt.why = 1; ++ if (late) ++ cur->jt.late = 1; ++ else ++ cur->jt.why = 1; + g5p_rv3_ahead_start(d, r, cur, next); ++ if (late) ++ next->jt.flags |= BIT(4); ++} ++ ++/* ++ * F5c: from job cur's own 3D wait (worker, core lock held, after a sample that left the phase ++ * undecided and saw no fault), when d->ahead_poke moved: re-run g5p_rv3_ahead() as at the TA ++ * decision. Race-free against the 3D's completion and the pick: all of it runs in the one worker, ++ * inside cur's execute, before the wait can return, so the scheduler cannot pick meanwhile; the ++ * head leaves its FIFO only through g5p_rv3_ahead_next(pop) under sched_lock, and rd->ahead set ++ * makes every further try a no-op (g5p_rv3_ahead_why() 4, and the arm is dropped). A 3D seen done ++ * (stamps landed in the last sample, or its completion event latched) drops the arm instead: the ++ * wait retires cur and the worker picks the next job normally. ++ */ ++static void g5p_rv3_ahead_late(struct g5p_drm *d, struct g5p_render *r, ++ struct g5p_rjob *cur, const struct g5p_rv3_ev *s) ++{ ++ struct g5p_rv3 *v = &cur->rv3; ++ int poke; ++ ++ if (!v->late) ++ return; ++ poke = atomic_read(&d->ahead_poke); ++ if (poke == v->late_poke) ++ return; ++ v->late_poke = poke; ++ smp_mb(); /* the poke read before the FIFO look: a later push pokes again */ ++ if (s->retired || s->event || d->rdev->ahead || ++ d->rdev->sequence != v->seq || ++ READ_ONCE(d->td_hit_ns[g5p_rv3_td_latch(g5p_ru_endpoint(v->seq))])) { ++ g5p_rv3_late_disarm(d, v); ++ return; ++ } ++ g5p_rv3_ahead(d, r, cur, true); ++ if (d->rdev->ahead) ++ g5p_rv3_late_disarm(d, v); + } + + static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, +@@ -3201,7 +3301,7 @@ ta_wait: + g5p_ru_touch_start(&v->touch[0], v->ta_pub_ns, r->touch_ms, seq); + g5p_ru_ta_start(&v->tr); + res = g5p_rv3_wait(g, r, 0, seq, &v->ta, &v->touch[0], NULL, &v->tr, +- &v->last[0], &v->ta_consumed_ns, &err); ++ &v->last[0], &v->ta_consumed_ns, &err, NULL); + r->consumed[0] = v->ta_consumed_ns != 0; + if (res != G5P_RU_WAIT_COMPLETE) { + struct g5p_rv3_ev late; +@@ -3257,7 +3357,10 @@ ta_wait: + */ + v->ta_event_ns = g5p_rv3_event_disarm(d, 0, seq, &job->jt); + v->ta_disarmed = true; +- g5p_rv3_ahead(d, r, job); ++ g5p_rv3_late_arm(d, v); /* F5c: before the first look */ ++ g5p_rv3_ahead(d, r, job, false); ++ if (rd->ahead) ++ g5p_rv3_late_disarm(d, v); + } + if (!v->joint) { + v->phase = G5P_RV3_TD_PUBLISH; +@@ -3276,7 +3379,9 @@ ta_wait: + g5p_ru_touch_start(&v->touch[1], v->td_pub_ns, r->touch_ms, seq); + g5p_ru_rt_start(&v->rt, r->settle_ms); + res = g5p_rv3_wait(g, r, 1, seq, &v->td, &v->touch[1], &v->rt, NULL, +- &v->last[1], &v->td_consumed_ns, &err); ++ &v->last[1], &v->td_consumed_ns, &err, ++ v->late ? job : NULL); ++ g5p_rv3_late_disarm(d, v); /* F5c: from here the worker picks the next job */ + /* F5: once a later job is prepared, the context's flags are that job's */ + if (rd->sequence == seq) + r->consumed[1] = v->td_consumed_ns != 0; +@@ -3336,6 +3441,7 @@ ta_wait: + 0; /* a job finished: a later fault starts a new streak */ + ret = 0; + out: ++ g5p_rv3_late_disarm(d, v); + if (v->phase >= G5P_RV3_TA_PUBLISH && !v->ta_disarmed) + v->ta_event_ns = g5p_rv3_event_disarm(d, 0, seq, &job->jt); + if (v->joint || v->phase >= G5P_RV3_TD_PUBLISH) +@@ -3452,7 +3558,7 @@ again: + g5p_ru_ta_start(&v->tr); + res = g5p_rv3_wait(g, r, 0, s->seq, &w, &v->touch[0], + NULL, &v->tr, &v->last[0], +- &v->ta_consumed_ns, &err); ++ &v->ta_consumed_ns, &err, NULL); + v->ta_event_ns = g5p_rv3_event_disarm(d, 0, s->seq, NULL); + } else { + if (!armed) +@@ -3461,7 +3567,7 @@ again: + g5p_ru_rt_start(&v->rt, r->settle_ms); + res = g5p_rv3_wait(g, r, 1, s->seq, &w, &v->touch[1], + &v->rt, NULL, &v->last[1], +- &v->td_consumed_ns, &err); ++ &v->td_consumed_ns, &err, NULL); + v->td_event_ns = g5p_rv3_event_disarm(d, 1, s->seq, NULL); + } + act = g5p_ru_drain_step(&st, res); diff --git a/kernel/patches/drivers/drm/0022-drm-apple-g5p-gate-TA-PM-records-on-per-set-dirty-st.patch b/kernel/patches/drivers/drm/0022-drm-apple-g5p-gate-TA-PM-records-on-per-set-dirty-st.patch new file mode 100644 index 00000000..22153a3e --- /dev/null +++ b/kernel/patches/drivers/drm/0022-drm-apple-g5p-gate-TA-PM-records-on-per-set-dirty-st.patch @@ -0,0 +1,220 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:47 +0000 +Subject: [PATCH] drm/apple-g5p: gate TA PM records on per-set dirty state + +[Orchard: from brkosebay/linux 797a84934ec054f4a9d6581cd363399cc1ef2630; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-layout.h | 21 ++++++++++---- + .../gpu/drm/apple-g5p/drm-render-unbind.inc | 13 +++++++++ + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 28 ++++++++++++++++--- + drivers/gpu/drm/apple-g5p/drm-render.inc | 12 ++++++++ + 4 files changed, 64 insertions(+), 10 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-layout.h b/drivers/gpu/apple-g5p/render-user-layout.h +index 675e9fbe6..aa8eb52e9 100644 +--- a/drivers/gpu/apple-g5p/render-user-layout.h ++++ b/drivers/gpu/apple-g5p/render-user-layout.h +@@ -474,7 +474,7 @@ static inline u32 g5p_ru_sku_at(u32 js, bool td) + static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, + bool td, u32 seq, const u8 *image, + u32 ta_finish, bool zero_timestamps, u32 js, +- u32 pmid, bool abs) ++ u32 pmid, bool abs, bool pm_record) + { + u32 w = meta + g5p_ru_w_at(js), state = pm + 0x80; + u32 finish = meta + G5P_R22_STAMP + 4; +@@ -616,11 +616,20 @@ static inline void g5p_ru_job_at(u8 *p, u32 meta, u32 global, u32 pm, u32 slot, + g5p_put32(p + cursor, 0x40000017); + } + memset(p + cmd, 0, G5P_RU_CMD_BYTES); +- g5p_put32(p + cmd, td ? 4 : 6); +- g5p_put32(p + cmd + 4, td ? ta_finish : slot); +- g5p_put32(p + cmd + 8, +- td ? seq : pmid); /* tag6 {6, STA, PMid, M(control)} */ +- g5p_put32(p + cmd + 12, td ? seq & 0x7fffffff : pm); ++ /* A clean PM needs only the already satisfied previous-TA dependency. */ ++ if (td || !pm_record) { ++ u32 required = td ? seq : seq - 1; ++ ++ g5p_put32(p + cmd, 4); ++ g5p_put32(p + cmd + 4, ta_finish); ++ g5p_put32(p + cmd + 8, required); ++ g5p_put32(p + cmd + 12, required & 0x7fffffff); ++ } else { ++ g5p_put32(p + cmd, 6); ++ g5p_put32(p + cmd + 4, slot); ++ g5p_put32(p + cmd + 8, pmid); ++ g5p_put32(p + cmd + 12, pm); ++ } + g5p_put32(p + G5P_WORK_RING + ring * 4, meta + cmd); + g5p_put32(p + G5P_WORK_RING + ((ring + 1) % G5P_WORK_SLOTS) * 4, w); + } +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +index 36550c443..92d1a57f7 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -65,6 +65,19 @@ static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, + why, n, cons, start, rd->dc_pending); + return -EBUSY; + } ++/* Invalidate PM-record reuse before any unbind is published. */ ++ mutex_lock(&rd->lock); ++ { ++ struct g5p_render *r; ++ ++ list_for_each_entry(r, &rd->contexts, rdev_link) ++ for (u32 i = 0; i < n; i++) ++ if (r->slot[0] == slots[i] || r->slot[1] == slots[i]) { ++ r->pm_dirty = true; ++ r->pm2_dirty = true; ++ } ++ } ++ mutex_unlock(&rd->lock); + rd->dc_pending = false; + for (u32 i = 0; i < n; i++) { + g5p_render_slot_mem(b, slots[i], start + i, &m); +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 0e308c96b..81f346623 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -83,6 +83,9 @@ static const uint render_user_pipeline_depth = 1; + * bumps d->ahead_poke, until a job is started or N's 3D is seen done. Apple kicks on submit + * (AGXChannelQueue_submitBlockInvoke). A no-op at render_user_pipeline_depth 1. + */ ++/* F2.1: Apple's pending-PM-record rule, read per job; default keeps tag6. */ ++static const bool render_user_pm_record; ++ + static const bool render_user_ahead_on_submit = true; + + /* F5c: a job joined a FIFO, or a queued head's input signalled. Any context: the submit path, +@@ -407,7 +410,7 @@ static void g5p_rv3_fw_touch(struct g5p_boot *b, const struct g5p_render *render + * (seq - 1) % 2 of the channel page, so a job never rewrites the record the firmware used for + * the job before it; the pointer ring carries each job's addresses (render-user-layout.h). + * render_user_pm_once 1: the PM pool, list and F are built by the epoch's first attach only and +- * never rewritten by the host between jobs; the TA's tag6 record has the firmware rebuild F, ++ * never rewritten by the host between jobs; a TA PM record has the firmware rebuild F, + * as on iOS (PM created per queue once; docs/gpu/experiments/freelist-lifetime-production-fix.md). + */ + /* release8 default */ +@@ -2261,8 +2264,10 @@ static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, + for (u32 k = 0; k < G5P_PMSET_PAGES; k++) + g5p_table_clean_page(g, &page[k]); + dma_wmb(); +- if (!ret) ++ if (!ret) { + r->pm_epoch = rd->recoveries + 1; ++ r->pm_dirty = true; ++ } + dev_info( + g->dev, + "RENDER-V3 pm-ctx seq=%u slots=%u/%u how=%s set=%d id=%u blocks=%u meta=%#x " +@@ -2339,6 +2344,7 @@ static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) + if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { + ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); + } else if (!rd->pm_once) { ++ r->pm_dirty = true; + physical = kvcalloc(r->pm_blocks * 8, sizeof(*physical), + GFP_KERNEL); + ret = physical ? 0 : -ENOMEM; +@@ -2487,6 +2493,7 @@ static bool g5p_rv3_pm2_ready(struct g5p_drm *d, struct g5p_render *r, + rd->pm_set_next = (set + 1) % b->pm_sets; + } + r->pm2_epoch = rd->recoveries + 1; ++ r->pm2_dirty = true; + return true; + } + +@@ -3026,7 +3033,9 @@ static void g5p_rv3_ahead_start(struct g5p_drm *d, struct g5p_render *r, + g5p_rv3_pm_meta_of(r, pm2), r->slot[e], e, seq, + e ? job->td_w : job->ta_w, + g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, js, +- g5p_rv3_pm_id_of(rd, r, pm2), v->abs); ++ g5p_rv3_pm_id_of(rd, r, pm2), v->abs, ++ !READ_ONCE(render_user_pm_record) || ++ (pm2 ? r->pm2_dirty : r->pm_dirty)); + r->submitted[e] = false; + r->consumed[e] = false; + r->retired[e] = false; +@@ -3257,7 +3266,9 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + g5p_rv3_pm_meta(r), r->slot[e], e, seq, + e ? job->td_w : job->ta_w, + g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, +- g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r), v->abs); ++ g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r), v->abs, ++ !READ_ONCE(render_user_pm_record) || ++ r->pm_set < 0 || r->pm_dirty); + r->submitted[e] = false; + r->consumed[e] = false; + r->retired[e] = false; +@@ -3343,6 +3354,13 @@ ta_wait: + g5p_r74_mark(g, G5P_R74_TA_RAM_TIMEOUT, ret); + goto out; + } ++/* Keep dirty on every pre-consumption failure. Completion also proves that ++ * the previous-TA stamp is available before this set can omit tag6. ++ */ ++ if (v->pm2) ++ r->pm2_dirty = false; ++ else ++ r->pm_dirty = false; + g5p_rv3_ta_count(g, d, v); + r->retired[0] = true; + g5p_rv3_terminal(g, G5P_RU_TERM_COMPLETE, "ta-complete"); +@@ -3697,6 +3715,8 @@ static bool g5p_rv3_recover(struct g5p_drm *d) + mutex_lock(&rd->lock); + list_for_each_entry(r, &rd->contexts, rdev_link) { + WRITE_ONCE(r->bound, false); ++ r->pm_dirty = true; ++ r->pm2_dirty = true; + count++; + } + mutex_unlock(&rd->lock); +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +index d51dc945a..4c6416ecb 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -312,6 +312,7 @@ struct g5p_render { + */ + int pm_set; + u32 pm_blocks, pm_epoch; ++ bool pm_dirty; /* PM record pending until a TA completes */ + struct g5p_dma_page *pm_pool; + struct g5p_dma_page pm_aux[G5P_RU_AUX_PAGES]; + bool pm_aux_mapped; +@@ -325,6 +326,7 @@ struct g5p_render { + */ + int pm2_set; + u32 pm2_blocks, pm2_epoch; ++ bool pm2_dirty; + struct g5p_dma_page *pm2_pool; + struct g5p_dma_page pm2_aux[G5P_RU_AUX_PAGES]; + struct drm_mm_node aux2_node; +@@ -480,6 +482,14 @@ static int g5p_rdev_grow(struct g5p_drm *d, u32 blocks) + kvfree(rd->physical); + rd->physical = physical; + rd->blocks = blocks; ++ { ++ struct g5p_render *r; ++ ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ r->pm_dirty = true; ++ r->pm2_dirty = true; ++ } ++ } + return 0; + } + +@@ -1491,6 +1501,8 @@ static int g5p_render_create_context(struct drm_device *drm, + } + r->pm_set = -1; + r->pm2_set = -1; ++ r->pm_dirty = true; ++ r->pm2_dirty = true; + c->gpu = d; + c->render = r; + r->ctx = c; diff --git a/kernel/patches/drivers/drm/0023-drm-apple-g5p-default-to-the-measured-pipeline-confi.patch b/kernel/patches/drivers/drm/0023-drm-apple-g5p-default-to-the-measured-pipeline-confi.patch new file mode 100644 index 00000000..b9f91621 --- /dev/null +++ b/kernel/patches/drivers/drm/0023-drm-apple-g5p-default-to-the-measured-pipeline-confi.patch @@ -0,0 +1,57 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 15:07:47 +0000 +Subject: [PATCH] drm/apple-g5p: default to the measured pipeline + configuration + +[Orchard: from brkosebay/linux 78775ea3b9ebd50647653d6e3b115447e5e75f9d; +fixed source defaults, no production module controls.] Preserve the +measured opt1 chain in dependency order. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 81f346623..c6c895756 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -54,7 +54,7 @@ static const bool render_user_ordered_waits = true; + * recovery bridge (terminal note and the observation-only CAPTURE entry); it is nothing without it. + */ + +-static const bool render_user_abs_stamps; ++static const bool render_user_abs_stamps = true; /* opt1: measured configuration */ + + /* + * F5, depth-2 job pipelining (docs/gpu/lane-a2-ta-done-latency-spec.md §4.3; latched per job at +@@ -69,7 +69,7 @@ static const bool render_user_abs_stamps; + * held from N's TA publish to N+1's 3D retire (Henrik's 0004: TA and 3D of one job adjacent, + * no other context's phase between them). + */ +-static const uint render_user_pipeline_depth = 1; ++static const uint render_user_pipeline_depth = 2; + + /* F5: consecutive ahead starts before the executor lets one job end with the core lock released, + * so a VM bind or an idle step waiting for that lock is never starved by a busy context. +@@ -83,8 +83,8 @@ static const uint render_user_pipeline_depth = 1; + * bumps d->ahead_poke, until a job is started or N's 3D is seen done. Apple kicks on submit + * (AGXChannelQueue_submitBlockInvoke). A no-op at render_user_pipeline_depth 1. + */ +-/* F2.1: Apple's pending-PM-record rule, read per job; default keeps tag6. */ +-static const bool render_user_pm_record; ++/* F2.1: Apple's pending-PM-record rule, read per job; 0 always emits tag6. */ ++static const bool render_user_pm_record = true; + + static const bool render_user_ahead_on_submit = true; + +@@ -365,7 +365,7 @@ static int g5p_rv3_errno(struct g5p_diag *g, int ret) + * 64-byte line, before every RAM sample of an in-flight phase. RAM only, and an idle GPU + * has no sampler, so it costs nothing then. + */ +-static const bool render_user_joint; ++static const bool render_user_joint = true; + static const unsigned int render_user_fw_ready_mode = 2; + + static void g5p_rv3_fw_touch(struct g5p_boot *b, const struct g5p_render *render) diff --git a/kernel/patches/drivers/drm/0024-drm-apple-g5p-refuse-system-sleep-without-retained-r.patch b/kernel/patches/drivers/drm/0024-drm-apple-g5p-refuse-system-sleep-without-retained-r.patch new file mode 100644 index 00000000..0f43a914 --- /dev/null +++ b/kernel/patches/drivers/drm/0024-drm-apple-g5p-refuse-system-sleep-without-retained-r.patch @@ -0,0 +1,100 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:02:37 +0000 +Subject: [PATCH] drm/apple-g5p: refuse system sleep without retained resume + +The shutdown quiesce is one-shot and restores platform power. It cannot +safely stand in for a normal sleep callback. Refuse system sleep before +device suspension until a stable-off and retained firmware resume +contract is available. Expose a fail-closed readiness query for the +platform pre-sync veto; a positive query will require a reservation +protocol. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 25 +++++++++++++++++++++++++ + include/linux/apple-g5p-pm.h | 25 +++++++++++++++++++++++++ + 2 files changed, 50 insertions(+) + create mode 100644 include/linux/apple-g5p-pm.h + +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index f184afb68..6ee3990b2 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -4,6 +4,7 @@ + #include "../../apple-g5p/drm-validate.h" + #include "../../apple-g5p/render-user-evwait.h" + #include "../../apple-g5p/render-user-sync.h" ++#include + #include + #include + #include +@@ -950,6 +951,29 @@ static const struct of_device_id g5p_drm_match[] = { + }; + MODULE_DEVICE_TABLE(of, g5p_drm_match); + ++/* ++ * Normal Apple power-off requires quiescent firmware and a stable-off ++ * certificate before removing platform power. The pinned firmware epoch does ++ * not yet supply that contract. Do not use the one-shot kexec cycle here: ++ * it reopens platform power and permanently closes submission admission. ++ */ ++bool apple_g5p_system_sleep_ready(struct device *dev) ++{ ++ /* Fail closed, including callers that cannot identify a bound device. */ ++ return false; ++} ++EXPORT_SYMBOL_GPL(apple_g5p_system_sleep_ready); ++ ++static int g5p_system_sleep_prepare(struct device *dev) ++{ ++ dev_warn(dev, "GPU-PM sleep refused: retained firmware resume unavailable\n"); ++ return -EBUSY; ++} ++ ++static const struct dev_pm_ops g5p_system_sleep_pm_ops = { ++ .prepare = g5p_system_sleep_prepare, ++}; ++ + static struct platform_driver g5p_drm_platform = { + .probe = g5p_drm_probe, + .remove = g5p_drm_remove, +@@ -958,6 +982,7 @@ static struct platform_driver g5p_drm_platform = { + .name = "apple-g5p-drm", + .of_match_table = g5p_drm_match, + .dev_groups = g5p_groups, ++ .pm = pm_sleep_ptr(&g5p_system_sleep_pm_ops), + .suppress_bind_attrs = true, + }, + }; +diff --git a/include/linux/apple-g5p-pm.h b/include/linux/apple-g5p-pm.h +new file mode 100644 +index 000000000..5b152b8fd +--- /dev/null ++++ b/include/linux/apple-g5p-pm.h +@@ -0,0 +1,25 @@ ++/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ ++#ifndef _LINUX_APPLE_G5P_PM_H ++#define _LINUX_APPLE_G5P_PM_H ++ ++#include ++#include ++ ++struct device; ++ ++/* ++ * For the platform pre-sync sleep veto. Missing support is never a promise ++ * that a bound GPU can sleep. Currently all bound G5P devices must be vetoed. ++ * A future positive result requires an admission reservation through abort ++ * or resume; a snapshot alone cannot authorize power removal. ++ */ ++#if IS_REACHABLE(CONFIG_DRM_APPLE_G5P) ++bool apple_g5p_system_sleep_ready(struct device *dev); ++#else ++static inline bool apple_g5p_system_sleep_ready(struct device *dev) ++{ ++ return false; ++} ++#endif ++ ++#endif /* _LINUX_APPLE_G5P_PM_H */ diff --git a/kernel/patches/drivers/drm/0025-drm-apple-g5p-implement-bounded-retained-system-slee.patch b/kernel/patches/drivers/drm/0025-drm-apple-g5p-implement-bounded-retained-system-slee.patch new file mode 100644 index 00000000..304e4d74 --- /dev/null +++ b/kernel/patches/drivers/drm/0025-drm-apple-g5p-implement-bounded-retained-system-slee.patch @@ -0,0 +1,890 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:17:18 +0000 +Subject: [PATCH] drm/apple-g5p: implement bounded retained system sleep + +Reserve reversible host admission before certifying quiescence and +stamps. Stop the Fender firmware and remove platform power only after +stable-off. Boot over retained shared memory and require a new ALIVE +generation before IRQ or submission admission. + +Rebuild host bindings and both pipelined PM epochs without treating +clean sleep as a fault. Freeze and error queued fences on a transition +failure; retain the busy refusal for unsupported states and hibernation. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/boot.inc | 21 +- + drivers/gpu/apple-g5p/core.inc | 47 ++- + drivers/gpu/apple-g5p/teardown.inc | 5 + + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 82 +++-- + .../gpu/drm/apple-g5p/drm-render-sched.inc | 6 + + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 110 ++++-- + drivers/gpu/drm/apple-g5p/drm-render.inc | 1 + + drivers/gpu/drm/apple-g5p/system-sleep.inc | 345 ++++++++++++++++++ + include/linux/apple-g5p-pm.h | 13 +- + 9 files changed, 556 insertions(+), 74 deletions(-) + create mode 100644 drivers/gpu/drm/apple-g5p/system-sleep.inc + +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +index 13267238b..286620e08 100644 +--- a/drivers/gpu/apple-g5p/boot.inc ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -843,6 +843,8 @@ static int g5p_boot_drain(struct g5p_diag *g) + if (words[0] == 0 && READ_ONCE(b->released) && + !READ_ONCE(b->stopped)) { + b->alive_word = words[1]; ++ if (g->sleep_boot && b->r73b) ++ b->r73b->r74f_alive++; + WRITE_ONCE(b->alive, true); + } + b->events++; +@@ -853,7 +855,7 @@ static int g5p_boot_drain(struct g5p_diag *g) + /* Return the ALIVE slot before ringing work. No logging or + * other MMIO may be inserted between these two operations. + */ +- if (!words[0] && g5p_r74f_on(b) && ++ if (!g->sleep_boot && !words[0] && g5p_r74f_on(b) && + READ_ONCE(b->r73b->owner) == G5P_R73B_OWNER_IRQ && + READ_ONCE(b->released) && !READ_ONCE(b->stopped)) + g5p_r74f_alive(g); +@@ -1305,10 +1307,11 @@ static int g5p_boot_program_stream(struct g5p_diag *g, bool release) + ret = g5p_boot_irq_setup(g); + if (ret) + return ret; +- g5p_boot_snapshot(g, release ? "before-release" : "before-arm", +- !g->reset_transition && !b->failed ? +- G5P_READBACK_BASELINE : +- G5P_READBACK_WRAPPER_ONLY); ++ if (!g->sleep_boot) ++ g5p_boot_snapshot(g, release ? "before-release" : "before-arm", ++ !g->reset_transition && !b->failed ? ++ G5P_READBACK_BASELINE : ++ G5P_READBACK_WRAPPER_ONLY); + ; + ret = 0; + if (ret) +@@ -1418,6 +1421,12 @@ static int g5p_boot_program_stream(struct g5p_diag *g, bool release) + if (b->seed_stamp) + b->seed_gpc1 = arch_timer_read_counter(); + } ++ if (g->sleep_boot) { ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_fender_write(g, 0x1000, 0x11); ++ return ret; ++ } + ret = release && b->r73b && 1 ? g5p_r74e_fender_start(g) : + release ? g5p_fender_start_observed_q(&io, g5p_release_observe, + b->perf_poweron ?: 1) : +@@ -1454,6 +1463,8 @@ static int g5p_boot_program_stream(struct g5p_diag *g, bool release) + b->armed = true; + return 0; + out: ++ if (g->sleep_boot) ++ return ret; + /* A slow r74e epoch is scoreable without another unsafe MMIO read. */ + if (release && b->r73b && 1 && READ_ONCE(b->r73b->early_power_abort)) + return ret; +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +index 709efed16..9177065d4 100644 +--- a/drivers/gpu/apple-g5p/core.inc ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -80,6 +80,8 @@ struct g5p_diag { + struct regmap *pmgr; + /* Serializes commands and all hardware access. */ + struct mutex lock; ++ atomic_t sleep_gate; /* closed bit plus active host operations */ ++ bool sleep_reserved, sleep_off, sleep_boot, sleep_failed, sleep_irq_off; + bool powered; + bool loaded; + +@@ -104,6 +106,28 @@ struct g5p_diag { + struct g5p_mmio_self *mmio_self; + }; + ++/* Separate from terminal retirement: refusal must not cancel accepted jobs. */ ++static bool g5p_sleep_enter(struct g5p_diag *g) ++{ ++ int old = atomic_read(&g->sleep_gate); ++ ++ for (;;) { ++ int seen; ++ ++ if (old < 0) ++ return false; ++ seen = atomic_cmpxchg(&g->sleep_gate, old, old + 1); ++ if (seen == old) ++ return true; ++ old = seen; ++ } ++} ++ ++static void g5p_sleep_exit(struct g5p_diag *g) ++{ ++ atomic_dec(&g->sleep_gate); ++} ++ + /* Before every PMGR gfx gate (boot R33 off, dirty-start, kexec quiesce and in-boot recovery + * cycles): no accessor MMIO from here until g5p_fender_wake_once() brings Fender up again. + * Apple clears fender+0xc inside setPlatformPower(0), before its gate (G5). +@@ -450,7 +474,7 @@ static DEVICE_ATTR_RO(result); + /* One bring-up phase, as the command file and the probe-time bring-up run it. *ran is set + * when the phase itself ran (and its result was recorded). + */ +-static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) ++static int g5p_run_phase_locked(struct g5p_diag *g, const char *buf, bool *ran) + { + int ret; + +@@ -486,6 +510,18 @@ static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) + return ret; + } + ++static int g5p_run_phase(struct g5p_diag *g, const char *buf, bool *ran) ++{ ++ int ret; ++ ++ *ran = false; ++ if (!g5p_sleep_enter(g)) ++ return -EBUSY; ++ ret = g5p_run_phase_locked(g, buf, ran); ++ g5p_sleep_exit(g); ++ return ret; ++} ++ + static ssize_t command_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) + { +@@ -495,8 +531,13 @@ static ssize_t command_store(struct device *dev, struct device_attribute *attr, + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; +- if (sysfs_streq(buf, "td-kexec")) +- return g5p_td_command(g, buf) ?: count; ++ if (sysfs_streq(buf, "td-kexec")) { ++ if (!g5p_sleep_enter(g)) ++ return -EBUSY; ++ ret = g5p_td_command(g, buf); ++ g5p_sleep_exit(g); ++ return ret ?: count; ++ } + ret = g5p_run_phase(g, buf, &ran); + if (ran) + dev_info(dev, "command %.*s result=%d\n", (int)count, buf, ret); +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +index e3875bc59..d40cfa26f 100644 +--- a/drivers/gpu/apple-g5p/teardown.inc ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -565,6 +565,11 @@ static int g5p_kq_run(struct g5p_diag *g, const char *why) + struct g5p_kq_ctx *c; + u64 t0 = ktime_get_ns(); + ++ if (READ_ONCE(g->sleep_failed)) { ++ dev_emerg(g->dev, ++ "GPU-KQ frozen after sleep failure; no MMIO\n"); ++ return -EIO; ++ } + if (atomic_xchg(&g5p_kq_ran, 1)) { + dev_emerg( + g->dev, +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 6ee3990b2..5aec0de6c 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -674,7 +674,8 @@ static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) + return 0; + } + +-static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++static void g5p_drm_postclose_host(struct drm_device *drm, ++ struct drm_file *file) + { + struct g5p_file *f = file->driver_priv; + struct g5p_context *c; +@@ -723,6 +724,16 @@ static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) + READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); + } + ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_drm *d = container_of(drm, struct g5p_drm, drm); ++ ++ while (!g5p_sleep_enter(d->core)) ++ msleep(20); ++ g5p_drm_postclose_host(drm, file); ++ g5p_sleep_exit(d->core); ++} ++ + static const struct drm_ioctl_desc g5p_ioctls[] = { + DRM_IOCTL_DEF_DRV(G5P_GET_INFO, g5p_get_info, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(G5P_CREATE_BO, g5p_create_bo, DRM_RENDER_ALLOW), +@@ -756,7 +767,51 @@ static const struct drm_ioctl_desc g5p_ioctls[] = { + DRM_RENDER_ALLOW), + }; + +-DEFINE_DRM_GEM_FOPS(g5p_drm_fops); ++static long g5p_sleep_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg) ++{ ++ struct drm_file *file = filp->private_data; ++ struct g5p_drm *d = container_of(file->minor->dev, struct g5p_drm, drm); ++ long ret; ++ ++ if (!g5p_sleep_enter(d->core)) ++ return -EBUSY; ++ ret = drm_ioctl(filp, cmd, arg); ++ g5p_sleep_exit(d->core); ++ return ret; ++} ++ ++#ifdef CONFIG_COMPAT ++static long g5p_sleep_compat_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg) ++{ ++ struct drm_file *file = filp->private_data; ++ struct g5p_drm *d = container_of(file->minor->dev, struct g5p_drm, drm); ++ long ret; ++ ++ if (!g5p_sleep_enter(d->core)) ++ return -EBUSY; ++ ret = drm_compat_ioctl(filp, cmd, arg); ++ g5p_sleep_exit(d->core); ++ return ret; ++} ++#endif ++ ++static const struct file_operations g5p_drm_fops = { ++ .owner = THIS_MODULE, ++ .open = drm_open, ++ .release = drm_release, ++ .unlocked_ioctl = g5p_sleep_ioctl, ++#ifdef CONFIG_COMPAT ++ .compat_ioctl = g5p_sleep_compat_ioctl, ++#endif ++ .poll = drm_poll, ++ .read = drm_read, ++ .llseek = noop_llseek, ++ .get_unmapped_area = drm_gem_get_unmapped_area, ++ .mmap = drm_gem_mmap, ++ .fop_flags = FOP_UNSIGNED_OFFSET, ++}; + + static const struct drm_driver g5p_drm_driver = { + .debugfs_init = g5p_jt_debugfs_init, +@@ -951,28 +1006,7 @@ static const struct of_device_id g5p_drm_match[] = { + }; + MODULE_DEVICE_TABLE(of, g5p_drm_match); + +-/* +- * Normal Apple power-off requires quiescent firmware and a stable-off +- * certificate before removing platform power. The pinned firmware epoch does +- * not yet supply that contract. Do not use the one-shot kexec cycle here: +- * it reopens platform power and permanently closes submission admission. +- */ +-bool apple_g5p_system_sleep_ready(struct device *dev) +-{ +- /* Fail closed, including callers that cannot identify a bound device. */ +- return false; +-} +-EXPORT_SYMBOL_GPL(apple_g5p_system_sleep_ready); +- +-static int g5p_system_sleep_prepare(struct device *dev) +-{ +- dev_warn(dev, "GPU-PM sleep refused: retained firmware resume unavailable\n"); +- return -EBUSY; +-} +- +-static const struct dev_pm_ops g5p_system_sleep_pm_ops = { +- .prepare = g5p_system_sleep_prepare, +-}; ++#include "system-sleep.inc" + + static struct platform_driver g5p_drm_platform = { + .probe = g5p_drm_probe, +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +index 744bfc317..c2463af31 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -153,6 +153,10 @@ static void g5p_rsched_work(struct work_struct *work) + struct g5p_rdev, sched_work); + struct g5p_drm *d = rd->drm; + ++ bool sleep_user = g5p_sleep_enter(d->core); ++ ++ if (!sleep_user && !READ_ONCE(d->core->sleep_failed)) ++ return; + /* Each item drops the references it held; the device must outlive this loop. */ + drm_dev_get(&d->drm); + for (;;) { +@@ -216,4 +220,6 @@ static void g5p_rsched_work(struct work_struct *work) + drm_dev_put(&d->drm); + } + drm_dev_put(&d->drm); ++ if (sleep_user) ++ g5p_sleep_exit(d->core); + } +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index c6c895756..7b4199cd4 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -2265,7 +2265,7 @@ static int g5p_rv3_pmctx_build(struct g5p_drm *d, struct g5p_render *r, u32 seq, + g5p_table_clean_page(g, &page[k]); + dma_wmb(); + if (!ret) { +- r->pm_epoch = rd->recoveries + 1; ++ r->pm_epoch = rd->recoveries + rd->sleep_rebuilds + 1; + r->pm_dirty = true; + } + dev_info( +@@ -2341,7 +2341,8 @@ static int g5p_rv3_pmctx_job(struct g5p_drm *d, struct g5p_render *r, u32 seq) + return 0; + page = &b->pm_set[r->pm_set * G5P_PMSET_PAGES]; + mutex_lock(&rd->lock); +- if (r->pm_epoch != rd->recoveries + 1 || r->pm_blocks < rd->blocks) { ++ if (r->pm_epoch != rd->recoveries + rd->sleep_rebuilds + 1 || ++ r->pm_blocks < rd->blocks) { + ret = g5p_rv3_pmctx_build(d, r, seq, "rebuild"); + } else if (!rd->pm_once) { + r->pm_dirty = true; +@@ -2412,7 +2413,7 @@ static bool g5p_rv3_pm2_ready(struct g5p_drm *d, struct g5p_render *r, + + if (r->pm2_failed || !r->aux2_mapped || r->pm_set < 0) + return false; +- if (!fresh && r->pm2_epoch == rd->recoveries + 1 && ++ if (!fresh && r->pm2_epoch == rd->recoveries + rd->sleep_rebuilds + 1 && + r->pm2_blocks >= rd->blocks) + return true; + if (fresh) +@@ -2492,7 +2493,7 @@ static bool g5p_rv3_pm2_ready(struct g5p_drm *d, struct g5p_render *r, + rd->pm_set_used[set] = true; + rd->pm_set_next = (set + 1) % b->pm_sets; + } +- r->pm2_epoch = rd->recoveries + 1; ++ r->pm2_epoch = rd->recoveries + rd->sleep_rebuilds + 1; + r->pm2_dirty = true; + return true; + } +@@ -3696,46 +3697,30 @@ static void g5p_render_fw_rebuild(struct g5p_drm *d) + * pool are zeroed as at their allocation. The held jobs run after this. Returns false, having + * done nothing, when the core lock is busy. A failed recovery leaves the device failed. + */ +-static bool g5p_rv3_recover(struct g5p_drm *d) ++/* Core -> rdev lock order. No user BO, VM contents or fence history is reset. ++ * Slots have been detached, or the failed firmware epoch has been replaced. ++ */ ++static u32 g5p_rv3_rebuild_host(struct g5p_drm *d, struct g5p_render **drop, ++ u32 count) + { + struct g5p_rdev *rd = d->rdev; + struct g5p_diag *g = d->core; +- unsigned long deadline = jiffies + 2 * HZ, flags; +- struct g5p_render **drop = NULL; + struct g5p_render *r; +- u32 n = 0, count = 0, dropped = 0, stage; +- int ret; ++ unsigned long flags; ++ u32 n = 0; + +- while (!mutex_trylock(&g->lock)) { +- if (time_after(jiffies, deadline)) +- return false; +- msleep(5); +- } +- /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ +- mutex_lock(&rd->lock); +- list_for_each_entry(r, &rd->contexts, rdev_link) { +- WRITE_ONCE(r->bound, false); +- r->pm_dirty = true; +- r->pm2_dirty = true; +- count++; +- } +- mutex_unlock(&rd->lock); +- drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); +- rd->recoveries++; +- rd->recover_streak++; +- ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; +- if (ret) { +- g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); +- WRITE_ONCE(rd->recovering, false); +- mutex_unlock(&g->lock); +- kfree(drop); +- dev_emerg( +- g->dev, +- "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", +- ret, rd->recoveries); +- return true; ++ lockdep_assert_held(&g->lock); ++ /* Sleep reuses the transport allocation. Retire its host-only one-shot ++ * doorbell state; firmware roots and ring cursors remain retained. ++ */ ++ if (g->sleep_reserved && g->boot->r73b) { ++ struct g5p_r73b *x = g->boot->r73b; ++ ++ spin_lock_irqsave(&g->boot->event_lock, flags); ++ memset(x->r74f_bell, 0, sizeof(x->r74f_bell)); ++ x->r74f_active = 0; ++ spin_unlock_irqrestore(&g->boot->event_lock, flags); + } +- g5p_ru_term_reset(&g5p_rv3_term); + /* the new firmware's PM has written nothing back yet: no context is its last or previous */ + rd->pm_last = NULL; + rd->pm_prev = NULL; +@@ -3773,6 +3758,11 @@ static bool g5p_rv3_recover(struct g5p_drm *d) + g5p_table_clean_page(g, &rd->physical[i]); + } + list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ r->pm_dirty = true; ++ r->pm2_dirty = true; ++ r->pm_epoch = 0; ++ r->pm2_epoch = 0; + r->jobs = 0; + r->dc_target = 0; + WRITE_ONCE(r->live_maps, false); +@@ -3788,6 +3778,50 @@ static bool g5p_rv3_recover(struct g5p_drm *d) + } + } + mutex_unlock(&rd->lock); ++ return n; ++} ++ ++static bool g5p_rv3_recover(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_diag *g = d->core; ++ unsigned long deadline = jiffies + 2 * HZ, flags; ++ struct g5p_render **drop = NULL; ++ struct g5p_render *r; ++ u32 n = 0, count = 0, dropped = 0, stage; ++ int ret; ++ ++ while (!mutex_trylock(&g->lock)) { ++ if (time_after(jiffies, deadline)) ++ return false; ++ msleep(5); ++ } ++ /* The old epoch's bindings are void from here: no edit may publish a tag into it. */ ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) { ++ WRITE_ONCE(r->bound, false); ++ r->pm_dirty = true; ++ r->pm2_dirty = true; ++ count++; ++ } ++ mutex_unlock(&rd->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ rd->recoveries++; ++ rd->recover_streak++; ++ ret = drop ? g5p_recover(g, "lost-job", &stage) : -ENOMEM; ++ if (ret) { ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "recover-failed"); ++ WRITE_ONCE(rd->recovering, false); ++ mutex_unlock(&g->lock); ++ kfree(drop); ++ dev_emerg( ++ g->dev, ++ "RENDER-V3 recover failed ret=%d n=%u: the epoch ends end=v3\n", ++ ret, rd->recoveries); ++ return true; ++ } ++ g5p_ru_term_reset(&g5p_rv3_term); ++ n = g5p_rv3_rebuild_host(d, drop, count); + spin_lock_irqsave(&d->status_lock, flags); + d->last_error = 0; + d->job_error = 0; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render.inc b/drivers/gpu/drm/apple-g5p/drm-render.inc +index 4c6416ecb..7effd5820 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render.inc +@@ -229,6 +229,7 @@ struct g5p_rdev { + */ + bool recovering; + u32 recoveries; ++ u32 sleep_rebuilds; + u32 recover_streak; /* recoveries since a job completed (core lock) */ + struct list_head + contexts; /* every render context (lock), for the recovery */ +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +new file mode 100644 +index 000000000..25b1e8b2c +--- /dev/null ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -0,0 +1,345 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* Apple 20H350: c81b18 -> c9fa10 -> ca0f78; wake c83b18. ++ * Admission is reserved through complete, including another device's abort. ++ * No terminal-retire latch is used for a reversible sleep attempt. ++ */ ++static bool g5p_sleep_idle(struct g5p_drm *d) ++{ ++ struct g5p_rdev *rd = d->rdev; ++ struct g5p_render *r; ++ unsigned long flags; ++ bool idle; ++ ++ if (rd->cu_sequence || atomic_read(&d->accepted) || ++ atomic_read(&d->r8_queued) || READ_ONCE(rd->executing) || ++ rd->ahead || rd->stuck || rd->recovering || rd->dc_pending || ++ rd->close_deferred[0] || rd->close_deferred[1] || ++ g5p_ru_term_failed(&g5p_rv3_term)) ++ return false; ++ spin_lock_irqsave(&rd->sched_lock, flags); ++ idle = list_empty(&rd->sched.run); ++ spin_unlock_irqrestore(&rd->sched_lock, flags); ++ if (!idle) ++ return false; ++ /* Re-read the latest TA/3D stamps, including the depth-2 record slot. ++ * A CPU retirement flag alone is not Apple's all-stamps certificate. ++ */ ++ if (rd->sequence) ++ for (u32 e = 0; e < 2; e++) ++ if (!g5p_ru_retired_pl(g5p_ru_page(d->core->boot, e), ++ g5p_ru_meta(e), e, rd->sequence, ++ g5p_rv3_js(rd, rd->sequence), ++ rd->sequence)) ++ return false; ++ mutex_lock(&rd->lock); ++ list_for_each_entry(r, &rd->contexts, rdev_link) ++ for (u32 i = 0; i < 2; i++) ++ if (r->submitted[i] && ++ (!r->consumed[i] || !r->retired[i])) ++ idle = false; ++ mutex_unlock(&rd->lock); ++ return idle; ++} ++ ++/* R1 only, as Apple's isQuiescent/isStableOff. Never infer bus retirement ++ * from a completed userspace fence or GPC=0 alone. ++ */ ++static bool g5p_sleep_certificate(struct g5p_diag *g, bool stable) ++{ ++ u32 state, mode; ++ ++ if (READ_ONCE(g->boot->failed) || READ_ONCE(g->boot->event_failed) || ++ READ_ONCE(g->boot->r73b->early_power_abort)) ++ return false; ++ state = readl(g->window[1] + 0x1000); ++ ++ if (state & 0x1f) ++ return false; ++ mode = readl(g->window[1]); ++ if (stable && mode >> 30 == 1) ++ return true; ++ return (mode & 0x60000000) == 0x60000000 && !readl(g->window[1] + 8); ++} ++ ++static bool g5p_sleep_rings(struct g5p_diag *g) ++{ ++ u32 cons, prod; ++ ++ for (u32 i = 0; i < G5P_TD_NRING; i++) ++ if (g5p_td_op_ring_read(g, i, &cons, &prod) || cons > 255 || ++ cons != prod) ++ return false; ++ return true; ++} ++ ++static void g5p_sleep_fail(struct g5p_drm *d, int ret) ++{ ++ struct g5p_diag *g = d->core; ++ ++ /* RAM only: no retry, ACK or diagnostic MMIO after a failed transition. */ ++ if (!g->sleep_irq_off && g->boot->irq_ready) { ++ disable_irq(g->boot->irq); ++ g->sleep_irq_off = true; ++ } ++ g->sleep_failed = true; ++ WRITE_ONCE(g->boot->failed, true); ++ WRITE_ONCE(g->boot->stopped, true); ++ g5p_mmio_gate(g); ++ g5p_rv3_terminal(g, G5P_RU_TERM_FAULT, "sleep-failed"); ++ g5p_r8_admit_close(&g5p_r8_state.admit); ++ g5p_rsched_kick( ++ d); /* pending fences fail, including unsignalled inputs */ ++ dev_err(g->dev, "GPU-PM frozen ret=%d; no further MMIO\n", ret); ++} ++ ++void apple_g5p_system_sleep_complete(struct device *dev) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ ++ if (!g || !g->sleep_reserved || g->sleep_off) ++ return; ++ g->sleep_reserved = false; ++ /* A failed epoch stays terminal, but host close must still be possible. */ ++ atomic_andnot(BIT(31), &g->sleep_gate); ++ g5p_rsched_kick(g->event_cookie); ++ dev_info(dev, "GPU-PM complete failed=%u\n", g->sleep_failed); ++} ++ ++EXPORT_SYMBOL_GPL(apple_g5p_system_sleep_complete); ++ ++static int g5p_system_sleep_prepare(struct device *dev) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_drm *d; ++ struct g5p_boot *b; ++ struct g5p_kq_ctx c = { .g = g }; ++ u64 deadline; ++ int ret = -EBUSY; ++ ++ if (!g || !g->event_cookie) ++ return -EBUSY; ++ if (READ_ONCE(g->sleep_reserved)) ++ return g->sleep_failed ? -EBUSY : 0; ++ if (atomic_fetch_or(BIT(31), &g->sleep_gate) < 0) ++ return -EBUSY; ++ deadline = ktime_get_ns() + 20 * NSEC_PER_MSEC; ++ while ((atomic_read(&g->sleep_gate) & INT_MAX) && ++ ktime_get_ns() < deadline) ++ usleep_range(100, 200); ++ if (atomic_read(&g->sleep_gate) & INT_MAX) ++ goto refuse; ++ if (!g5p_kq_lock(&c)) ++ goto refuse; ++ d = g->event_cookie; ++ b = g->boot; ++ if (g->sleep_failed || g5p_r8_admit_closed(&g5p_r8_state.admit) || !b || ++ !g->powered || !g->fender_ready || g->mmio_off || !b->released || ++ !b->alive || b->failed || b->event_failed || b->stopped || ++ !b->irq_ready || !b->r73b || ++ READ_ONCE(b->r73b->early_power_abort) || !g5p_sleep_idle(d)) ++ goto unlock; ++ /* No submitters, queued work or IRQ-side ALIVE action can publish now. */ ++ if (g5p_boot_drain(g) || !g5p_sleep_rings(g)) ++ goto unlock; ++ deadline = ktime_get_ns() + 20 * NSEC_PER_MSEC; ++ do { ++ if (g5p_sleep_certificate(g, false)) { ++ g->sleep_reserved = true; ++ ret = 0; ++ break; ++ } ++ usleep_range(100, 200); ++ } while (ktime_get_ns() < deadline); ++unlock: ++ mutex_unlock(&g->lock); ++ if (!ret) { ++ dev_info(dev, "GPU-PM prepared: admission reserved\n"); ++ return 0; ++ } ++refuse: ++ atomic_andnot(BIT(31), &g->sleep_gate); ++ g5p_rsched_kick(g->event_cookie); ++ dev_info(dev, ++ "GPU-PM sleep refused: busy or no quiescent certificate\n"); ++ return -EBUSY; ++} ++ ++/* The early caller reserves admission; PM complete releases it on abort/wake. ++ * This is deliberately stronger than an observation-only readiness query. ++ */ ++bool apple_g5p_system_sleep_ready(struct device *dev) ++{ ++ return !g5p_system_sleep_prepare(dev); ++} ++EXPORT_SYMBOL_GPL(apple_g5p_system_sleep_ready); ++ ++static int g5p_system_suspend(struct device *dev) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_drm *d = g->event_cookie; ++ struct g5p_boot *b = g->boot; ++ struct g5p_render **drop, *r; ++ u32 slots[G5P_RU_USER_SLOTS], held, count = 0, n = 0; ++ int ret; ++ ++ if (!g->sleep_reserved || g->sleep_failed) ++ return -EBUSY; ++ mutex_lock(&g->lock); ++ if (!READ_ONCE(b->alive) || READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || READ_ONCE(b->stopped) || ++ READ_ONCE(b->r73b->early_power_abort) || !g5p_sleep_idle(d) || ++ !g5p_sleep_rings(g) || !g5p_sleep_certificate(g, false)) { ++ ret = -EBUSY; ++ goto out; ++ } ++ mutex_lock(&d->rdev->lock); ++ list_for_each_entry(r, &d->rdev->contexts, rdev_link) ++ count++; ++ mutex_unlock(&d->rdev->lock); ++ drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); ++ if (!drop) { ++ ret = -EBUSY; ++ goto out; ++ } ++ held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), ++ g5p_boot_shared(b, 0x84), slots, ++ ARRAY_SIZE(slots)); ++ ret = held ? g5p_rv3_unbind_slots(d, slots, held, "sleep") : 0; ++ if (ret) ++ goto failed; ++ /* Detach has changed firmware ownership: any later failure is terminal. */ ++ if (!g5p_sleep_rings(g) || !g5p_sleep_certificate(g, false)) { ++ ret = -EBUSY; ++ goto failed; ++ } ++ ret = g5p_fender_write(g, 0x10, 0); /* Fender mode 3 */ ++ if (!ret) ++ ret = g5p_fender_write(g, 0x20000, 0); ++ if (!ret) ++ ret = g5p_fender_poll(g, 0x1000, 0x10, 0); ++ if (!ret) ++ ret = g5p_fender_write(g, 0x1000, 0x10); ++ WRITE_ONCE(b->alive, false); ++ WRITE_ONCE(b->stopped, true); ++ disable_irq(b->irq); ++ g->sleep_irq_off = true; ++ if (ret || !g5p_sleep_certificate(g, true)) { ++ ret = ret ?: -EIO; ++ goto failed; ++ } ++ n = g5p_rv3_rebuild_host(d, drop, count); ++ d->rdev->sleep_rebuilds++; ++ g5p_mmio_gate(g); ++ g->fender_ready = false; ++ ret = pm_runtime_put_sync_suspend(dev); ++ g->powered = false; /* put consumed our reference, including on error */ ++ if (ret < 0) ++ goto failed; ++ g->sleep_off = true; ++ dev_info(dev, "GPU-PM off: stable-off epoch=%u\n", ++ d->rdev->sleep_rebuilds); ++ ret = 0; ++ goto free; ++failed: ++ g5p_sleep_fail(d, ret); ++free: ++ mutex_unlock(&g->lock); ++ for (u32 i = 0; i < n; i++) { ++ g5p_context_put(drop[i]->ctx); ++ drm_dev_put(&d->drm); ++ } ++ kfree(drop); ++ return ret; ++out: ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_system_resume(struct device *dev) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_drm *d = g->event_cookie; ++ struct g5p_boot *b = g->boot; ++ unsigned long deadline; ++ int ret; ++ ++ if (!g->sleep_off) ++ return g->sleep_failed ? -EIO : 0; ++ mutex_lock(&g->lock); ++ ret = pm_runtime_resume_and_get(dev); ++ if (ret < 0) ++ goto failed; ++ g->powered = true; ++ g->sleep_boot = true; ++ /* A witnessed platform off/on invalidates DS108's old ALIVE generation. ++ * Retained maps/image are validated, never allocated or reinitialised. ++ */ ++ ret = g5p_ds_hook_recover(g, 0); ++ if (ret) ++ goto failed; ++ g->fender_ready = true; ++ g->mmio_off = false; ++ b->stopped = false; ++ ret = g5p_boot_program_stream(g, true); ++ g5p_ds_hook_wake(g, ret); ++ if (ret) ++ goto failed; ++ g5p_ds_hook_maps(g, 0); ++ g5p_ds_hook_load(g, 0); ++ g5p_ds_hook_release(g, 0); ++ deadline = jiffies + 3 * HZ; ++ do { ++ /* IRQ remains masked. Consume fresh ALIVE in RAM without cmd8 or kicks. */ ++ ret = g5p_boot_drain(g); ++ if (ret || READ_ONCE(b->alive)) ++ break; ++ usleep_range(1000, 2000); ++ } while (time_before(jiffies, deadline)); ++ if (!ret && !b->alive) ++ ret = -ETIMEDOUT; ++ if (!ret) { ++ u32 raw = readl(g->window[1] + 0x1000); ++ ++ b->r73b->early_state = raw; ++ if (!g5p_gpc_ok(raw)) { ++ WRITE_ONCE(b->r73b->early_power_abort, true); ++ ret = -EHOSTDOWN; ++ } ++ } ++ g5p_ds_hook_alive(g, ret); ++ if (!ret) ++ ret = g5p_ds_hook_claim(g); ++ if (ret) ++ goto failed; ++ g5p_ru_term_reset(&g5p_rv3_term); ++ g->sleep_boot = false; ++ g->sleep_off = false; ++ g5p_td_op_aic_ack(g); ++ g->sleep_irq_off = false; ++ enable_irq(b->irq); ++ dev_info(dev, "GPU-PM on: fresh ALIVE gen=%u epoch=%u\n", g5p_ds_gen(), ++ d->rdev->sleep_rebuilds); ++ mutex_unlock(&g->lock); ++ return 0; ++failed: ++ g->sleep_boot = false; ++ g->sleep_off = false; ++ g5p_sleep_fail(d, ret); ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static int g5p_system_hibernate(struct device *dev) ++{ ++ return -EBUSY; ++} ++ ++static const struct dev_pm_ops g5p_system_sleep_pm_ops = { ++ .prepare = g5p_system_sleep_prepare, ++ .complete = apple_g5p_system_sleep_complete, ++ .suspend = g5p_system_suspend, ++ .resume = g5p_system_resume, ++ .freeze = g5p_system_hibernate, ++ .poweroff = g5p_system_hibernate, ++ .restore = g5p_system_hibernate, ++}; +diff --git a/include/linux/apple-g5p-pm.h b/include/linux/apple-g5p-pm.h +index 5b152b8fd..7d3316769 100644 +--- a/include/linux/apple-g5p-pm.h ++++ b/include/linux/apple-g5p-pm.h +@@ -8,18 +8,23 @@ + struct device; + + /* +- * For the platform pre-sync sleep veto. Missing support is never a promise +- * that a bound GPU can sleep. Currently all bound G5P devices must be vetoed. +- * A future positive result requires an admission reservation through abort +- * or resume; a snapshot alone cannot authorize power removal. ++ * The pre-sync veto reserves admission on success, through PM complete. ++ * Call only for the bound G5P platform device, from sleepable context. ++ * If sleep aborts before device PM, the caller MUST release the reservation ++ * with apple_g5p_system_sleep_complete(), including every PM_POST_* path. ++ * A false result has no reservation and leaves the running epoch intact. + */ + #if IS_REACHABLE(CONFIG_DRM_APPLE_G5P) + bool apple_g5p_system_sleep_ready(struct device *dev); ++void apple_g5p_system_sleep_complete(struct device *dev); + #else + static inline bool apple_g5p_system_sleep_ready(struct device *dev) + { + return false; + } ++static inline void apple_g5p_system_sleep_complete(struct device *dev) ++{ ++} + #endif + + #endif /* _LINUX_APPLE_G5P_PM_H */ diff --git a/kernel/patches/drivers/drm/0026-drm-apple-g5p-register-the-sleep-reservation-as-susp.patch b/kernel/patches/drivers/drm/0026-drm-apple-g5p-register-the-sleep-reservation-as-susp.patch new file mode 100644 index 00000000..083d70e1 --- /dev/null +++ b/kernel/patches/drivers/drm/0026-drm-apple-g5p-register-the-sleep-reservation-as-susp.patch @@ -0,0 +1,94 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:30:18 +0000 +Subject: [PATCH] drm/apple-g5p: register the sleep reservation as suspend + admission + +The early suspend precheck admits a bound GPU only when the GPU holds a +reserved suspend admission; without a provider every sleep with the GPU +bound is refused before sync. Register the existing reservation as that +provider: prepare is the device prepare callback, which reserves the +host gate and the quiescence certificate without changing hardware state +and refuses with -EBUSY otherwise, and complete is the release that +already runs after resume and every abort. Both are idempotent within +one transition, so device PM prepare/complete keep working as before. + +Register before the render node exists and unregister before any +teardown. The refusal, the certificate and the never-park firmware +behaviour are unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 22 +++++++++++++++++++++- + 1 file changed, 21 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 5aec0de6c..6802578ff 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -5,6 +5,7 @@ + #include "../../apple-g5p/render-user-evwait.h" + #include "../../apple-g5p/render-user-sync.h" + #include ++#include + #include + #include + #include +@@ -116,6 +117,8 @@ module_param_named(job_time_log, g5p_jt_log, bool, 0600); + struct g5p_drm { + struct drm_device drm; + struct g5p_diag *core; ++ /* Reserves the system-sleep certificate before sync, through complete. */ ++ struct pm_suspend_admission sleep_admission; + /* Protects cached IRQ event/status fields; never held for hardware IO. */ + spinlock_t status_lock; + struct g5p_jt jt_ring[G5P_JT_RING]; +@@ -903,6 +906,8 @@ static void g5p_autoboot_work(struct work_struct *work) + g5p_idle_queue(d); + } + ++static int g5p_system_sleep_prepare(struct device *dev); ++ + static int g5p_drm_probe(struct platform_device *pdev) + { + struct g5p_drm *d; +@@ -951,9 +956,20 @@ static int g5p_drm_probe(struct platform_device *pdev) + drm_debugfs_add_file(&d->drm, "g5p_fwclock", g5p_fwclock_show, NULL); + drm_debugfs_add_file(&d->drm, "g5p_cache_receipts", g5p_cache_show, NULL); + INIT_DELAYED_WORK(&d->idle_work, g5p_idle_work); +- ret = drm_dev_register(&d->drm, 0); ++ /* ++ * Before the render node exists. The early suspend precheck admits a ++ * bound GPU only through this reservation; prepare and complete are the ++ * device PM callbacks, which are idempotent for the same transition. ++ */ ++ d->sleep_admission.dev = &pdev->dev; ++ d->sleep_admission.prepare = g5p_system_sleep_prepare; ++ d->sleep_admission.complete = apple_g5p_system_sleep_complete; ++ ret = register_pm_suspend_admission(&d->sleep_admission); + if (ret) + goto err_pm; ++ ret = drm_dev_register(&d->drm, 0); ++ if (ret) ++ goto err_admission; + g5p_r8_register_epoch(g5p_drm_r8_epoch, d); + g5p_td_register_detach(g5p_drm_td_detach, d); + g5p_kq_register(&g5p_kq_drm, d); +@@ -965,6 +981,8 @@ static int g5p_drm_probe(struct platform_device *pdev) + queue_delayed_work(system_unbound_wq, &d->autoboot, 0); + } + return 0; ++err_admission: ++ unregister_pm_suspend_admission(&d->sleep_admission); + err_pm: + g->event_notify = NULL; + pm_runtime_disable(&pdev->dev); +@@ -976,6 +994,8 @@ static void g5p_drm_remove(struct platform_device *pdev) + struct g5p_diag *g = platform_get_drvdata(pdev); + struct g5p_drm *d = g->event_cookie; + ++ /* Before any teardown; waits out a transition that holds it. */ ++ unregister_pm_suspend_admission(&d->sleep_admission); + cancel_delayed_work_sync(&d->autoboot); + cancel_delayed_work_sync(&d->idle_work); + drm_dev_unplug(&d->drm); diff --git a/kernel/patches/drivers/drm/0027-drm-apple-g5p-hold-clients-across-the-sleep-reservat.patch b/kernel/patches/drivers/drm/0027-drm-apple-g5p-hold-clients-across-the-sleep-reservat.patch new file mode 100644 index 00000000..dd4ed7fe --- /dev/null +++ b/kernel/patches/drivers/drm/0027-drm-apple-g5p-hold-clients-across-the-sleep-reservat.patch @@ -0,0 +1,393 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:10:28 +0000 +Subject: [PATCH] drm/apple-g5p: hold clients across the sleep reservation + instead of failing + +While a system sleep holds the G5P reservation, from the pre-sync +admission through complete, every ioctl returns -EBUSY at once. A +compositor or Vulkan client sees that on a submit or a syncobj wait and +may report a lost device. A render-node close inside the window sleeps +in a non-freezable loop, so freezing fails and the whole sleep aborts. +And a syncobj wait that started earlier holds the host count, so the 20 +ms drain refuses a sleep for an idle GPU. + +Apple's GPU drivers never show their clients an error across sleep: work +is simply not admitted until the GPU is powered again, and the same +contexts continue. + +Split admission into two stages. Prepare first shuts a new client gate +and lets the scheduler retire what was already accepted, bounded at one +second; only then does it shut the host gate and run the unchanged +drain, quiescence and certificate checks. Client ioctls wait for the +client gate in a freezable wait that ends when the transition reopens +admission after resume or any abort; a signal restarts the call. Queries +that never touch the GPU (device and render info, job, context and fault +status, BO create, mmap offset and CPU sync, and the DRM syncobj family) +are neither gated nor counted, so a long syncobj wait can sleep through +the transition. A file closed while client admission is shut is parked +and torn down from a work item after admission reopens: its contexts +stay idle survivors like an open file's, and no close item makes the GPU +busy inside the window. + +Every refusal before a destructive step, the freeze after a sampled +power abort or a failed resume, and the terminal error on failure are +unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 181 ++++++++++++++++++--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 51 +++++- + 2 files changed, 200 insertions(+), 32 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 6802578ff..523fdfa44 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -106,6 +106,7 @@ struct g5p_file { + struct xarray contexts; + struct xarray results; + struct xarray targets; /* render targets; each holds its context */ ++ struct list_head sleep_link; /* closed while client admission was shut */ + u32 render_results; + bool submitted; + u64 job_id; +@@ -186,6 +187,16 @@ struct g5p_drm { + /* Memory mapped into any VM, each object once (render_pinned_mib). */ + struct mutex pin_lock; + u64 pinned_bytes; ++ /* ++ * Client admission, closed before the host gate for a system sleep: ++ * closed bit plus active client operations. Clients block on sleep_wq ++ * instead of failing; files closed meanwhile wait on sleep_closes. ++ */ ++ atomic_t client_gate; ++ wait_queue_head_t sleep_wq; ++ spinlock_t sleep_close_lock; /* protects sleep_closes */ ++ struct list_head sleep_closes; ++ struct work_struct sleep_close_work; + }; + + static inline struct g5p_drm *to_g5p(struct drm_device *drm) +@@ -677,10 +688,9 @@ static int g5p_drm_open(struct drm_device *drm, struct drm_file *file) + return 0; + } + +-static void g5p_drm_postclose_host(struct drm_device *drm, +- struct drm_file *file) ++static void g5p_drm_postclose_host(struct g5p_drm *d, struct g5p_file *f) + { +- struct g5p_file *f = file->driver_priv; ++ struct drm_device *drm = &d->drm; + struct g5p_context *c; + struct g5p_vm *vm; + unsigned long id; +@@ -727,14 +737,76 @@ static void g5p_drm_postclose_host(struct drm_device *drm, + READ_ONCE(to_g5p(drm)->pinned_bytes) >> 10); + } + +-static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++/* Client admission: open, and the host gate behind it open too. */ ++static bool g5p_client_enter(struct g5p_drm *d) + { +- struct g5p_drm *d = container_of(drm, struct g5p_drm, drm); ++ int old = atomic_read(&d->client_gate); ++ ++ do { ++ if (old < 0) ++ return false; ++ } while (!atomic_try_cmpxchg(&d->client_gate, &old, old + 1)); ++ if (g5p_sleep_enter(d->core)) ++ return true; ++ atomic_dec(&d->client_gate); ++ return false; ++} + +- while (!g5p_sleep_enter(d->core)) +- msleep(20); +- g5p_drm_postclose_host(drm, file); ++static void g5p_client_exit(struct g5p_drm *d) ++{ + g5p_sleep_exit(d->core); ++ atomic_dec(&d->client_gate); ++} ++ ++/* ++ * While a system sleep holds client admission, a closing file must neither ++ * block the freezer nor queue a close item that makes the GPU busy. Park the ++ * whole teardown; its contexts stay idle like an open file's and close once ++ * admission reopens, after resume or an abort. ++ */ ++static void g5p_sleep_close_work(struct work_struct *work) ++{ ++ struct g5p_drm *d = container_of(work, struct g5p_drm, sleep_close_work); ++ struct g5p_file *f; ++ ++ for (;;) { ++ /* Shut again: the next reopen queues this work. */ ++ if (!g5p_client_enter(d)) ++ return; ++ spin_lock(&d->sleep_close_lock); ++ f = list_first_entry_or_null(&d->sleep_closes, struct g5p_file, ++ sleep_link); ++ if (f) ++ list_del(&f->sleep_link); ++ spin_unlock(&d->sleep_close_lock); ++ if (f) ++ g5p_drm_postclose_host(d, f); ++ g5p_client_exit(d); ++ if (!f) ++ return; ++ drm_dev_put(&d->drm); ++ } ++} ++ ++static void g5p_drm_postclose(struct drm_device *drm, struct drm_file *file) ++{ ++ struct g5p_drm *d = to_g5p(drm); ++ struct g5p_file *f = file->driver_priv; ++ ++ if (g5p_client_enter(d)) { ++ g5p_drm_postclose_host(d, f); ++ g5p_client_exit(d); ++ return; ++ } ++ drm_dev_get(drm); ++ spin_lock(&d->sleep_close_lock); ++ list_add_tail(&f->sleep_link, &d->sleep_closes); ++ spin_unlock(&d->sleep_close_lock); ++ file->driver_priv = NULL; ++ /* Pairs with the barrier in g5p_sleep_reopen(). */ ++ smp_mb(); ++ if (atomic_read(&d->client_gate) >= 0) ++ queue_work(system_unbound_wq, &d->sleep_close_work); + } + + static const struct drm_ioctl_desc g5p_ioctls[] = { +@@ -770,33 +842,86 @@ static const struct drm_ioctl_desc g5p_ioctls[] = { + DRM_RENDER_ALLOW), + }; + +-static long g5p_sleep_ioctl(struct file *filp, unsigned int cmd, +- unsigned long arg) ++/* ++ * Neither MMIO, firmware memory nor scheduled work: these keep working while ++ * a system sleep holds client admission, and never delay its drain. A ++ * syncobj wait in particular may sleep across the whole transition. ++ */ ++static bool g5p_ioctl_host_only(unsigned int cmd) ++{ ++ unsigned int nr = DRM_IOCTL_NR(cmd); ++ ++ if (nr >= DRM_COMMAND_BASE && nr < DRM_COMMAND_END) { ++ switch (nr - DRM_COMMAND_BASE) { ++ case DRM_G5P_GET_INFO: ++ case DRM_G5P_CREATE_BO: ++ case DRM_G5P_BO_MMAP_OFFSET: ++ case DRM_G5P_BO_SYNC: ++ case DRM_G5P_GET_EXEC_INFO: ++ case DRM_G5P_GET_JOB_STATUS: ++ case DRM_G5P_GET_CONTEXT_STATUS: ++ case DRM_G5P_GET_FAULT_INFO: ++ case DRM_G5P_GET_RENDER_INFO: ++ return true; ++ } ++ return false; ++ } ++ switch (nr) { ++ case DRM_IOCTL_NR(DRM_IOCTL_VERSION): ++ case DRM_IOCTL_NR(DRM_IOCTL_GET_CAP): ++ case DRM_IOCTL_NR(DRM_IOCTL_SET_CLIENT_CAP): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_CREATE): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_DESTROY): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_WAIT): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_RESET): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_SIGNAL): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_QUERY): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_TRANSFER): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_TIMELINE_SIGNAL): ++ case DRM_IOCTL_NR(DRM_IOCTL_SYNCOBJ_EVENTFD): ++ return true; ++ } ++ return false; ++} ++ ++/* ++ * Every other ioctl waits for client admission instead of failing. The wait ++ * is freezable, so it never holds up the freezer, and it ends when the sleep ++ * transition reopens admission after resume or any abort. Nothing has run ++ * yet, so a signal restarts the call rather than surfacing to the client. ++ */ ++static long g5p_client_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg, ++ long (*ioctl)(struct file *, unsigned int, ++ unsigned long)) + { + struct drm_file *file = filp->private_data; +- struct g5p_drm *d = container_of(file->minor->dev, struct g5p_drm, drm); ++ struct g5p_drm *d = to_g5p(file->minor->dev); + long ret; + +- if (!g5p_sleep_enter(d->core)) +- return -EBUSY; +- ret = drm_ioctl(filp, cmd, arg); +- g5p_sleep_exit(d->core); ++ if (g5p_ioctl_host_only(cmd)) ++ return ioctl(filp, cmd, arg); ++ if (wait_event_freezable(d->sleep_wq, g5p_client_enter(d))) ++ return -ERESTARTNOINTR; ++ ret = ioctl(filp, cmd, arg); ++ g5p_client_exit(d); + return ret; + } + ++static long g5p_sleep_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg) ++{ ++ return g5p_client_ioctl(filp, cmd, arg, drm_ioctl); ++} ++ + #ifdef CONFIG_COMPAT + static long g5p_sleep_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) + { +- struct drm_file *file = filp->private_data; +- struct g5p_drm *d = container_of(file->minor->dev, struct g5p_drm, drm); +- long ret; +- +- if (!g5p_sleep_enter(d->core)) +- return -EBUSY; +- ret = drm_compat_ioctl(filp, cmd, arg); +- g5p_sleep_exit(d->core); +- return ret; ++ return g5p_client_ioctl(filp, cmd, arg, drm_compat_ioctl); + } + #endif + +@@ -931,6 +1056,11 @@ static int g5p_drm_probe(struct platform_device *pdev) + spin_lock_init(&d->status_lock); + d->jt_stats.since_ns = ktime_get_ns(); + init_waitqueue_head(&d->ev_wq); ++ atomic_set(&d->client_gate, 0); ++ init_waitqueue_head(&d->sleep_wq); ++ spin_lock_init(&d->sleep_close_lock); ++ INIT_LIST_HEAD(&d->sleep_closes); ++ INIT_WORK(&d->sleep_close_work, g5p_sleep_close_work); + atomic_set(&d->ahead_poke, 0); + atomic_set(&d->accepted, 0); + atomic64_set(&d->job_sequence, 0); +@@ -996,6 +1126,9 @@ static void g5p_drm_remove(struct platform_device *pdev) + + /* Before any teardown; waits out a transition that holds it. */ + unregister_pm_suspend_admission(&d->sleep_admission); ++ /* Admission is open again: finish files closed during the last sleep. */ ++ queue_work(system_unbound_wq, &d->sleep_close_work); ++ flush_work(&d->sleep_close_work); + cancel_delayed_work_sync(&d->autoboot); + cancel_delayed_work_sync(&d->idle_work); + drm_dev_unplug(&d->drm); +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 25b1e8b2c..96a77078a 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -92,6 +92,23 @@ static void g5p_sleep_fail(struct g5p_drm *d, int ret) + dev_err(g->dev, "GPU-PM frozen ret=%d; no further MMIO\n", ret); + } + ++/* ++ * Reopen host, then client admission, after a refused prepare or complete. ++ * Blocked clients continue where they waited; parked closes run now. ++ */ ++static void g5p_sleep_reopen(struct g5p_diag *g) ++{ ++ struct g5p_drm *d = g->event_cookie; ++ ++ atomic_andnot(BIT(31), &g->sleep_gate); ++ atomic_andnot(BIT(31), &d->client_gate); ++ /* Pairs with g5p_drm_postclose(): a parked file is seen or queued. */ ++ smp_mb__after_atomic(); ++ wake_up_all(&d->sleep_wq); ++ g5p_rsched_kick(d); ++ queue_work(system_unbound_wq, &d->sleep_close_work); ++} ++ + void apple_g5p_system_sleep_complete(struct device *dev) + { + struct g5p_diag *g = dev_get_drvdata(dev); +@@ -99,14 +116,16 @@ void apple_g5p_system_sleep_complete(struct device *dev) + if (!g || !g->sleep_reserved || g->sleep_off) + return; + g->sleep_reserved = false; +- /* A failed epoch stays terminal, but host close must still be possible. */ +- atomic_andnot(BIT(31), &g->sleep_gate); +- g5p_rsched_kick(g->event_cookie); ++ /* A failed epoch stays terminal, but clients and host close continue. */ ++ g5p_sleep_reopen(g); + dev_info(dev, "GPU-PM complete failed=%u\n", g->sleep_failed); + } + + EXPORT_SYMBOL_GPL(apple_g5p_system_sleep_complete); + ++/* Client work admitted before the client gate closed drains in this bound. */ ++#define G5P_SLEEP_DRAIN_MS 1000 ++ + static int g5p_system_sleep_prepare(struct device *dev) + { + struct g5p_diag *g = dev_get_drvdata(dev); +@@ -116,12 +135,24 @@ static int g5p_system_sleep_prepare(struct device *dev) + u64 deadline; + int ret = -EBUSY; + +- if (!g || !g->event_cookie) ++ if (!g || !g->event_cookie || !g->boot) + return -EBUSY; ++ d = g->event_cookie; + if (READ_ONCE(g->sleep_reserved)) + return g->sleep_failed ? -EBUSY : 0; +- if (atomic_fetch_or(BIT(31), &g->sleep_gate) < 0) ++ /* ++ * Stage 1: shut client admission only. New client work and closes wait, ++ * while the scheduler keeps running so accepted jobs retire. ++ */ ++ if (atomic_fetch_or(BIT(31), &d->client_gate) < 0) + return -EBUSY; ++ deadline = ktime_get_ns() + G5P_SLEEP_DRAIN_MS * NSEC_PER_MSEC; ++ while (((atomic_read(&d->client_gate) & INT_MAX) || !g5p_sleep_idle(d)) && ++ ktime_get_ns() < deadline) ++ usleep_range(500, 1000); ++ /* Stage 2: shut the host gate and certify, unchanged. */ ++ if (atomic_fetch_or(BIT(31), &g->sleep_gate) < 0) ++ goto refuse_client; + deadline = ktime_get_ns() + 20 * NSEC_PER_MSEC; + while ((atomic_read(&g->sleep_gate) & INT_MAX) && + ktime_get_ns() < deadline) +@@ -130,7 +161,6 @@ static int g5p_system_sleep_prepare(struct device *dev) + goto refuse; + if (!g5p_kq_lock(&c)) + goto refuse; +- d = g->event_cookie; + b = g->boot; + if (g->sleep_failed || g5p_r8_admit_closed(&g5p_r8_state.admit) || !b || + !g->powered || !g->fender_ready || g->mmio_off || !b->released || +@@ -157,11 +187,16 @@ unlock: + return 0; + } + refuse: +- atomic_andnot(BIT(31), &g->sleep_gate); +- g5p_rsched_kick(g->event_cookie); ++ g5p_sleep_reopen(g); + dev_info(dev, + "GPU-PM sleep refused: busy or no quiescent certificate\n"); + return -EBUSY; ++refuse_client: ++ atomic_andnot(BIT(31), &d->client_gate); ++ smp_mb__after_atomic(); ++ wake_up_all(&d->sleep_wq); ++ queue_work(system_unbound_wq, &d->sleep_close_work); ++ return -EBUSY; + } + + /* The early caller reserves admission; PM complete releases it on abort/wake. diff --git a/kernel/patches/drivers/drm/0028-drm-apple-g5p-answer-the-suspend-readiness-query.patch b/kernel/patches/drivers/drm/0028-drm-apple-g5p-answer-the-suspend-readiness-query.patch new file mode 100644 index 00000000..fe18c3cf --- /dev/null +++ b/kernel/patches/drivers/drm/0028-drm-apple-g5p-answer-the-suspend-readiness-query.patch @@ -0,0 +1,91 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:12:11 +0000 +Subject: [PATCH] drm/apple-g5p: answer the suspend readiness query + +/sys/power/suspend_ready asks each sleep admission provider whether its +prepare would admit now, without reserving. Provide the G5P check: +report a frozen or failed epoch, a sampled power abort, a stuck or +recovering channel and closed admission as -EIO, and a GPU that is not +booted and powered as -ENODEV. Accepted or in-flight work is not a +refusal, since prepare drains it before certifying; when the GPU is +already idle and the core lock is free, sample the rings and the +quiescence certificate prepare uses and report a failure as -EBUSY. + +No gate is shut, no firmware event is drained and nothing is written. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 2 ++ + drivers/gpu/drm/apple-g5p/system-sleep.inc | 38 ++++++++++++++++++++++ + 2 files changed, 40 insertions(+) + +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 523fdfa44..730138be9 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -1032,6 +1032,7 @@ static void g5p_autoboot_work(struct work_struct *work) + } + + static int g5p_system_sleep_prepare(struct device *dev); ++static int g5p_system_sleep_check(struct device *dev); + + static int g5p_drm_probe(struct platform_device *pdev) + { +@@ -1094,6 +1095,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + d->sleep_admission.dev = &pdev->dev; + d->sleep_admission.prepare = g5p_system_sleep_prepare; + d->sleep_admission.complete = apple_g5p_system_sleep_complete; ++ d->sleep_admission.check = g5p_system_sleep_check; + ret = register_pm_suspend_admission(&d->sleep_admission); + if (ret) + goto err_pm; +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 96a77078a..2f8f30ae6 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -199,6 +199,44 @@ refuse_client: + return -EBUSY; + } + ++/* ++ * Readiness query: what prepare would decide, without shutting any gate or ++ * draining firmware events. Accepted or in-flight work is not a refusal, ++ * since prepare drains it; the certificate is sampled only when the GPU is ++ * already idle and the core lock is free. ++ */ ++static int g5p_system_sleep_check(struct device *dev) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_drm *d; ++ struct g5p_boot *b; ++ int ret = 0; ++ ++ if (!g || !g->event_cookie) ++ return -ENODEV; ++ d = g->event_cookie; ++ b = READ_ONCE(g->boot); ++ if (READ_ONCE(g->sleep_failed) || !b || READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || !b->r73b || ++ READ_ONCE(b->r73b->early_power_abort) || ++ g5p_r8_admit_closed(&g5p_r8_state.admit) || ++ g5p_ru_term_failed(&g5p_rv3_term) || READ_ONCE(d->rdev->stuck) || ++ READ_ONCE(d->rdev->recovering)) ++ return -EIO; ++ if (!READ_ONCE(b->alive) || READ_ONCE(b->stopped) || !b->released || ++ !b->irq_ready) ++ return -ENODEV; ++ if (!mutex_trylock(&g->lock)) ++ return 0; ++ if (!g->powered || !g->fender_ready || g->mmio_off) ++ ret = -ENODEV; ++ else if (g5p_sleep_idle(d) && ++ (!g5p_sleep_rings(g) || !g5p_sleep_certificate(g, false))) ++ ret = -EBUSY; ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ + /* The early caller reserves admission; PM complete releases it on abort/wake. + * This is deliberately stronger than an observation-only readiness query. + */ diff --git a/kernel/patches/drivers/drm/0029-drm-apple-g5p-keep-the-boot-perf-readback-off-the-sy.patch b/kernel/patches/drivers/drm/0029-drm-apple-g5p-keep-the-boot-perf-readback-off-the-sy.patch new file mode 100644 index 00000000..e9aef2c1 --- /dev/null +++ b/kernel/patches/drivers/drm/0029-drm-apple-g5p-keep-the-boot-perf-readback-off-the-sy.patch @@ -0,0 +1,88 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 19:56:59 +0000 +Subject: [PATCH] drm/apple-g5p: keep the boot perf readback off the + synchronous console + +The GPUPERF readback after the release's first ALIVE was one 330 +character KERN_EMERG record. An emergency record is always written to +every console, and the legacy console flush in vprintk_emit() runs with +preemption disabled. On the 115200 baud serial console that one record +holds CPU0 for about 29 ms, so an expedited grace period started in the +window reports: + + rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { +0-.... } 21 jiffies + +with the NMI backtrace in console_flush_all() under g5p_perf_readback() +<- g5p_r73c_fast_control() <- g5p_boot_release(). + +Print the same fields at info level as three short records. The readback +runs after the control submit, and its RAM and register-licence reads +are unchanged, so the hardware sequence and its timing are not affected. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/lifecycle.inc | 49 +++++++++++++++++------------ + 1 file changed, 29 insertions(+), 20 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +index 9a896569c..bf2a76a08 100644 +--- a/drivers/gpu/apple-g5p/lifecycle.inc ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -421,26 +421,35 @@ static void g5p_perf_readback(struct g5p_diag *g, const char *why) + (u64)g5p_dc_root_word(b, meta + 0x788) << 32; + if (!g5p_mmio_hold(g, &t, G5P_MMIO_CENSUS)) + g5p_mmio_release(g, &t); +- dev_emerg( +- g->dev, +- "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " +- "pol28=%u pol2c=%u pol30=%u pol34=%u p8f8=%llx fw_now918=%llx p920=%llx " +- "meta784=%llx host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " +- "host_minus_fw_counter_us=%lld r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", +- why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, +- g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), +- g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), +- g5p_dc_root_word(b, pol + 0x28), +- g5p_dc_root_word(b, pol + 0x2c), +- g5p_dc_root_word(b, pol + 0x30), +- g5p_dc_root_word(b, pol + 0x34), p8f8, p918, p920, m784, host, +- b->seed_stamp, +- b->seed_stamp ? +- div_s64((s64)(p918 - m784 - b->seed_stamp) * 1000000LL, +- rate) : +- 0, +- div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate), +- g5p_mmio_did(&t), t.gpc, t.r1_0); ++ /* ++ * Info level, three short records. One 330-character emergency record ++ * was written to the 115200 baud console inside vprintk_emit()'s ++ * preempt-disabled flush (~29 ms), which stalled expedited RCU on CPU0. ++ */ ++ dev_info(g->dev, ++ "GPUPERF readback why=%s seed=%u/%u poweron=%u pol0=%u pol4=%u pol8=%u polc=%u " ++ "pol28=%u pol2c=%u pol30=%u pol34=%u end=gpuperf\n", ++ why, b->perf_seed_mode, b->perf_seed, b->perf_poweron, ++ g5p_dc_root_word(b, pol + 0), g5p_dc_root_word(b, pol + 4), ++ g5p_dc_root_word(b, pol + 8), g5p_dc_root_word(b, pol + 0xc), ++ g5p_dc_root_word(b, pol + 0x28), ++ g5p_dc_root_word(b, pol + 0x2c), ++ g5p_dc_root_word(b, pol + 0x30), ++ g5p_dc_root_word(b, pol + 0x34)); ++ dev_info(g->dev, ++ "GPUPERF readback why=%s p8f8=%llx fw_now918=%llx p920=%llx meta784=%llx " ++ "r1=%s r1_1000=%08x r1_0=%08x end=gpuperf\n", ++ why, p8f8, p918, p920, m784, g5p_mmio_did(&t), t.gpc, t.r1_0); ++ dev_info(g->dev, ++ "GPUPERF readback why=%s host_now=%llx stamp=%llx fw_counter_minus_stamp_us=%lld " ++ "host_minus_fw_counter_us=%lld end=gpuperf\n", ++ why, host, b->seed_stamp, ++ b->seed_stamp ? ++ div_s64((s64)(p918 - m784 - b->seed_stamp) * ++ 1000000LL, ++ rate) : ++ 0, ++ div_s64((s64)(host - (p918 - m784)) * 1000000LL, rate)); + } + + static int g5p_r73c_fast_control(struct g5p_diag *g) diff --git a/kernel/patches/drivers/drm/0030-drm-apple-g5p-name-the-sleep-readiness-refusal-and-r.patch b/kernel/patches/drivers/drm/0030-drm-apple-g5p-name-the-sleep-readiness-refusal-and-r.patch new file mode 100644 index 00000000..460cb0d9 --- /dev/null +++ b/kernel/patches/drivers/drm/0030-drm-apple-g5p-name-the-sleep-readiness-refusal-and-r.patch @@ -0,0 +1,216 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 19:59:32 +0000 +Subject: [PATCH] drm/apple-g5p: name the sleep readiness refusal, and refuse + never-park + +The readiness check answered -EBUSY for both a ring that has not drained +and a firmware that is not quiescent, and answered ready whenever work +was in flight or the core lock was busy, without sampling. On J71t the +same desktop read "ready" after a cold boot and "not-ready 201000000.gpu +-16" after a warm restart, and the two answers could not be told apart. + +The shipping perf envelope pins the firmware (perf-envelope.h: floor = +pin 400, overrides = pin, sampler off, policy+4 = 0). The firmware then +never runs its idle park (0x1001cec8: full drain, GPC 0, R1+0 bit 29), +and Apple's isQuiescent (6ca66e4) needs exactly that state. When it does +not hold, Apple's powerDownIfQuiescent (c81b18), reached from the IOPM +off ordinal (c83d28), neither acknowledges nor touches hardware; it +re-arms and waits for the firmware to park. So a pinned firmware cannot +pass prepare's certificate after any boot, and a busy GPU reported ready +only because nothing was sampled. Recorded samples on pin-400 boots (GPC +at every bell, R1+0x1000 and policy+0x34 at boot) all read state 4; they +are taken at kicks and boot, not after long idle, so they corroborate +this rather than prove it. + +Make the check report its reason through the admission reason callback: +- never-park/busy: pinned envelope, work in flight (g5p_sleep_idle() +false); prepare would drain and then fail the certificate; - +never-park/locked: pinned envelope, the core lock is held; - +never-park/gpc=NN: pinned envelope, idle, sampled GPC state NN; - +not-quiescent/gpc=NN or not-quiescent/r1_0=X/r1_8=Y: the certificate's +own reads failed on an unpinned envelope; - ringN=C/P or +ringN-unreadable: a ring has not drained; - sleep-failed, no-boot, +firmware-failed, power-abort, terminal, recovering, not-booted, +not-powered for the existing -EIO and -ENODEV answers. + +The sampled reads are the certificate's, in its order and under the same +conditions (idle, core lock held, powered, MMIO licensed). A sampled GPC +0 falls through to the certificate, so a firmware that does park is +judged as before. prepare, the certificate and suspend are unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 4 + + drivers/gpu/drm/apple-g5p/system-sleep.inc | 107 ++++++++++++++++++--- + 2 files changed, 95 insertions(+), 16 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 730138be9..d5f44c3b0 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -120,6 +120,8 @@ struct g5p_drm { + struct g5p_diag *core; + /* Reserves the system-sleep certificate before sync, through complete. */ + struct pm_suspend_admission sleep_admission; ++ /* Why the last readiness check refused; written by the check only. */ ++ char sleep_why[64]; + /* Protects cached IRQ event/status fields; never held for hardware IO. */ + spinlock_t status_lock; + struct g5p_jt jt_ring[G5P_JT_RING]; +@@ -1033,6 +1035,7 @@ static void g5p_autoboot_work(struct work_struct *work) + + static int g5p_system_sleep_prepare(struct device *dev); + static int g5p_system_sleep_check(struct device *dev); ++static const char *g5p_system_sleep_reason(struct device *dev); + + static int g5p_drm_probe(struct platform_device *pdev) + { +@@ -1096,6 +1099,7 @@ static int g5p_drm_probe(struct platform_device *pdev) + d->sleep_admission.prepare = g5p_system_sleep_prepare; + d->sleep_admission.complete = apple_g5p_system_sleep_complete; + d->sleep_admission.check = g5p_system_sleep_check; ++ d->sleep_admission.reason = g5p_system_sleep_reason; + ret = register_pm_suspend_admission(&d->sleep_admission); + if (ret) + goto err_pm; +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 2f8f30ae6..effe37166 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -199,44 +199,119 @@ refuse_client: + return -EBUSY; + } + ++/* ++ * The pinned perf envelope (perf-envelope.h: floor = pin, sampler off, ++ * policy+4 = 0) keeps the firmware from ever running its idle park ++ * (0x1001cec8: GPC 0 and R1+0 bit 29). Apple's isQuiescent (6ca66e4) needs ++ * that park, and powerDownIfQuiescent (c81b18) neither acknowledges nor ++ * touches hardware without it: it waits for the firmware to park. RAM only. ++ */ ++static bool g5p_sleep_never_park(struct g5p_boot *b) ++{ ++ return b->maps_ready && b->gpu_perf && ++ !g5p_dc_root_word(b, b->layout.offset[0x4a] + 4); ++} ++ ++/* The readiness reason: one token, no white space or colon. */ ++static __printf(3, 4) int g5p_sleep_why(struct g5p_drm *d, int ret, ++ const char *fmt, ...) ++{ ++ va_list args; ++ ++ va_start(args, fmt); ++ vsnprintf(d->sleep_why, sizeof(d->sleep_why), fmt, args); ++ va_end(args); ++ return ret; ++} ++ + /* + * Readiness query: what prepare would decide, without shutting any gate or + * draining firmware events. Accepted or in-flight work is not a refusal, +- * since prepare drains it; the certificate is sampled only when the GPU is +- * already idle and the core lock is free. ++ * since prepare drains it, except under the never-park envelope, where the ++ * drained firmware still cannot certify. Register samples are taken only ++ * when the GPU is already idle and the core lock is free, with the same ++ * reads as g5p_sleep_certificate(); a sampled GPC 0 falls through to it. + */ + static int g5p_system_sleep_check(struct device *dev) + { + struct g5p_diag *g = dev_get_drvdata(dev); + struct g5p_drm *d; + struct g5p_boot *b; ++ u32 cons, prod, state, mode = 0, r8 = 0; ++ bool never_park; + int ret = 0; + + if (!g || !g->event_cookie) + return -ENODEV; + d = g->event_cookie; ++ d->sleep_why[0] = 0; + b = READ_ONCE(g->boot); +- if (READ_ONCE(g->sleep_failed) || !b || READ_ONCE(b->failed) || +- READ_ONCE(b->event_failed) || !b->r73b || +- READ_ONCE(b->r73b->early_power_abort) || +- g5p_r8_admit_closed(&g5p_r8_state.admit) || +- g5p_ru_term_failed(&g5p_rv3_term) || READ_ONCE(d->rdev->stuck) || +- READ_ONCE(d->rdev->recovering)) +- return -EIO; ++ if (READ_ONCE(g->sleep_failed)) ++ return g5p_sleep_why(d, -EIO, "sleep-failed"); ++ if (!b || !b->r73b) ++ return g5p_sleep_why(d, -EIO, "no-boot"); ++ if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed)) ++ return g5p_sleep_why(d, -EIO, "firmware-failed"); ++ if (READ_ONCE(b->r73b->early_power_abort)) ++ return g5p_sleep_why(d, -EIO, "power-abort"); ++ if (g5p_r8_admit_closed(&g5p_r8_state.admit) || ++ g5p_ru_term_failed(&g5p_rv3_term)) ++ return g5p_sleep_why(d, -EIO, "terminal"); ++ if (READ_ONCE(d->rdev->stuck) || READ_ONCE(d->rdev->recovering)) ++ return g5p_sleep_why(d, -EIO, "recovering"); + if (!READ_ONCE(b->alive) || READ_ONCE(b->stopped) || !b->released || + !b->irq_ready) +- return -ENODEV; ++ return g5p_sleep_why(d, -ENODEV, "not-booted"); ++ never_park = g5p_sleep_never_park(b); + if (!mutex_trylock(&g->lock)) +- return 0; +- if (!g->powered || !g->fender_ready || g->mmio_off) +- ret = -ENODEV; +- else if (g5p_sleep_idle(d) && +- (!g5p_sleep_rings(g) || !g5p_sleep_certificate(g, false))) +- ret = -EBUSY; ++ return never_park ? g5p_sleep_why(d, -EBUSY, "never-park/locked") : ++ 0; ++ if (!g->powered || !g->fender_ready || g->mmio_off) { ++ ret = g5p_sleep_why(d, -ENODEV, "not-powered"); ++ goto unlock; ++ } ++ if (!g5p_sleep_idle(d)) { ++ if (never_park) ++ ret = g5p_sleep_why(d, -EBUSY, "never-park/busy"); ++ goto unlock; ++ } ++ for (u32 i = 0; i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ ret = g5p_sleep_why(d, -EBUSY, "ring%u-unreadable", i); ++ goto unlock; ++ } ++ if (cons > 255 || cons != prod) { ++ ret = g5p_sleep_why(d, -EBUSY, "ring%u=%u/%u", i, cons, ++ prod); ++ goto unlock; ++ } ++ } ++ state = readl(g->window[1] + 0x1000); ++ if (state & 0x1f) { ++ ret = g5p_sleep_why(d, -EBUSY, "%s/gpc=%02x", ++ never_park ? "never-park" : "not-quiescent", ++ state & 0x1f); ++ goto unlock; ++ } ++ mode = readl(g->window[1]); ++ if ((mode & 0x60000000) == 0x60000000) ++ r8 = readl(g->window[1] + 8); ++ if ((mode & 0x60000000) != 0x60000000 || r8) ++ ret = g5p_sleep_why(d, -EBUSY, "not-quiescent/r1_0=%08x/r1_8=%08x", ++ mode, r8); ++unlock: + mutex_unlock(&g->lock); + return ret; + } + ++static const char *g5p_system_sleep_reason(struct device *dev) ++{ ++ struct g5p_diag *g = dev_get_drvdata(dev); ++ struct g5p_drm *d = g ? g->event_cookie : NULL; ++ ++ return d ? d->sleep_why : NULL; ++} ++ + /* The early caller reserves admission; PM complete releases it on abort/wake. + * This is deliberately stronger than an observation-only readiness query. + */ diff --git a/kernel/patches/drivers/drm/0031-drm-apple-g5p-arm-firmware-self-park-after-pinned-br.patch b/kernel/patches/drivers/drm/0031-drm-apple-g5p-arm-firmware-self-park-after-pinned-br.patch new file mode 100644 index 00000000..e2a2936a --- /dev/null +++ b/kernel/patches/drivers/drm/0031-drm-apple-g5p-arm-firmware-self-park-after-pinned-br.patch @@ -0,0 +1,622 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 13:28:33 +0100 +Subject: [PATCH] drm/apple-g5p: arm firmware self-park after pinned bring-up + +Keep the proven pinned record through ALIVE and control completion. Arm +the idle policy only on a complete live epoch, with policy+4 published +before ctrl+0x20. Refused epochs remain pinned. The coefficient block is +Apple's observed record; its role in avoiding pstate zero is inferred, +not independently isolated. Never disarm an armed live epoch. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/boot-r74f.inc | 5 +- + drivers/gpu/apple-g5p/boot.inc | 89 ++++++++++++++ + drivers/gpu/apple-g5p/idle-power.inc | 8 +- + drivers/gpu/apple-g5p/kick.inc | 5 +- + drivers/gpu/apple-g5p/mmio-licence.h | 8 +- + drivers/gpu/apple-g5p/mmio.inc | 2 +- + drivers/gpu/apple-g5p/perf-envelope.h | 121 +++++++++++++++++++ + drivers/gpu/apple-g5p/recover.inc | 1 + + drivers/gpu/apple-g5p/render-user-wake.h | 6 +- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 111 +++++++++++++++++ + drivers/gpu/drm/apple-g5p/drm-compute-v1.inc | 5 +- + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 34 ++++-- + 12 files changed, 377 insertions(+), 18 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +index 71cf79d..9ab4947 100644 +--- a/drivers/gpu/apple-g5p/boot-r74f.inc ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -100,7 +100,10 @@ static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, + *state = readl(g->window[1] + 0x1000); + } + } +- if (!g5p_gpc_ok(*state)) ++ /* self-park: a parked firmware wakes through this doorbell (Apple: any nibble) */ ++ if (!g5p_gpc_ok(*state) && ++ !g5p_park_may_ring(READ_ONCE(b->park_epoch), *state, ++ readl(g->window[1]))) + return -EHOSTDOWN; + dma_wmb(); + writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +index 286620e..beaf75c 100644 +--- a/drivers/gpu/apple-g5p/boot.inc ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -197,6 +197,11 @@ struct g5p_boot { + u64 seed_stamp, + seed_gpc1; /* seed 3/4: the stamp and the GPC1 write (arch timer) */ + u32 meta6d4; ++ /* self-park: iOS's policy record (latched at map), ctrl+0x20 as last written, and ++ * park_epoch: armed at least once since this firmware boot, so the firmware may be parked ++ * (the doorbell and R0 rules follow it; only a host-driven boot clears it) ++ */ ++ bool self_park, self_park_armed, park_epoch; + u32 r78_ctrl0; + u32 r73_aux_drain; + u32 r73_aux_entries[6]; +@@ -365,6 +370,84 @@ static u8 *g5p_boot_shared(struct g5p_boot *b, u32 slot) + return b->shared[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE; + } + ++/* self-park: one word of a live boot object (root or shared group, paged), RAM only */ ++static u32 *g5p_park_word(struct g5p_boot *b, u32 slot, u32 offset) ++{ ++ u32 off = b->layout.offset[slot] + offset; ++ struct g5p_dma_page *pages = b->layout.group[slot] ? b->shared : b->root; ++ ++ return (u32 *)(pages[off / G5P_GPU_PAGE].cpu + off % G5P_GPU_PAGE); ++} ++ ++/* ++ * self-park (perf-envelope.h): the epoch's park in the live record. The firmware re-reads ++ * ctrl+0x20 on every tick (0x1001061c) and the governor reads its policy record live, so ++ * arming needs no reboot. Armed only on a complete epoch: ALIVE seen, control stream done, ++ * nothing failed or stopped; a refused arm writes nothing, which leaves the pinned never-park ++ * epoch exactly as it was booted. Arm order: iOS's coefficient block and envelope, then ++ * policy+4 = 16 (the wake's boot sampler divides by it), then ctrl+0x20 = 1, each store set ++ * before the next (dma_wmb). Disarm (a host-driven boot of a stopped firmware): ctrl+0x20 = 0 ++ * and the pinned record's boot words. The read-back is a Normal-NC load of the line (the ++ * visibility rule of g5p_rv3_fw_touch()). RAM only. ++ */ ++static void g5p_park_set(struct g5p_diag *g, bool on, const char *why) ++{ ++ struct g5p_boot *b = g->boot; ++ u32 *w, now, i; ++ ++ lockdep_assert_held(&g->lock); ++ if (!b || !b->maps_ready || !b->self_park) ++ return; ++ if (on && (!READ_ONCE(b->alive) || !READ_ONCE(b->control_complete) || ++ READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped))) { ++ dev_info(g->dev, ++ "G5P self-park arm refused why=%s alive=%u control=%u failed=%u stopped=%u " ++ "end=park\n", ++ why, b->alive, b->control_complete, b->failed, ++ b->stopped); ++ return; ++ } ++ w = g5p_park_word(b, G5P_PERF_CTL, G5P_PARK_CTL_FLAG); ++ if (on) { ++ for (i = 0; i < ARRAY_SIZE(g5p_park_policy); i++) ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_POLICY, ++ g5p_park_policy[i].offset), ++ g5p_park_policy[i].value); ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_CTL, 0x1028), ++ G5P_PARK_ENVELOPE); ++ dma_wmb(); ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_POLICY, 4), ++ G5P_PARK_SAMPLE_MS); ++ dma_wmb(); ++ WRITE_ONCE(*w, 1); ++ } else { ++ WRITE_ONCE(*w, 0); ++ dma_wmb(); ++ for (i = 0; i < ARRAY_SIZE(g5p_park_policy); i++) ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_POLICY, ++ g5p_park_policy[i].offset), ++ g5p_park_pinned_value( ++ g5p_park_policy[i].offset)); ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_CTL, 0x1028), ++ G5P_PIN_ENVELOPE); ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_POLICY, 4), 0); ++ WRITE_ONCE(*g5p_park_word(b, G5P_PERF_POLICY, 0), 0); ++ } ++ dma_wmb(); ++ now = READ_ONCE(*w); ++ WRITE_ONCE(b->self_park_armed, on); ++ if (on) ++ WRITE_ONCE(b->park_epoch, true); ++ dev_info(g->dev, ++ "G5P self-park %s why=%s ctl20=%u policy4=%u envelope=%u pin=%u/%u end=park\n", ++ on ? "armed" : "disarmed", why, now, ++ READ_ONCE(*g5p_park_word(b, G5P_PERF_POLICY, 4)), ++ READ_ONCE(*g5p_park_word(b, G5P_PERF_CTL, 0x1028)), ++ READ_ONCE(*g5p_park_word(b, G5P_PERF_CTL, 0x1038)), ++ READ_ONCE(*g5p_park_word(b, G5P_PERF_CTL, 0x1040))); ++} ++ + static int g5p_boot_maps(struct g5p_diag *g) + { + struct g5p_boot *b; +@@ -433,6 +516,12 @@ static int g5p_boot_maps(struct g5p_diag *g) + ; + b->perf_poweron = + (1) ? (0 ? G5P_OC_STATE : g5p_perf_seed_q(b->gpu_perf)) : 0; ++ /* self-park: bring-up is the pinned never-park record; the park is armed in the live ++ * record once the epoch is complete (g5p_park_set(), perf-envelope.h). ++ */ ++ b->self_park = true; ++ b->self_park_armed = false; ++ b->park_epoch = false; + b->perf_seed = seed && !b->perf_poweron ? g5p_perf_seed_q(b->gpu_perf) : + 0; + b->perf_seed_mode = b->perf_seed ? seed : 0; +diff --git a/drivers/gpu/apple-g5p/idle-power.inc b/drivers/gpu/apple-g5p/idle-power.inc +index d2ce718..4401fd6 100644 +--- a/drivers/gpu/apple-g5p/idle-power.inc ++++ b/drivers/gpu/apple-g5p/idle-power.inc +@@ -28,9 +28,13 @@ static ssize_t idle_power_status_show(struct device *dev, + ret = sysfs_emit(buf, + "available=1 gen=%u alive=%u failed=%u stopped=%u powered=%u " + "policy_state=%u target=%u policy_pstate=%u sample_ms=%u " +- "meta71c=%u source=shared-ram atomic=0 hw_ack=0 mmio=0\n", ++ "meta71c=%u self_park=%u armed=%u park_epoch=%u ctl20=%u " ++ "source=shared-ram atomic=0 hw_ack=0 mmio=0\n", + g5p_ds_gen(), b->alive, b->failed, b->stopped, +- g->powered, state, target, mirror, period, b->meta71c); ++ g->powered, state, target, mirror, period, b->meta71c, ++ b->self_park, b->self_park_armed, b->park_epoch, ++ READ_ONCE(*(u32 *)(g5p_boot_shared(b, G5P_PERF_CTL) + ++ G5P_PARK_CTL_FLAG))); + out: + mutex_unlock(&g->lock); + return ret; +diff --git a/drivers/gpu/apple-g5p/kick.inc b/drivers/gpu/apple-g5p/kick.inc +index 6ef7395..1addf0a 100644 +--- a/drivers/gpu/apple-g5p/kick.inc ++++ b/drivers/gpu/apple-g5p/kick.inc +@@ -19,7 +19,10 @@ static int g5p_prepare_kick(struct g5p_diag *g, const char *name) + if (raw & ~7U) + return -EPROTO; + state = readl(g->window[1] + 0x1000); +- if (!g5p_gpc_ok(state)) ++ /* self-park: a parked firmware wakes through the kick that follows */ ++ if (!g5p_gpc_ok(state) && ++ !g5p_park_may_ring(READ_ONCE(g->boot->park_epoch), state, ++ readl(g->window[1]))) + return -EHOSTDOWN; + if (READ_ONCE(g->boot->failed)) + return -EIO; +diff --git a/drivers/gpu/apple-g5p/mmio-licence.h b/drivers/gpu/apple-g5p/mmio-licence.h +index d9d23ca..19135ea 100644 +--- a/drivers/gpu/apple-g5p/mmio-licence.h ++++ b/drivers/gpu/apple-g5p/mmio-licence.h +@@ -86,11 +86,13 @@ static inline enum g5p_mmio_why g5p_mmio_check_gpc(u32 gpc, u32 max) + * has no such window; at pin 400/500 the 3D-completion and clock samples saw bit 29 0 times, while + * the park tuple's rung 5 (2026-11-03) saw it on 749 of 983 3D completions and then froze the SoC. + * iOS reads R0 only at a fault event, never at idle. `override` = mmio_r0_park (DFU-class rungs +- * only). ++ * only). `park_epoch` (self-park): the firmware's own idle park is armed in this epoch, so ++ * the GPC can drop to 0 between any R1 check and the load whatever the envelope. + */ +-static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool override) ++static inline bool g5p_mmio_r0_regime_ok(u32 gpu_perf, bool park_epoch, ++ bool override) + { +- return override || gpu_perf >= 100; ++ return override || (!park_epoch && gpu_perf >= 100); + } + + /* +diff --git a/drivers/gpu/apple-g5p/mmio.inc b/drivers/gpu/apple-g5p/mmio.inc +index b8c25bd..d59e869 100644 +--- a/drivers/gpu/apple-g5p/mmio.inc ++++ b/drivers/gpu/apple-g5p/mmio.inc +@@ -141,7 +141,7 @@ static int g5p_mmio_hold(struct g5p_diag *g, struct g5p_mmio_ticket *t, + } + /* before any bus access: an R0 ticket in a park epoch is refused outright */ + if (!(flags & G5P_MMIO_CENSUS) && b && +- !g5p_mmio_r0_regime_ok(b->gpu_perf, (0))) { ++ !g5p_mmio_r0_regime_ok(b->gpu_perf, READ_ONCE(b->park_epoch), (0))) { + t->why = G5P_MMIO_PARKEPOCH; + t->t_end_ns = ktime_get_mono_fast_ns(); + spin_unlock_irqrestore(&g->mmio_lock, irq); +diff --git a/drivers/gpu/apple-g5p/perf-envelope.h b/drivers/gpu/apple-g5p/perf-envelope.h +index 57387bc..7ec1ca1 100644 +--- a/drivers/gpu/apple-g5p/perf-envelope.h ++++ b/drivers/gpu/apple-g5p/perf-envelope.h +@@ -104,4 +104,125 @@ static inline void g5p_perf_seed(const struct g5p_boot_layout *p, u8 *root, + } + } + ++/* ++ * Self-park: the firmware's own idle park (Apple's Level 1), as the AGX kext configures it on J71t ++ * (iOS 16.7.10 20H350; firmware addresses are the META image's, kext addresses unslid). ++ * ++ * Who parks. The firmware governor 0x1000c0d8 ends every pass in the idle tail 0x10009bd8. ++ * The tail calls the try-park wrapper 0x1000968c -> 0x1001cec8 only when, in order: ++ * priv+0x188 != 0 (0x10009bec), policy+0x34 != 0, the engines are idle (0x100013f4), every ++ * root ring is empty (0x1000176c), 0x10001bdc == 0 and 0x1001ee98 == 0. ++ * priv+0x188 is the host's control word ctrl+0x20, re-read on every firmware tick ++ * (0x1001061c -> 0x10001328, copy at 0x10001388). The kext writes ctrl+0x20 = ++ * platform+0x510 & 1 (fffffff006c97888); j71t's platform word is 0x00200545, so iOS runs with ++ * 1. The same bit decides the host's submit path (AGXMetaFirmware_submitDeviceControl ++ * fffffff006ca05d4): with it clear a parked GPC is power-cycled and rebooted; with it set the ++ * submit only kicks. With ctrl+0x20 = 0 the tail returns at once: the firmware never parks, ++ * whatever the pstate policy. That, not the pin, is why no Linux boot could sleep. ++ * ++ * The park (0x1001cec8). The full drain 0x1001b638, R1+0x1004 = the wake target ++ * (policy+0x28), R1+0 |= bit 29, an abort if a ring took work meanwhile (0x1001cd8c), else ++ * bit 30 is kept (0x1001cf74: only priv+0x188 == 0 clears it), setter(0) = GPC 0 and the ++ * self-branch 0x1001d074. R1+0[30:29] = 11, GPC 0, R1+8 = 0 is exactly ++ * AGXMetaFender_isQuiescent (fffffff006ca66e4), so Apple's sleep certificate assumes this bit. ++ * ++ * The wake. A parked firmware does not return from 0x1001d074; it reboots. Every boot runs the ++ * unpark handshake 0x1001cd8c(1) (0x10000b88) and then the boot sampler 0x1000aa80 ++ * (0x10000b54), which applies the target to Fender before the scheduler runs. The host's part ++ * is Apple's submit order: R1+0x14 = 7 and its read-back (Fender vt+0x80, fffffff006ca6644), ++ * one GPC read (vt+0x18), and R0+0xb00 at any nibble. ++ * ++ * Why the policy record must be iOS's. The boot sampler takes its sampling path on any boot ++ * after the first (policy+0 = 1, stored by its own init 0x1000aec8) and divides the elapsed ++ * time by policy+4 (0x1000ac28). The pinned envelope leaves policy+4 = 0, so a parked pinned ++ * firmware would wake into a division by zero. iOS's record (16 ms sampler, its coefficient ++ * block, no override) is the only configuration Apple runs through this path. The target ++ * then moves only across park -> wake, as on iOS (policy+0x34: 4 <-> 0 under load, 1 <-> 0 idle). ++ * ++ * Values: the kext's init (6c974f0, 6c95770) and six 2026-11-01 J71t captures, which agree on ++ * every word below. They are the platform power model (3851 mW envelope, filter constants), ++ * the class of data the pstate table already is. No sampled state, deadline, accumulator or ++ * clock is written: a fresh record starts as the kext starts it (state 0, substate 4, target 1). ++ * ++ * Bring-up is the pinned never-park record, unchanged. The record is not installed at map: ++ * integration-g did (and the release at pstate 1, no override, meta+0x6c4 = meta+0x700 = 1), ++ * and on its cold boot the firmware came ALIVE but never consumed the control command 8 ++ * (IP0, 2026-11-11), where every pinned boot consumes it. meta+0x6c4 is copied once per ++ * firmware boot into priv+0x230 (0x10003cf0; the copy is not linear: priv+0x22c = meta+0x6c8), ++ * and the main loop calls 0x100044f0 on every idle pass while priv+0x230 != 0 (0x10010808); the ++ * pinned record never runs that routine. The governor also wanted target 2 at pstate 1 with ++ * ctrl+0x20 = 0, a move it only makes across a park. Which of these held the command is not ++ * isolated; none is needed to park, so none is written before the epoch is complete. ++ * ++ * Arming (complete epoch only: ALIVE, control stream done, nothing failed or stopped). The ++ * idle tail needs only priv+0x188 != 0 and a powered pstate policy+0x34 != 0 (0x10009bec, ++ * 0x10009c0c), which the pinned record has. What the wake needs is a sampler record that ++ * neither divides by zero nor lands on pstate 0: seed 2 (pinned record, +0 = 1, +4 = 16, no ++ * coefficients) ran the load governor at a stale stamp and gave no ALIVE. So the arm writes, ++ * live and in this order: the coefficient block and filter initialisation below and the ++ * 3851 mW envelope; then policy+4 = 16; then ctrl+0x20 = 1. The pin (ctrl+0x1038/0x1040 = ++ * 400) stays, so the target the park hands to the wake (R1+0x1004 = policy+0x28) is the ++ * pstate the release requested. meta+0x6c4 and meta+0x700 stay 0. policy+4 is written only ++ * just before ctrl+0x20: latency123 T2/T3 (policy+4 = 16 with ctrl+0x20 = 0) wedged the first ++ * TA, and IP0 is the same pair. The B2 device test (ctrl+0x20 = 1 before release) parked before ++ * its first-ALIVE GPC read and latched a power abort. ++ * ++ * Disarming is for a host-driven boot of a stopped firmware (resume) only: ctrl+0x20 = 0 and ++ * the pinned record's boot words again (policy+0 = 0 so the boot sampler takes its first-call ++ * init, policy+4 = 0, the block cleared, envelope 3348), so that boot is a pinned bring-up. ++ * A live epoch is never disarmed (debugfs "park 0" is refused once the epoch has parked). ++ */ ++#define G5P_PARK_CTL_FLAG 0x20U /* ctrl+0x20 -> priv+0x188 */ ++#define G5P_PARK_SAMPLE_MS 16U /* policy+4 */ ++#define G5P_PARK_PIN_OFF 0U /* debugfs pin: the firmware governor owns the target */ ++ ++struct g5p_park_word { ++ u32 offset, value; ++}; ++ ++/* policy (slot 0x4a): the coefficient block and the kext's filter initialisation */ ++static const struct g5p_park_word g5p_park_policy[] = { ++ { 0x03c, 0x40635631 }, { 0x040, 0x42bbbc1a }, { 0x044, 0x41493627 }, ++ { 0x048, 0x3dcf8fa7 }, { 0x04c, 0x3e1af366 }, { 0x0dc, 0x4226a00d }, ++ { 0x160, 0x3f8187d7 }, { 0x164, 0x3fad683b }, { 0x168, 0x3fbcd580 }, ++ { 0x16c, 0x3fdca811 }, { 0x638, 0x3f7f2e9f }, { 0x640, 0x3b516154 }, ++ { 0x648, 0x3ca59586 }, { 0x650, 0x00000000 }, { 0x654, 0x4570b000 }, ++ { 0x658, 0x40a90fdb }, { 0x660, 0xbdd4b956 }, { 0x664, 0xfffffe98 }, ++ { 0x668, 0x00000000 }, { 0x66c, 0x00000190 }, { 0x674, 0x00000f0b }, ++ { 0x684, 0x00000000 }, { 0x688, 0x00000190 }, { 0x694, 0x0000005f }, ++ { 0x698, 0x00000001 }, { 0x69c, 0x00000000 }, { 0x6a0, 0x3f4ccccd }, ++ { 0x6a4, 0x3f7ae148 }, { 0x6a8, 0x3e4ccccd }, { 0x6ac, 0x3ca3d70a }, ++ { 0x6b0, 0x40fca970 }, { 0x6b4, 0x3e4a2121 }, { 0x6b8, 0x00000000 }, ++ { 0x6bc, 0x42be0000 }, { 0x6c0, 0x416b53d1 }, { 0x6c4, 0x40db53d0 }, ++ { 0x6c8, 0x404a1af3 }, { 0x6cc, 0x00000064 }, { 0x6d0, 0x00000190 }, ++ { 0x6d4, 0x00000064 }, { 0x6dc, 0x0000005f }, { 0x6ec, 0x00000000 }, ++ { 0x6f8, 0x42c80000 }, { 0x6fc, 0x00000190 }, { 0x704, 0x3f4ccccd }, ++ { 0x708, 0x3e4ccccd }, { 0x7cc, 0x00000190 }, ++}; ++ ++#define G5P_PARK_ENVELOPE 3851U /* ctrl+0x1028, iOS */ ++#define G5P_PIN_ENVELOPE 3348U /* ctrl+0x1028, g5p_perf_install() */ ++ ++/* ++ * The block's value in the pinned record: 0, except policy+0x688, which g5p_perf_install() ++ * writes as G5P_PERF_TOP (the same value iOS has). ++ */ ++static inline u32 g5p_park_pinned_value(u32 offset) ++{ ++ return offset == 0x688 ? G5P_PERF_TOP : 0; ++} ++ ++/* ++ * The doorbell rule after R1+0x14 = 7. A settled powered GPC rings as before. In a park ++ * epoch a GPC that is still off or in transition rings too (Apple writes R0+0xb00 at any ++ * nibble, 6ca05d4); only a dead wrapper (R1+0 == 0) is refused, which Apple would answer with a ++ * power cycle we do not run. ++ */ ++static inline bool g5p_park_may_ring(bool self_park, u32 gpc, u32 r1_0) ++{ ++ if (!(gpc & 0x10) && (gpc & 0xf)) ++ return true; ++ return self_park && r1_0; ++} ++ + #endif +diff --git a/drivers/gpu/apple-g5p/recover.inc b/drivers/gpu/apple-g5p/recover.inc +index f550d10..af837c0 100644 +--- a/drivers/gpu/apple-g5p/recover.inc ++++ b/drivers/gpu/apple-g5p/recover.inc +@@ -82,6 +82,7 @@ static int __maybe_unused g5p_recover(struct g5p_diag *g, const char *why, + if (!ret) { + /* The render claim is kept: no diagnostic command may reset this epoch. */ + WRITE_ONCE(g->boot->user_compute_owned, true); ++ g5p_park_set(g, true, "recover"); + stage = 0; + } + out: +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +index 64a51ea..c8fd0aa 100644 +--- a/drivers/gpu/apple-g5p/render-user-wake.h ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -132,6 +132,10 @@ struct g5p_wk_io { + u32 (*r1)(void *ctx, u32 reg); + void (*wbarrier)(void *ctx); + void (*w0)(void *ctx, u32 reg, u32 value); ++ /* self-park: the firmware may be parked; a GPC that is not on still rings unless ++ * R1+0 reads 0 (Apple's submitDeviceControl rings at any nibble) ++ */ ++ bool park; + }; + + static inline bool g5p_wk_mode_valid(u32 mode) +@@ -233,7 +237,7 @@ static inline u32 g5p_wk_ring(const struct g5p_wk_io *io, u32 e, u32 *fence, + if (*fence & ~7U) + return G5P_WK_SKIP_FENCE; + *gpc = io->r1(io->ctx, G5P_WK_R1_GPC); +- if (!G5P_GPC_ON(*gpc)) ++ if (!G5P_GPC_ON(*gpc) && !(io->park && io->r1(io->ctx, 0))) + return G5P_WK_SKIP_GPC; + /* the publish's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ + io->wbarrier(io->ctx); +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index d5f44c3..5576a41 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -389,11 +389,118 @@ static int g5p_fwclock_show(struct seq_file *m, void *unused) + kvfree(offsets); + return 0; + } ++ ++/* ++ * Qualification control for the firmware's idle park (self-park), root only. Reading prints ++ * the RAM state: the latches, ctrl+0x20 and the policy record's live words (no MMIO). Writes: ++ * "park 1" arm (a complete epoch only); "park 0" before the first arm only, ++ * a park epoch is not disarmed (-EBUSY); ++ * "pin 0" the firmware governor owns the target (the fixed policy); ++ * "pin N" N in 100..400 by 100: ctrl+0x1038/0x1040 = N, the kext's override ++ * input, copied by the governor into policy+0x68c/0x6f4, so the target ++ * is N / 100 from the next park -> wake on. ++ * Nothing here wakes, boots or reads the GPU; a value takes effect at the firmware's next tick. ++ */ ++static int g5p_self_park_show(struct seq_file *m, void *unused) ++{ ++ struct g5p_drm *d = m->private; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b; ++ u8 *ctl; ++ u32 p; ++ ++ mutex_lock(&g->lock); ++ b = g->boot; ++ if (!b || !b->maps_ready) { ++ seq_puts(m, "available=0\n"); ++ goto out; ++ } ++ p = b->layout.offset[G5P_PERF_POLICY]; ++ ctl = g5p_boot_shared(b, G5P_PERF_CTL); ++ dma_rmb(); ++ seq_printf(m, "self_park=%u armed=%u park_epoch=%u ctl20=%u\n", ++ b->self_park, b->self_park_armed, b->park_epoch, ++ READ_ONCE(*(u32 *)(ctl + G5P_PARK_CTL_FLAG))); ++ seq_printf(m, "ovr=%u/%u ceiling=%u\n", ++ READ_ONCE(*(u32 *)(ctl + 0x1038)), ++ READ_ONCE(*(u32 *)(ctl + 0x1040)), ++ READ_ONCE(*(u32 *)(ctl + 0x102c))); ++ seq_printf(m, "policy0=%u sample_ms=%u state=%u target=%u pstate=%u x=%u\n", ++ g5p_dc_root_word(b, p), g5p_dc_root_word(b, p + 4), ++ g5p_dc_root_word(b, p + 8), g5p_dc_root_word(b, p + 0x28), ++ g5p_dc_root_word(b, p + 0x34), g5p_dc_root_word(b, p + 0x38)); ++ seq_printf(m, "alive=%u stopped=%u mmio=0\n", b->alive, b->stopped); ++out: ++ mutex_unlock(&g->lock); ++ return 0; ++} ++ ++static int g5p_self_park_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, g5p_self_park_show, inode->i_private); ++} ++ ++static ssize_t g5p_self_park_write(struct file *file, const char __user *ubuf, ++ size_t len, loff_t *pos) ++{ ++ struct seq_file *m = file->private_data; ++ struct g5p_drm *d = m->private; ++ struct g5p_diag *g = d->core; ++ struct g5p_boot *b; ++ char buf[16], what[8]; ++ u32 v; ++ int ret = len; ++ ++ if (len >= sizeof(buf)) ++ return -EINVAL; ++ if (copy_from_user(buf, ubuf, len)) ++ return -EFAULT; ++ buf[len] = 0; ++ if (sscanf(buf, "%7s %u", what, &v) != 2) ++ return -EINVAL; ++ mutex_lock(&g->lock); ++ b = g->boot; ++ if (!b || !b->maps_ready || !b->self_park) { ++ ret = -ENODEV; ++ } else if (!strcmp(what, "park") && !v && b->park_epoch) { ++ /* a live park epoch is never disarmed: the wake needs the armed record */ ++ ret = -EBUSY; ++ } else if (!strcmp(what, "park") && v <= 1) { ++ g5p_park_set(g, v, "debugfs"); ++ if (v && !b->self_park_armed) ++ ret = -EBUSY; ++ } else if (!strcmp(what, "pin") && v <= G5P_PERF_TOP && !(v % 100)) { ++ u8 *ctl = g5p_boot_shared(b, G5P_PERF_CTL); ++ ++ WRITE_ONCE(*(u32 *)(ctl + 0x1038), v ? G5P_PERF_TOP : 0); ++ WRITE_ONCE(*(u32 *)(ctl + 0x1040), v); ++ dma_wmb(); ++ (void)READ_ONCE(*(u32 *)(ctl + 0x1040)); ++ dev_info(g->dev, "G5P self-park pin=%u ovr=%u/%u end=park\n", v, ++ v ? G5P_PERF_TOP : 0, v); ++ } else { ++ ret = -EINVAL; ++ } ++ mutex_unlock(&g->lock); ++ return ret; ++} ++ ++static const struct file_operations g5p_self_park_fops = { ++ .owner = THIS_MODULE, ++ .open = g5p_self_park_open, ++ .read = seq_read, ++ .write = g5p_self_park_write, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ + static void g5p_jt_debugfs_init(struct drm_minor *minor) + { + struct g5p_drm *d = to_g5p(minor->dev); + debugfs_create_file("g5p_gpu_stats", 0600, d->drm.debugfs_root, d, + &g5p_gpu_stats_fops); ++ debugfs_create_file("g5p_self_park", 0600, d->drm.debugfs_root, d, ++ &g5p_self_park_fops); + } + + static inline struct g5p_bo *to_g5p_bo(struct drm_gem_object *obj) +@@ -1030,6 +1137,10 @@ static void g5p_autoboot_work(struct work_struct *work) + d->autoboot_phase++; + } + dev_info(g->dev, "autoboot complete: GPU up\n"); ++ /* self-park: the epoch is complete, so the firmware may now park at idle */ ++ mutex_lock(&g->lock); ++ g5p_park_set(g, true, "autoboot"); ++ mutex_unlock(&g->lock); + g5p_idle_queue(d); + } + +diff --git a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +index 2873ca7..4a32078 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-compute-v1.inc +@@ -163,9 +163,10 @@ static u32 g5p_cu_wait(struct g5p_drm *d, struct g5p_diag *g, u32 seq, + */ + static int g5p_cu_wake(struct g5p_diag *g, u32 seq, u32 start, u32 target) + { +- const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, +- g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; + struct g5p_boot *b = g->boot; ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0, ++ .park = READ_ONCE(b->park_epoch) }; + u32 *ring = (u32 *)g5p_boot_shared(b, 0x68); + u8 *p = g5p_cu_page(b); + struct g5p_wk_in in; +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 7b4199c..a2f7501 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -88,6 +88,17 @@ static const bool render_user_pm_record = true; + + static const bool render_user_ahead_on_submit = true; + ++/* ++ * Self-park (perf-envelope.h): a parked firmware loses GPC power between two jobs and reboots ++ * on the next doorbell. The clean-PM shortcut above (tag4 instead of the TA's PM record) was ++ * measured only on never-park epochs, so a park epoch sends the record with every TA, the ++ * behaviour before the shortcut. ++ */ ++static bool g5p_rv3_pm_always(struct g5p_boot *b) ++{ ++ return READ_ONCE(b->park_epoch); ++} ++ + /* F5c: a job joined a FIFO, or a queued head's input signalled. Any context: the submit path, + * or a fence callback under the fence lock (possibly hard IRQ): a counter and a wake only, never + * the start itself. The full barrier orders the FIFO add (or the signal) before the armed look, +@@ -966,9 +977,10 @@ static s64 g5p_rv3_wk_ms(u64 from_ns, u64 to_ns) + static void g5p_rv3_wake(struct g5p_diag *g, struct g5p_render *r, + struct g5p_rv3 *v, u32 e, u32 start, u32 seq) + { +- const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, +- g5p_rv3_wk_wmb, g5p_rv3_wk_w0 }; + struct g5p_boot *b = g->boot; ++ const struct g5p_wk_io io = { g, g5p_rv3_wk_w1, g5p_rv3_wk_r1, ++ g5p_rv3_wk_wmb, g5p_rv3_wk_w0, ++ .park = READ_ONCE(b->park_epoch) }; + struct g5p_drm *d = r->ctx->gpu; + u32 *ring = (u32 *)g5p_boot_shared(b, e ? 0x64 : 0x60); + u8 *p = g5p_ru_page(b, e); +@@ -1159,13 +1171,19 @@ static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, + n++; + } + if (!g5p_gpc_ok(gpc)) { +- did = "skip-gpc"; +- goto out; ++ /* self-park: a parked firmware wakes through this doorbell */ ++ if (!g5p_park_may_ring(READ_ONCE(b->park_epoch), gpc, ++ readl(g->window[1]))) { ++ did = "skip-gpc"; ++ goto out; ++ } ++ did = "rang-parked"; ++ } else { ++ did = "rang"; + } + /* the publication's RAM stores before the MTS write (Apple: dmb ish before vt+0x80) */ + wmb(); + writel(0x10, g->rgx + 0xb00); +- did = "rang"; + out: + spin_unlock_irqrestore(&b->event_lock, flags); + dev_info( +@@ -3036,7 +3054,8 @@ static void g5p_rv3_ahead_start(struct g5p_drm *d, struct g5p_render *r, + g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, js, + g5p_rv3_pm_id_of(rd, r, pm2), v->abs, + !READ_ONCE(render_user_pm_record) || +- (pm2 ? r->pm2_dirty : r->pm_dirty)); ++ g5p_rv3_pm_always(b) || ++ (pm2 ? r->pm2_dirty : r->pm_dirty)); + r->submitted[e] = false; + r->consumed[e] = false; + r->retired[e] = false; +@@ -3269,7 +3288,8 @@ static int g5p_rv3_execute(struct g5p_drm *d, struct g5p_render *r, + g5p_ru_meta(0) + G5P_R22_STAMP + 4, 0, + g5p_rv3_js(rd, seq), g5p_rv3_pm_id(rd, r), v->abs, + !READ_ONCE(render_user_pm_record) || +- r->pm_set < 0 || r->pm_dirty); ++ g5p_rv3_pm_always(b) || r->pm_set < 0 || ++ r->pm_dirty); + r->submitted[e] = false; + r->consumed[e] = false; + r->retired[e] = false; diff --git a/kernel/patches/drivers/drm/0032-drm-apple-g5p-certify-system-sleep-on-the-firmware-s.patch b/kernel/patches/drivers/drm/0032-drm-apple-g5p-certify-system-sleep-on-the-firmware-s.patch new file mode 100644 index 00000000..0987d5a8 --- /dev/null +++ b/kernel/patches/drivers/drm/0032-drm-apple-g5p-certify-system-sleep-on-the-firmware-s.patch @@ -0,0 +1,139 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 13:28:33 +0100 +Subject: [PATCH] drm/apple-g5p: certify system sleep on the firmware's own + park + +With the firmware's idle park armed (ctrl+0x20, see the self-park block +in perf-envelope.h), Apple's quiescent certificate can hold on an idle +GPU. Make the sleep path match it: + +- never-park now means "the park is not armed": ctrl+0x20 is 0 in RAM or +the arm was refused. The idle tail returns at once in that case +(0x10009bec), so such an epoch can never certify. The previous test +(pinned envelope, policy+4 = 0) no longer describes the shipping record. +- prepare samples the certificate for up to 50 ms (was 20 ms), and +suspend re-samples it for the same bound after the DC3 detach instead of +once: the detach wakes a parked firmware, which parks again only after +its rings drain. - resume clears ctrl+0x20 and the park epoch before the +retained boot, so the boot's ALIVE and GPC proofs run on a firmware +that cannot park, and re-arms it after the DS108 claim, as every other +bring-up does. + +prepare's gates, the stop sequence, the platform-off and the failure +handling are unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 60 +++++++++++++++------- + 1 file changed, 42 insertions(+), 18 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index effe371..9da1850 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -125,6 +125,23 @@ EXPORT_SYMBOL_GPL(apple_g5p_system_sleep_complete); + + /* Client work admitted before the client gate closed drains in this bound. */ + #define G5P_SLEEP_DRAIN_MS 1000 ++/* ++ * The firmware's own park (self-park) follows its idle tail; a DC3 detach wakes a parked ++ * firmware, which parks again once its rings drain. Both certificate waits use this bound. ++ */ ++#define G5P_SLEEP_PARK_MS 50 ++ ++static bool g5p_sleep_certificate_wait(struct g5p_diag *g) ++{ ++ u64 deadline = ktime_get_ns() + G5P_SLEEP_PARK_MS * NSEC_PER_MSEC; ++ ++ do { ++ if (g5p_sleep_certificate(g, false)) ++ return true; ++ usleep_range(100, 200); ++ } while (ktime_get_ns() < deadline); ++ return false; ++} + + static int g5p_system_sleep_prepare(struct device *dev) + { +@@ -171,15 +188,10 @@ static int g5p_system_sleep_prepare(struct device *dev) + /* No submitters, queued work or IRQ-side ALIVE action can publish now. */ + if (g5p_boot_drain(g) || !g5p_sleep_rings(g)) + goto unlock; +- deadline = ktime_get_ns() + 20 * NSEC_PER_MSEC; +- do { +- if (g5p_sleep_certificate(g, false)) { +- g->sleep_reserved = true; +- ret = 0; +- break; +- } +- usleep_range(100, 200); +- } while (ktime_get_ns() < deadline); ++ if (g5p_sleep_certificate_wait(g)) { ++ g->sleep_reserved = true; ++ ret = 0; ++ } + unlock: + mutex_unlock(&g->lock); + if (!ret) { +@@ -200,16 +212,20 @@ refuse_client: + } + + /* +- * The pinned perf envelope (perf-envelope.h: floor = pin, sampler off, +- * policy+4 = 0) keeps the firmware from ever running its idle park +- * (0x1001cec8: GPC 0 and R1+0 bit 29). Apple's isQuiescent (6ca66e4) needs +- * that park, and powerDownIfQuiescent (c81b18) neither acknowledges nor +- * touches hardware without it: it waits for the firmware to park. RAM only. ++ * The firmware runs its idle park (0x1001cec8: GPC 0 and R1+0 bits 30:29) only ++ * with ctrl+0x20 set (self-park: the idle tail 0x10009bec returns at once on ++ * priv+0x188 == 0). Apple's isQuiescent (6ca66e4) needs that park, and ++ * powerDownIfQuiescent (c81b18) neither acknowledges nor touches hardware ++ * without it: it waits for the firmware to park. So an epoch whose park is not ++ * armed (ctrl+0x20 = 0 in RAM: bring-up, a refused arm, debugfs "park 0") can ++ * never certify. RAM only. + */ + static bool g5p_sleep_never_park(struct g5p_boot *b) + { +- return b->maps_ready && b->gpu_perf && +- !g5p_dc_root_word(b, b->layout.offset[0x4a] + 4); ++ return b->maps_ready && ++ (!READ_ONCE(b->self_park_armed) || ++ !READ_ONCE(*(u32 *)(g5p_boot_shared(b, G5P_PERF_CTL) + ++ G5P_PARK_CTL_FLAG))); + } + + /* The readiness reason: one token, no white space or colon. */ +@@ -355,8 +371,10 @@ static int g5p_system_suspend(struct device *dev) + ret = held ? g5p_rv3_unbind_slots(d, slots, held, "sleep") : 0; + if (ret) + goto failed; +- /* Detach has changed firmware ownership: any later failure is terminal. */ +- if (!g5p_sleep_rings(g) || !g5p_sleep_certificate(g, false)) { ++ /* Detach has changed firmware ownership: any later failure is terminal. ++ * The detach woke a parked firmware; it parks again once the DC ring drains. ++ */ ++ if (!g5p_sleep_rings(g) || !g5p_sleep_certificate_wait(g)) { + ret = -EBUSY; + goto failed; + } +@@ -427,6 +445,11 @@ static int g5p_system_resume(struct device *dev) + goto failed; + g->fender_ready = true; + g->mmio_off = false; ++ /* self-park: this boot's ALIVE and GPC proofs run before the firmware may ++ * park (B2: a park before the first-ALIVE read latched a power abort). ++ */ ++ g5p_park_set(g, false, "resume-boot"); ++ WRITE_ONCE(b->park_epoch, false); + b->stopped = false; + ret = g5p_boot_program_stream(g, true); + g5p_ds_hook_wake(g, ret); +@@ -465,6 +488,7 @@ static int g5p_system_resume(struct device *dev) + g5p_td_op_aic_ack(g); + g->sleep_irq_off = false; + enable_irq(b->irq); ++ g5p_park_set(g, true, "resume"); + dev_info(dev, "GPU-PM on: fresh ALIVE gen=%u epoch=%u\n", g5p_ds_gen(), + d->rdev->sleep_rebuilds); + mutex_unlock(&g->lock); diff --git a/kernel/patches/drivers/drm/0033-drm-apple-g5p-ring-a-parked-firmware-without-the-GPC.patch b/kernel/patches/drivers/drm/0033-drm-apple-g5p-ring-a-parked-firmware-without-the-GPC.patch new file mode 100644 index 00000000..cb02f8a9 --- /dev/null +++ b/kernel/patches/drivers/drm/0033-drm-apple-g5p-ring-a-parked-firmware-without-the-GPC.patch @@ -0,0 +1,129 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 13:28:33 +0100 +Subject: [PATCH] drm/apple-g5p: ring a parked firmware without the GPC settle + wait + +In a park epoch every doorbell still ran the GPC settle: after +R1+0x14 = 7 it re-read R1+0x1000 every 10 us for up to 2 ms, waiting for +a settled powered GPC, and only then rang R0+0xb00 at any nibble. A +parked firmware powers its GPC on only after the doorbell, so the wait +always ran to its end: in one test run the DC kicks rang 455 times at +gpc=00000000 after settle=185..186 reads, and the per-job publish spent +394..410 us of host time in the bell (1..2 us without park). + +Apple's AGXMetaFirmware_submitDeviceControl (20H350 fffffff006ca05d4) +issues Fender +0x80 (R1+0x14 = 7 and its read-back), one +0x18 nibble +read, at nibble 0 only the +0x70/+0x78/+0x90 read-backs, then R0+0xb00 = +0x10, with no wait. Do the same in park epochs: the r74f bells, the DC +kicks and the wake110 publish (render and compute) read the GPC once and +ring. The refusal of a dead wrapper (R1+0 == 0) is kept. Never-park +epochs keep the settle unchanged. g5p_prepare_kick already read the GPC +once. + +Not device-proven. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/boot-r74f.inc | 8 +++++--- + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 18 ++++++++++++------ + 2 files changed, 17 insertions(+), 9 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/boot-r74f.inc b/drivers/gpu/apple-g5p/boot-r74f.inc +index 9ab4947..c7f234a 100644 +--- a/drivers/gpu/apple-g5p/boot-r74f.inc ++++ b/drivers/gpu/apple-g5p/boot-r74f.inc +@@ -71,6 +71,7 @@ static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, + u32 queued = (f->target - f->start) & 255; + u32 progressed = (consumer - f->start) & 255; + u32 expected = f->kind == G5P_R74F_DC3 ? 2 : 1; ++ bool park = READ_ONCE(b->park_epoch); + + if (READ_ONCE(b->failed)) + return -EIO; +@@ -90,8 +91,10 @@ static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, + *state = readl(g->window[1] + 0x1000); + /* perf60 (gpc_any_pstate): a GPC request in flight or a not-yet-awake GPC after the wake + * above; re-read for up to 2 ms, as Apple's setGPCPerformanceState waits for bit 4. Reads only. ++ * Not in a park epoch: a parked GPC powers on only after this doorbell, and Apple's ++ * submitDeviceControl (6ca05d4) reads the nibble once and rings. + */ +- { ++ if (!park) { + u64 t0 = ktime_get_mono_fast_ns(); + + while (!g5p_gpc_ok(*state) && ktime_get_mono_fast_ns() - t0 < +@@ -102,8 +105,7 @@ static int g5p_r74f_ring(struct g5p_diag *g, const struct g5p_r74f_bell *f, + } + /* self-park: a parked firmware wakes through this doorbell (Apple: any nibble) */ + if (!g5p_gpc_ok(*state) && +- !g5p_park_may_ring(READ_ONCE(b->park_epoch), *state, +- readl(g->window[1]))) ++ !g5p_park_may_ring(park, *state, readl(g->window[1]))) + return -EHOSTDOWN; + dma_wmb(); + writel(f->kind == G5P_R74F_3D ? 0x11 : 0x10, g->rgx + 0xb00); +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index a2f7501..70b4ba2 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -857,18 +857,22 @@ static void g5p_rv3_wk_w1(void *ctx, u32 reg, u32 value) + /* The GPC state read of g5p_wk_ring() comes right after its R1+0x14 = 7 wake: in bell mode + * (render_user_wake=2) it waits up to G5P_RV3_WK_SETTLE_US, re-reading every 10 us, for a settled + * powered pstate (bit 4 clear, nibble nonzero), as Apple's setGPCPerformanceState waits for bit 4. +- * Reads only; the ring's own check of the final value is unchanged. ++ * Reads only; the ring's own check of the final value is unchanged. Not in a park epoch: a ++ * parked GPC powers on only after the doorbell, and Apple's submitDeviceControl (6ca05d4) reads ++ * the nibble once and rings. The epoch only goes from never-park to park while it is live, so ++ * if this read and the caller's io.park disagree the ring refuses an off GPC (fail closed). + */ + #define G5P_RV3_WK_SETTLE_US 2000U + + static u32 g5p_rv3_wk_r1(void *ctx, u32 reg) + { +- void __iomem *r1 = ((struct g5p_diag *)ctx)->window[1]; ++ struct g5p_diag *g = ctx; ++ void __iomem *r1 = g->window[1]; + u32 raw = readl(r1 + reg); + u64 t0; + + if (reg != G5P_WK_R1_GPC || (2) != G5P_WK_ALWAYS || +- (!(raw & 0x10) && (raw & 0xf))) ++ (!(raw & 0x10) && (raw & 0xf)) || READ_ONCE(g->boot->park_epoch)) + return raw; + t0 = ktime_get_mono_fast_ns(); + while (((raw & 0x10) || !(raw & 0xf)) && +@@ -1137,11 +1141,13 @@ static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, + u32 fence = U32_MAX, gpc = U32_MAX, power, cons, n = 0; + const char *did = "skip-guard"; + unsigned long flags; ++ bool park; + u64 t0; + + ; + ; + spin_lock_irqsave(&b->event_lock, flags); ++ park = READ_ONCE(b->park_epoch); + dma_rmb(); + cons = READ_ONCE(ctl[0]); + power = g5p_wk_power_mask(READ_ONCE(g->powered), +@@ -1163,7 +1169,8 @@ static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, + } + t0 = ktime_get_mono_fast_ns(); + gpc = readl(g->window[1] + 0x1000); +- while (!g5p_gpc_ok(gpc) && ++ /* a park epoch reads once and rings, as Apple does (6ca05d4) */ ++ while (!park && !g5p_gpc_ok(gpc) && + ktime_get_mono_fast_ns() - t0 < + (u64)G5P_RV3_DC_SETTLE_US * NSEC_PER_USEC) { + udelay(10); +@@ -1172,8 +1179,7 @@ static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, + } + if (!g5p_gpc_ok(gpc)) { + /* self-park: a parked firmware wakes through this doorbell */ +- if (!g5p_park_may_ring(READ_ONCE(b->park_epoch), gpc, +- readl(g->window[1]))) { ++ if (!g5p_park_may_ring(park, gpc, readl(g->window[1]))) { + did = "skip-gpc"; + goto out; + } diff --git a/kernel/patches/drivers/drm/0034-drm-apple-g5p-name-every-system-sleep-refusal.patch b/kernel/patches/drivers/drm/0034-drm-apple-g5p-name-every-system-sleep-refusal.patch new file mode 100644 index 00000000..06f21bcb --- /dev/null +++ b/kernel/patches/drivers/drm/0034-drm-apple-g5p-name-every-system-sleep-refusal.patch @@ -0,0 +1,491 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:26:20 +0100 +Subject: [PATCH] drm/apple-g5p: name every system sleep refusal + +Prepare refused with one line for every cause ("busy or no quiescent +certificate") and with no line at all for its early exits (no boot, a +failed epoch, a gate already closed). Suspend and resume failures logged +only "GPU-PM frozen". A refused device test could not say which +condition held. + +Replace g5p_sleep_idle() by g5p_sleep_busy(), which returns a token for +the first busy condition and says whether the client drain can clear it. +Prepare logs the stage and reason of every refusal (and the R1 words +when the certificate does not hold), plus one line when the client drain +hits its bound. Its client drain stops early on a state no drain clears. +The readiness query refuses those states too, instead of answering +"ready" for a GPU that prepare will refuse after a 1 s drain. Suspend +names its pre-detach refusal and each failing step, including the +re-park after the DC3 detach and its duration; resume names its failing +step. Register reads are the certificate's own, taken only where it has +just read them; none follow a failed transition. + +Behaviour is otherwise unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 269 +++++++++++++++++---- + 1 file changed, 221 insertions(+), 48 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 9da18505b..6a89e5f48 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -3,24 +3,51 @@ + * Admission is reserved through complete, including another device's abort. + * No terminal-retire latch is used for a reversible sleep attempt. + */ +-static bool g5p_sleep_idle(struct g5p_drm *d) ++/* ++ * Why the host side is not idle: one token (no white space or colon), or NULL ++ * when idle. *drains is set when the client drain in prepare can clear it ++ * (work in flight); otherwise nothing at idle clears it and only a refusal is ++ * honest. ++ */ ++static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) + { + struct g5p_rdev *rd = d->rdev; + struct g5p_render *r; + unsigned long flags; ++ const char *why = NULL; + bool idle; + +- if (rd->cu_sequence || atomic_read(&d->accepted) || +- atomic_read(&d->r8_queued) || READ_ONCE(rd->executing) || +- rd->ahead || rd->stuck || rd->recovering || rd->dc_pending || +- rd->close_deferred[0] || rd->close_deferred[1] || +- g5p_ru_term_failed(&g5p_rv3_term)) +- return false; ++ *drains = false; ++ /* No CDM stamp certificate yet (lane GS): compute epochs refuse. */ ++ if (rd->cu_sequence) ++ return "compute-epoch"; ++ if (g5p_ru_term_failed(&g5p_rv3_term)) ++ return "terminal"; ++ /* A bind's DC3 pair the firmware never consumed; only a later DC op clears it. */ ++ if (rd->dc_pending) ++ return "dc-pending"; ++ /* Released only after a job of a third context (render-user-pmset.h). */ ++ if (rd->close_deferred[0] || rd->close_deferred[1]) ++ return "close-deferred"; ++ *drains = true; ++ /* A lost-job drain or recovery may finish inside the client drain bound. */ ++ if (rd->stuck) ++ return "stuck"; ++ if (rd->recovering) ++ return "recovering"; ++ if (atomic_read(&d->accepted)) ++ return "accepted"; ++ if (atomic_read(&d->r8_queued)) ++ return "r8-queued"; ++ if (READ_ONCE(rd->executing)) ++ return "executing"; ++ if (rd->ahead) ++ return "ahead"; + spin_lock_irqsave(&rd->sched_lock, flags); + idle = list_empty(&rd->sched.run); + spin_unlock_irqrestore(&rd->sched_lock, flags); + if (!idle) +- return false; ++ return "queued"; + /* Re-read the latest TA/3D stamps, including the depth-2 record slot. + * A CPU retirement flag alone is not Apple's all-stamps certificate. + */ +@@ -30,15 +57,35 @@ static bool g5p_sleep_idle(struct g5p_drm *d) + g5p_ru_meta(e), e, rd->sequence, + g5p_rv3_js(rd, rd->sequence), + rd->sequence)) +- return false; ++ return e ? "stamp-3d" : "stamp-ta"; + mutex_lock(&rd->lock); + list_for_each_entry(r, &rd->contexts, rdev_link) + for (u32 i = 0; i < 2; i++) +- if (r->submitted[i] && +- (!r->consumed[i] || !r->retired[i])) +- idle = false; ++ if (r->submitted[i] && !why) ++ why = !r->consumed[i] ? ++ (i ? "unconsumed-3d" : "unconsumed-ta") : ++ !r->retired[i] ? ++ (i ? "unretired-3d" : "unretired-ta") : ++ NULL; + mutex_unlock(&rd->lock); +- return idle; ++ return why; ++} ++ ++/* ++ * The certificate's own R1 reads, in its order and under its conditions ++ * (R1+8 only with bits 30:29 set), for a refusal line. Only where the ++ * certificate itself has just read them. ++ */ ++static void g5p_sleep_sample(struct g5p_diag *g, u32 *state, u32 *mode, u32 *r8) ++{ ++ *r8 = 0; ++ *state = readl(g->window[1] + 0x1000); ++ *mode = 0; ++ if (*state & 0x1f) ++ return; ++ *mode = readl(g->window[1]); ++ if ((*mode & 0x60000000) == 0x60000000) ++ *r8 = readl(g->window[1] + 8); + } + + /* R1 only, as Apple's isQuiescent/isStableOff. Never infer bus retirement +@@ -149,59 +196,128 @@ static int g5p_system_sleep_prepare(struct device *dev) + struct g5p_drm *d; + struct g5p_boot *b; + struct g5p_kq_ctx c = { .g = g }; +- u64 deadline; +- int ret = -EBUSY; +- +- if (!g || !g->event_cookie || !g->boot) ++ const char *stage = "entry", *busy; ++ char why[64] = ""; ++ u64 t0 = ktime_get_ns(), deadline, drain_ns; ++ u32 cons, prod, state, mode, r8; ++ bool drains; ++ int ret = -EBUSY, users; ++ ++ /* Every refusal below names its stage and reason: no silent -EBUSY. */ ++ if (!g || !g->event_cookie || !g->boot) { ++ dev_info(dev, "GPU-PM sleep refused: stage=entry why=%s\n", ++ !g ? "no-core" : !g->event_cookie ? "no-drm" : "no-boot"); + return -EBUSY; ++ } + d = g->event_cookie; +- if (READ_ONCE(g->sleep_reserved)) +- return g->sleep_failed ? -EBUSY : 0; ++ if (READ_ONCE(g->sleep_reserved)) { ++ if (!g->sleep_failed) ++ return 0; ++ dev_info(dev, "GPU-PM sleep refused: stage=entry why=sleep-failed\n"); ++ return -EBUSY; ++ } + /* + * Stage 1: shut client admission only. New client work and closes wait, + * while the scheduler keeps running so accepted jobs retire. + */ +- if (atomic_fetch_or(BIT(31), &d->client_gate) < 0) ++ if (atomic_fetch_or(BIT(31), &d->client_gate) < 0) { ++ dev_info(dev, "GPU-PM sleep refused: stage=client why=client-gate-closed\n"); + return -EBUSY; +- deadline = ktime_get_ns() + G5P_SLEEP_DRAIN_MS * NSEC_PER_MSEC; +- while (((atomic_read(&d->client_gate) & INT_MAX) || !g5p_sleep_idle(d)) && +- ktime_get_ns() < deadline) ++ } ++ deadline = t0 + G5P_SLEEP_DRAIN_MS * NSEC_PER_MSEC; ++ for (;;) { ++ users = atomic_read(&d->client_gate) & INT_MAX; ++ busy = g5p_sleep_busy(d, &drains); ++ /* Stop waiting on what no drain clears; stage 2 names it. */ ++ if ((!users && !busy) || (busy && !drains) || ++ ktime_get_ns() >= deadline) ++ break; + usleep_range(500, 1000); ++ } ++ drain_ns = ktime_get_ns() - t0; ++ if (users || busy) ++ dev_info(dev, "GPU-PM sleep drain bound: client_users=%d busy=%s drain_ms=%llu\n", ++ users, busy ?: "none", drain_ns / NSEC_PER_MSEC); + /* Stage 2: shut the host gate and certify, unchanged. */ +- if (atomic_fetch_or(BIT(31), &g->sleep_gate) < 0) ++ stage = "host"; ++ if (atomic_fetch_or(BIT(31), &g->sleep_gate) < 0) { ++ dev_info(dev, "GPU-PM sleep refused: stage=host why=host-gate-closed\n"); + goto refuse_client; ++ } + deadline = ktime_get_ns() + 20 * NSEC_PER_MSEC; + while ((atomic_read(&g->sleep_gate) & INT_MAX) && + ktime_get_ns() < deadline) + usleep_range(100, 200); +- if (atomic_read(&g->sleep_gate) & INT_MAX) ++ users = atomic_read(&g->sleep_gate) & INT_MAX; ++ if (users) { ++ snprintf(why, sizeof(why), "host-users=%d", users); + goto refuse; +- if (!g5p_kq_lock(&c)) ++ } ++ stage = "lock"; ++ if (!g5p_kq_lock(&c)) { ++ strscpy(why, "core-lock"); + goto refuse; ++ } ++ stage = "state"; + b = g->boot; +- if (g->sleep_failed || g5p_r8_admit_closed(&g5p_r8_state.admit) || !b || +- !g->powered || !g->fender_ready || g->mmio_off || !b->released || +- !b->alive || b->failed || b->event_failed || b->stopped || +- !b->irq_ready || !b->r73b || +- READ_ONCE(b->r73b->early_power_abort) || !g5p_sleep_idle(d)) ++ busy = !b ? "no-boot" : ++ g->sleep_failed ? "sleep-failed" : ++ g5p_r8_admit_closed(&g5p_r8_state.admit) ? "r8-admission-closed" : ++ !g->powered ? "not-powered" : ++ !g->fender_ready ? "fender-not-ready" : ++ g->mmio_off ? "mmio-off" : ++ !b->released ? "not-released" : ++ !b->alive ? "not-alive" : ++ b->failed ? "firmware-failed" : ++ b->event_failed ? "event-failed" : ++ b->stopped ? "stopped" : ++ !b->irq_ready ? "irq-not-ready" : ++ !b->r73b ? "no-r73b" : ++ READ_ONCE(b->r73b->early_power_abort) ? "power-abort" : ++ g5p_sleep_busy(d, &drains); ++ if (busy) { ++ strscpy(why, busy); + goto unlock; ++ } + /* No submitters, queued work or IRQ-side ALIVE action can publish now. */ +- if (g5p_boot_drain(g) || !g5p_sleep_rings(g)) ++ stage = "drain"; ++ ret = g5p_boot_drain(g); ++ if (ret) { ++ snprintf(why, sizeof(why), "boot-drain=%d", ret); ++ ret = -EBUSY; + goto unlock; ++ } ++ ret = -EBUSY; ++ for (u32 i = 0; i < G5P_TD_NRING; i++) { ++ if (g5p_td_op_ring_read(g, i, &cons, &prod)) { ++ snprintf(why, sizeof(why), "ring%u-unreadable", i); ++ goto unlock; ++ } ++ if (cons > 255 || cons != prod) { ++ snprintf(why, sizeof(why), "ring%u=%u/%u", i, cons, prod); ++ goto unlock; ++ } ++ } ++ stage = "certificate"; + if (g5p_sleep_certificate_wait(g)) { + g->sleep_reserved = true; + ret = 0; ++ } else { ++ g5p_sleep_sample(g, &state, &mode, &r8); ++ snprintf(why, sizeof(why), "not-quiescent/r1_1000=%08x/r1_0=%08x/r1_8=%08x", ++ state, mode, r8); + } + unlock: + mutex_unlock(&g->lock); + if (!ret) { +- dev_info(dev, "GPU-PM prepared: admission reserved\n"); ++ dev_info(dev, "GPU-PM prepared: admission reserved ms=%llu\n", ++ (ktime_get_ns() - t0) / NSEC_PER_MSEC); + return 0; + } + refuse: + g5p_sleep_reopen(g); +- dev_info(dev, +- "GPU-PM sleep refused: busy or no quiescent certificate\n"); ++ dev_info(dev, "GPU-PM sleep refused: stage=%s why=%s ms=%llu\n", stage, ++ why, (ktime_get_ns() - t0) / NSEC_PER_MSEC); + return -EBUSY; + refuse_client: + atomic_andnot(BIT(31), &d->client_gate); +@@ -244,7 +360,9 @@ static __printf(3, 4) int g5p_sleep_why(struct g5p_drm *d, int ret, + * Readiness query: what prepare would decide, without shutting any gate or + * draining firmware events. Accepted or in-flight work is not a refusal, + * since prepare drains it, except under the never-park envelope, where the +- * drained firmware still cannot certify. Register samples are taken only ++ * drained firmware still cannot certify. A busy state no drain clears ++ * (g5p_sleep_busy() without *drains) refuses here exactly as prepare would, ++ * so "ready" is never answered for it. Register samples are taken only + * when the GPU is already idle and the core lock is free, with the same + * reads as g5p_sleep_certificate(); a sampled GPC 0 falls through to it. + */ +@@ -254,7 +372,8 @@ static int g5p_system_sleep_check(struct device *dev) + struct g5p_drm *d; + struct g5p_boot *b; + u32 cons, prod, state, mode = 0, r8 = 0; +- bool never_park; ++ const char *busy; ++ bool never_park, drains; + int ret = 0; + + if (!g || !g->event_cookie) +@@ -286,9 +405,13 @@ static int g5p_system_sleep_check(struct device *dev) + ret = g5p_sleep_why(d, -ENODEV, "not-powered"); + goto unlock; + } +- if (!g5p_sleep_idle(d)) { +- if (never_park) +- ret = g5p_sleep_why(d, -EBUSY, "never-park/busy"); ++ busy = g5p_sleep_busy(d, &drains); ++ if (busy) { ++ /* Prepare drains work in flight; it cannot drain the rest. */ ++ if (never_park || !drains) ++ ret = g5p_sleep_why(d, -EBUSY, "%s%s", ++ never_park ? "never-park/" : "busy/", ++ busy); + goto unlock; + } + for (u32 i = 0; i < G5P_TD_NRING; i++) { +@@ -343,16 +466,35 @@ static int g5p_system_suspend(struct device *dev) + struct g5p_drm *d = g->event_cookie; + struct g5p_boot *b = g->boot; + struct g5p_render **drop, *r; +- u32 slots[G5P_RU_USER_SLOTS], held, count = 0, n = 0; ++ u32 slots[G5P_RU_USER_SLOTS], held, count = 0, n = 0, state, mode, r8; ++ const char *busy, *step = "entry"; ++ bool drains; ++ u64 t0; + int ret; + +- if (!g->sleep_reserved || g->sleep_failed) ++ if (!g->sleep_reserved || g->sleep_failed) { ++ dev_info(dev, "GPU-PM suspend refused: why=%s\n", ++ g->sleep_failed ? "sleep-failed" : "not-reserved"); + return -EBUSY; ++ } + mutex_lock(&g->lock); +- if (!READ_ONCE(b->alive) || READ_ONCE(b->failed) || +- READ_ONCE(b->event_failed) || READ_ONCE(b->stopped) || +- READ_ONCE(b->r73b->early_power_abort) || !g5p_sleep_idle(d) || +- !g5p_sleep_rings(g) || !g5p_sleep_certificate(g, false)) { ++ busy = !READ_ONCE(b->alive) ? "not-alive" : ++ READ_ONCE(b->failed) ? "firmware-failed" : ++ READ_ONCE(b->event_failed) ? "event-failed" : ++ READ_ONCE(b->stopped) ? "stopped" : ++ READ_ONCE(b->r73b->early_power_abort) ? "power-abort" : ++ g5p_sleep_busy(d, &drains); ++ if (!busy && !g5p_sleep_rings(g)) ++ busy = "rings"; ++ if (!busy && !g5p_sleep_certificate(g, false)) { ++ g5p_sleep_sample(g, &state, &mode, &r8); ++ dev_info(dev, "GPU-PM suspend certificate r1_1000=%08x r1_0=%08x r1_8=%08x\n", ++ state, mode, r8); ++ busy = "not-quiescent"; ++ } ++ if (busy) { ++ /* Before the detach: reversible, admission reopens at complete. */ ++ dev_info(dev, "GPU-PM suspend refused: why=%s\n", busy); + ret = -EBUSY; + goto out; + } +@@ -362,22 +504,40 @@ static int g5p_system_suspend(struct device *dev) + mutex_unlock(&d->rdev->lock); + drop = kcalloc(max(count, 1U), sizeof(*drop), GFP_KERNEL); + if (!drop) { ++ dev_info(dev, "GPU-PM suspend refused: why=no-memory\n"); + ret = -EBUSY; + goto out; + } + held = g5p_ru_held_slots(g5p_boot_shared(b, 0x86), + g5p_boot_shared(b, 0x84), slots, + ARRAY_SIZE(slots)); ++ dev_info(dev, "GPU-PM suspend detach held=%u contexts=%u deferred=%u/%u\n", ++ held, count, !!d->rdev->close_deferred[0], ++ !!d->rdev->close_deferred[1]); ++ step = "detach"; + ret = held ? g5p_rv3_unbind_slots(d, slots, held, "sleep") : 0; + if (ret) + goto failed; + /* Detach has changed firmware ownership: any later failure is terminal. + * The detach woke a parked firmware; it parks again once the DC ring drains. + */ +- if (!g5p_sleep_rings(g) || !g5p_sleep_certificate_wait(g)) { ++ step = "rings-after-detach"; ++ if (!g5p_sleep_rings(g)) { ++ ret = -EBUSY; ++ goto failed; ++ } ++ step = "park-after-detach"; ++ t0 = ktime_get_ns(); ++ if (!g5p_sleep_certificate_wait(g)) { ++ g5p_sleep_sample(g, &state, &mode, &r8); ++ dev_info(dev, "GPU-PM suspend certificate r1_1000=%08x r1_0=%08x r1_8=%08x\n", ++ state, mode, r8); + ret = -EBUSY; + goto failed; + } ++ dev_info(dev, "GPU-PM suspend parked again after detach us=%llu\n", ++ (ktime_get_ns() - t0) / NSEC_PER_USEC); ++ step = "fender-stop"; + ret = g5p_fender_write(g, 0x10, 0); /* Fender mode 3 */ + if (!ret) + ret = g5p_fender_write(g, 0x20000, 0); +@@ -389,6 +549,8 @@ static int g5p_system_suspend(struct device *dev) + WRITE_ONCE(b->stopped, true); + disable_irq(b->irq); + g->sleep_irq_off = true; ++ if (!ret) ++ step = "stable-off"; + if (ret || !g5p_sleep_certificate(g, true)) { + ret = ret ?: -EIO; + goto failed; +@@ -397,6 +559,7 @@ static int g5p_system_suspend(struct device *dev) + d->rdev->sleep_rebuilds++; + g5p_mmio_gate(g); + g->fender_ready = false; ++ step = "platform-off"; + ret = pm_runtime_put_sync_suspend(dev); + g->powered = false; /* put consumed our reference, including on error */ + if (ret < 0) +@@ -407,6 +570,8 @@ static int g5p_system_suspend(struct device *dev) + ret = 0; + goto free; + failed: ++ /* No MMIO here: g5p_sleep_fail() freezes the epoch. */ ++ dev_err(dev, "GPU-PM suspend failed step=%s ret=%d\n", step, ret); + g5p_sleep_fail(d, ret); + free: + mutex_unlock(&g->lock); +@@ -427,6 +592,7 @@ static int g5p_system_resume(struct device *dev) + struct g5p_drm *d = g->event_cookie; + struct g5p_boot *b = g->boot; + unsigned long deadline; ++ const char *step = "platform-on"; + int ret; + + if (!g->sleep_off) +@@ -440,6 +606,7 @@ static int g5p_system_resume(struct device *dev) + /* A witnessed platform off/on invalidates DS108's old ALIVE generation. + * Retained maps/image are validated, never allocated or reinitialised. + */ ++ step = "ds108-recover"; + ret = g5p_ds_hook_recover(g, 0); + if (ret) + goto failed; +@@ -451,6 +618,7 @@ static int g5p_system_resume(struct device *dev) + g5p_park_set(g, false, "resume-boot"); + WRITE_ONCE(b->park_epoch, false); + b->stopped = false; ++ step = "boot-stream"; + ret = g5p_boot_program_stream(g, true); + g5p_ds_hook_wake(g, ret); + if (ret) +@@ -458,6 +626,7 @@ static int g5p_system_resume(struct device *dev) + g5p_ds_hook_maps(g, 0); + g5p_ds_hook_load(g, 0); + g5p_ds_hook_release(g, 0); ++ step = "alive"; + deadline = jiffies + 3 * HZ; + do { + /* IRQ remains masked. Consume fresh ALIVE in RAM without cmd8 or kicks. */ +@@ -474,12 +643,15 @@ static int g5p_system_resume(struct device *dev) + b->r73b->early_state = raw; + if (!g5p_gpc_ok(raw)) { + WRITE_ONCE(b->r73b->early_power_abort, true); ++ step = "gpc"; + ret = -EHOSTDOWN; + } + } + g5p_ds_hook_alive(g, ret); +- if (!ret) ++ if (!ret) { ++ step = "ds108-claim"; + ret = g5p_ds_hook_claim(g); ++ } + if (ret) + goto failed; + g5p_ru_term_reset(&g5p_rv3_term); +@@ -494,6 +666,7 @@ static int g5p_system_resume(struct device *dev) + mutex_unlock(&g->lock); + return 0; + failed: ++ dev_err(dev, "GPU-PM resume failed step=%s ret=%d\n", step, ret); + g->sleep_boot = false; + g->sleep_off = false; + g5p_sleep_fail(d, ret); diff --git a/kernel/patches/drivers/drm/0035-drm-apple-g5p-admit-system-sleep-with-a-deferred-clo.patch b/kernel/patches/drivers/drm/0035-drm-apple-g5p-admit-system-sleep-with-a-deferred-clo.patch new file mode 100644 index 00000000..c555099f --- /dev/null +++ b/kernel/patches/drivers/drm/0035-drm-apple-g5p-admit-system-sleep-with-a-deferred-clo.patch @@ -0,0 +1,64 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:26:20 +0100 +Subject: [PATCH] drm/apple-g5p: admit system sleep with a deferred close + pending + +A context that closes while the PM may still write its free list back +stays bound until a job of a third context runs (render-user-pmset.h). +With one compositor running that never happens: a client that ran at +boot and exited stays the PM's previous context for good, and system +sleep refused on it at every attempt. On J71t with labwc idle and the +firmware parked, every s2idle attempt drained for 1 s and then refused +("close-defer slots=8/7 ... pm=last" at boot, no "close-deferred" +after). + +The hazard is a write-back on a later job. No job runs between the sleep +reservation and platform off: the client and host gates are shut, the +sleep detach unbinds every slot the firmware holds, and the resumed +firmware's PM starts with no last or previous context +(g5p_rv3_rebuild_host() clears both and every context's retained +binding). The pending close then releases after the next job with a +release plan, as after an in-boot recovery, never an unbind of old-epoch +slots. + +Drop the deferred close from the sleep busy states. The R1 quiescent +certificate, the ring and stamp checks and every other busy state are +unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 13 ++++++++++--- + 1 file changed, 10 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 6a89e5f48..6e3ab71c5 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -8,6 +8,16 @@ + * when idle. *drains is set when the client drain in prepare can clear it + * (work in flight); otherwise nothing at idle clears it and only a refusal is + * honest. ++ * ++ * A deferred close (render-user-pmset.h) is not a reason: it keeps a closed ++ * context's slots bound while the PM may still write its free list back on a ++ * LATER job of the context that followed. No job runs between this ++ * reservation and platform off, the sleep detach unbinds every held slot, and ++ * the resumed firmware's PM starts with no last or previous context ++ * (g5p_rv3_rebuild_host()), so the pending close releases after resume like ++ * after an in-boot recovery. With one compositor running, a context that ++ * closed earlier stays "previous" for good, so refusing on it refused every ++ * sleep. + */ + static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) + { +@@ -26,9 +36,6 @@ static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) + /* A bind's DC3 pair the firmware never consumed; only a later DC op clears it. */ + if (rd->dc_pending) + return "dc-pending"; +- /* Released only after a job of a third context (render-user-pmset.h). */ +- if (rd->close_deferred[0] || rd->close_deferred[1]) +- return "close-deferred"; + *drains = true; + /* A lost-job drain or recovery may finish inside the client drain bound. */ + if (rd->stuck) diff --git a/kernel/patches/drivers/drm/0036-drm-apple-g5p-refuse-system-sleep-while-a-kexec-boot.patch b/kernel/patches/drivers/drm/0036-drm-apple-g5p-refuse-system-sleep-while-a-kexec-boot.patch new file mode 100644 index 00000000..c7d3e9d0 --- /dev/null +++ b/kernel/patches/drivers/drm/0036-drm-apple-g5p-refuse-system-sleep-while-a-kexec-boot.patch @@ -0,0 +1,67 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:41:54 +0100 +Subject: [PATCH] drm/apple-g5p: refuse system sleep while a kexec boot holds + GPU power + +On a kexec boot pmgr-pwrstate keeps every S8000 power state on for the +whole boot (keep_on), so genpd never powers gfx off. A system sleep +there would stop the firmware without the platform off, and resume would +boot it over a GPU that was never power-cycled: a dirty start, which the +dirty-start gate admits only through its one bounded reset at probe. +Apple's wake is platform-on plus a firmware boot. + +Refuse such a sleep before any gate is shut, named "warm-gfx-power-held" +in the log and in /sys/power/suspend_ready, until a warm boot can power +the GPU off as a cold one does. Cold boots are unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 20 ++++++++++++++++++++ + 1 file changed, 20 insertions(+) + +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 6e3ab71c5..e93d1b02c 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -95,6 +95,20 @@ static void g5p_sleep_sample(struct g5p_diag *g, u32 *state, u32 *mode, u32 *r8) + *r8 = readl(g->window[1] + 8); + } + ++/* ++ * A kexec boot (DS108 mode 3) keeps every S8000 PMGR power state on for the ++ * whole boot (pmgr-pwrstate keep_on, "retain the tethered warm boot state"): ++ * genpd power-off of gfx returns -EBUSY. Sleep would then stop the firmware ++ * without the platform off, and resume would boot it over a GPU that was ++ * never power-cycled: a dirty start, which DS108 admits only through its one ++ * bounded reset at probe. Refuse before anything is shut, until the warm boot ++ * can power the GPU off like a cold one. RAM only. ++ */ ++static bool g5p_sleep_gfx_power_held(void) ++{ ++ return g5p_ds_mode() >= 3; ++} ++ + /* R1 only, as Apple's isQuiescent/isStableOff. Never infer bus retirement + * from a completed userspace fence or GPC=0 alone. + */ +@@ -217,6 +231,10 @@ static int g5p_system_sleep_prepare(struct device *dev) + return -EBUSY; + } + d = g->event_cookie; ++ if (g5p_sleep_gfx_power_held()) { ++ dev_info(dev, "GPU-PM sleep refused: stage=entry why=warm-gfx-power-held\n"); ++ return -EBUSY; ++ } + if (READ_ONCE(g->sleep_reserved)) { + if (!g->sleep_failed) + return 0; +@@ -404,6 +422,8 @@ static int g5p_system_sleep_check(struct device *dev) + if (!READ_ONCE(b->alive) || READ_ONCE(b->stopped) || !b->released || + !b->irq_ready) + return g5p_sleep_why(d, -ENODEV, "not-booted"); ++ if (g5p_sleep_gfx_power_held()) ++ return g5p_sleep_why(d, -EBUSY, "warm-gfx-power-held"); + never_park = g5p_sleep_never_park(b); + if (!mutex_trylock(&g->lock)) + return never_park ? g5p_sleep_why(d, -EBUSY, "never-park/locked") : diff --git a/kernel/patches/drivers/drm/0037-drm-apple-g5p-do-not-refuse-system-sleep-on-the-rend.patch b/kernel/patches/drivers/drm/0037-drm-apple-g5p-do-not-refuse-system-sleep-on-the-rend.patch new file mode 100644 index 00000000..729a8543 --- /dev/null +++ b/kernel/patches/drivers/drm/0037-drm-apple-g5p-do-not-refuse-system-sleep-on-the-rend.patch @@ -0,0 +1,49 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:12:42 +0100 +Subject: [PATCH] drm/apple-g5p: do not refuse system sleep on the render claim + +The sleep busy predicate (since "implement bounded retained system +sleep") reads d->accepted as accepted work in flight. It is not: it is +the render claim, set 0 -> 1 by the first accepted job in +g5p_render_device_claim() and never cleared ("one render attachment per +boot: accepted stays held"). Once any client has rendered, which on a +desktop is at boot, every sleep attempt drained for 1 s and refused +"accepted", although no client was in the driver and the firmware was +parked. The deferred-close term hid it until it was dropped. + +Accepted jobs not yet signalled are r8_queued (incremented at accept, +decremented at signal), which the predicate already checks. Drop the +claim from the busy states. The R1 quiescent certificate, the rings, the +TA/3D stamps and every other busy state are unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index e93d1b02c..ad782758a 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -18,6 +18,11 @@ + * after an in-boot recovery. With one compositor running, a context that + * closed earlier stays "previous" for good, so refusing on it refused every + * sleep. ++ * ++ * Neither is the render claim (d->accepted, g5p_render_device_claim()): it ++ * latches on the first accepted job and holds for the rest of the boot (one ++ * render attachment per boot), so it says nothing about work in flight. ++ * Accepted jobs not yet signalled are r8_queued. + */ + static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) + { +@@ -42,8 +47,6 @@ static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) + return "stuck"; + if (rd->recovering) + return "recovering"; +- if (atomic_read(&d->accepted)) +- return "accepted"; + if (atomic_read(&d->r8_queued)) + return "r8-queued"; + if (READ_ONCE(rd->executing)) diff --git a/kernel/patches/drivers/drm/0038-drm-apple-g5p-name-every-held-sleep-busy-state.patch b/kernel/patches/drivers/drm/0038-drm-apple-g5p-name-every-held-sleep-busy-state.patch new file mode 100644 index 00000000..7d355ec0 --- /dev/null +++ b/kernel/patches/drivers/drm/0038-drm-apple-g5p-name-every-held-sleep-busy-state.patch @@ -0,0 +1,307 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:15:23 +0100 +Subject: [PATCH] drm/apple-g5p: name every held sleep busy state + +The sleep busy predicate returned its first matching reason only, so a +refusal hid every term behind it: the deferred close hid the render +claim, which in turn hid whatever follows it. Each hidden term costs +another device test to find. + +Compute the reasons as a mask and print every held one, '+'-joined so +the readiness reason stays one token, in the prepare drain-bound and +state refusal lines, the suspend pre-detach refusal and the readiness +query. The stamp and per-context reads still run only with no work in +device test and no failure or recovery pending, as before. The decision +is unchanged: prepare still drains every non-static reason for at most 1 +s and refuses on any reason left. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/a9-gpu-drm.c | 6 +- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 154 ++++++++++++++------- + 2 files changed, 108 insertions(+), 52 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +index 5576a41ec..a24f5c5d0 100644 +--- a/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c ++++ b/drivers/gpu/drm/apple-g5p/a9-gpu-drm.c +@@ -120,8 +120,10 @@ struct g5p_drm { + struct g5p_diag *core; + /* Reserves the system-sleep certificate before sync, through complete. */ + struct pm_suspend_admission sleep_admission; +- /* Why the last readiness check refused; written by the check only. */ +- char sleep_why[64]; ++ /* Why the last readiness check refused; written by the check only. ++ * Room for a prefix and every busy name (system-sleep.inc). ++ */ ++ char sleep_why[208]; + /* Protects cached IRQ event/status fields; never held for hardware IO. */ + spinlock_t status_lock; + struct g5p_jt jt_ring[G5P_JT_RING]; +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index ad782758a..e06b48430 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -4,10 +4,10 @@ + * No terminal-retire latch is used for a reversible sleep attempt. + */ + /* +- * Why the host side is not idle: one token (no white space or colon), or NULL +- * when idle. *drains is set when the client drain in prepare can clear it +- * (work in flight); otherwise nothing at idle clears it and only a refusal is +- * honest. ++ * Why the host side is not idle, every reason at once (G5P_SB_* bits): a ++ * refusal names all of them, not only the first, so one device test shows every ++ * term that holds. The first three are static: nothing at idle clears them ++ * and only a refusal is honest. The rest drain with work in flight. + * + * A deferred close (render-user-pmset.h) is not a reason: it keeps a closed + * context's slots bound while the PM may still write its free list back on a +@@ -24,40 +24,68 @@ + * render attachment per boot), so it says nothing about work in flight. + * Accepted jobs not yet signalled are r8_queued. + */ +-static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) ++enum { ++ G5P_SB_COMPUTE, /* no CDM stamp certificate yet (lane GS) */ ++ G5P_SB_TERMINAL, ++ G5P_SB_DC_PENDING, /* a bind's DC3 pair the firmware never consumed */ ++ G5P_SB_STUCK, /* a lost-job drain or recovery may finish */ ++ G5P_SB_RECOVERING, /* inside the client drain bound */ ++ G5P_SB_R8_QUEUED, ++ G5P_SB_EXECUTING, ++ G5P_SB_AHEAD, ++ G5P_SB_QUEUED, ++ G5P_SB_STAMP_TA, ++ G5P_SB_STAMP_3D, ++ G5P_SB_UNCONSUMED_TA, ++ G5P_SB_UNCONSUMED_3D, ++ G5P_SB_UNRETIRED_TA, ++ G5P_SB_UNRETIRED_3D, ++ G5P_SB_NR ++}; ++ ++#define G5P_SB_STATIC \ ++ (BIT(G5P_SB_COMPUTE) | BIT(G5P_SB_TERMINAL) | BIT(G5P_SB_DC_PENDING)) ++ ++static const char *const g5p_sleep_busy_names[G5P_SB_NR] = { ++ "compute-epoch", "terminal", "dc-pending", "stuck", ++ "recovering", "r8-queued", "executing", "ahead", ++ "queued", "stamp-ta", "stamp-3d", "unconsumed-ta", ++ "unconsumed-3d", "unretired-ta", "unretired-3d", ++}; ++ ++static u32 g5p_sleep_busy_mask(struct g5p_drm *d) + { + struct g5p_rdev *rd = d->rdev; + struct g5p_render *r; + unsigned long flags; +- const char *why = NULL; +- bool idle; ++ u32 m = 0; + +- *drains = false; +- /* No CDM stamp certificate yet (lane GS): compute epochs refuse. */ + if (rd->cu_sequence) +- return "compute-epoch"; ++ m |= BIT(G5P_SB_COMPUTE); + if (g5p_ru_term_failed(&g5p_rv3_term)) +- return "terminal"; +- /* A bind's DC3 pair the firmware never consumed; only a later DC op clears it. */ ++ m |= BIT(G5P_SB_TERMINAL); + if (rd->dc_pending) +- return "dc-pending"; +- *drains = true; +- /* A lost-job drain or recovery may finish inside the client drain bound. */ ++ m |= BIT(G5P_SB_DC_PENDING); + if (rd->stuck) +- return "stuck"; ++ m |= BIT(G5P_SB_STUCK); + if (rd->recovering) +- return "recovering"; ++ m |= BIT(G5P_SB_RECOVERING); + if (atomic_read(&d->r8_queued)) +- return "r8-queued"; ++ m |= BIT(G5P_SB_R8_QUEUED); + if (READ_ONCE(rd->executing)) +- return "executing"; ++ m |= BIT(G5P_SB_EXECUTING); + if (rd->ahead) +- return "ahead"; ++ m |= BIT(G5P_SB_AHEAD); + spin_lock_irqsave(&rd->sched_lock, flags); +- idle = list_empty(&rd->sched.run); ++ if (!list_empty(&rd->sched.run)) ++ m |= BIT(G5P_SB_QUEUED); + spin_unlock_irqrestore(&rd->sched_lock, flags); +- if (!idle) +- return "queued"; ++ /* The stamp and context reads run, as they always have, only with no ++ * work in flight and no failure or recovery that may replace the boot. ++ * They are the terms an earlier first-match reason used to hide. ++ */ ++ if (m & ~(BIT(G5P_SB_COMPUTE) | BIT(G5P_SB_DC_PENDING))) ++ return m; + /* Re-read the latest TA/3D stamps, including the depth-2 record slot. + * A CPU retirement flag alone is not Apple's all-stamps certificate. + */ +@@ -67,18 +95,41 @@ static const char *g5p_sleep_busy(struct g5p_drm *d, bool *drains) + g5p_ru_meta(e), e, rd->sequence, + g5p_rv3_js(rd, rd->sequence), + rd->sequence)) +- return e ? "stamp-3d" : "stamp-ta"; ++ m |= BIT(G5P_SB_STAMP_TA + e); + mutex_lock(&rd->lock); + list_for_each_entry(r, &rd->contexts, rdev_link) + for (u32 i = 0; i < 2; i++) +- if (r->submitted[i] && !why) +- why = !r->consumed[i] ? +- (i ? "unconsumed-3d" : "unconsumed-ta") : +- !r->retired[i] ? +- (i ? "unretired-3d" : "unretired-ta") : +- NULL; ++ if (r->submitted[i]) ++ m |= !r->consumed[i] ? ++ BIT(G5P_SB_UNCONSUMED_TA + i) : ++ !r->retired[i] ? ++ BIT(G5P_SB_UNRETIRED_TA + i) : ++ 0; + mutex_unlock(&rd->lock); +- return why; ++ return m; ++} ++ ++/* Every held reason, '+'-joined: still one token (no white space or colon). */ ++static const char *g5p_sleep_busy_list(u32 m, char *buf, size_t len) ++{ ++ unsigned long bits = m; ++ size_t n = 0; ++ u32 i; ++ ++ buf[0] = 0; ++ for_each_set_bit(i, &bits, G5P_SB_NR) ++ n += scnprintf(buf + n, len - n, "%s%s", n ? "+" : "", ++ g5p_sleep_busy_names[i]); ++ return m ? buf : NULL; ++} ++ ++/* Room for every name, '+'-joined. */ ++#define G5P_SB_LIST 192 ++ ++/* Prepare's client drain stops early once a static reason holds. */ ++static bool g5p_sleep_busy_drains(u32 m) ++{ ++ return !(m & G5P_SB_STATIC); + } + + /* +@@ -221,10 +272,9 @@ static int g5p_system_sleep_prepare(struct device *dev) + struct g5p_boot *b; + struct g5p_kq_ctx c = { .g = g }; + const char *stage = "entry", *busy; +- char why[64] = ""; ++ char why[G5P_SB_LIST] = ""; + u64 t0 = ktime_get_ns(), deadline, drain_ns; +- u32 cons, prod, state, mode, r8; +- bool drains; ++ u32 cons, prod, state, mode, r8, held; + int ret = -EBUSY, users; + + /* Every refusal below names its stage and reason: no silent -EBUSY. */ +@@ -255,17 +305,19 @@ static int g5p_system_sleep_prepare(struct device *dev) + deadline = t0 + G5P_SLEEP_DRAIN_MS * NSEC_PER_MSEC; + for (;;) { + users = atomic_read(&d->client_gate) & INT_MAX; +- busy = g5p_sleep_busy(d, &drains); ++ held = g5p_sleep_busy_mask(d); + /* Stop waiting on what no drain clears; stage 2 names it. */ +- if ((!users && !busy) || (busy && !drains) || ++ if ((!users && !held) || (held && !g5p_sleep_busy_drains(held)) || + ktime_get_ns() >= deadline) + break; + usleep_range(500, 1000); + } + drain_ns = ktime_get_ns() - t0; +- if (users || busy) ++ if (users || held) + dev_info(dev, "GPU-PM sleep drain bound: client_users=%d busy=%s drain_ms=%llu\n", +- users, busy ?: "none", drain_ns / NSEC_PER_MSEC); ++ users, g5p_sleep_busy_list(held, why, sizeof(why)) ?: "none", ++ drain_ns / NSEC_PER_MSEC); ++ why[0] = 0; + /* Stage 2: shut the host gate and certify, unchanged. */ + stage = "host"; + if (atomic_fetch_or(BIT(31), &g->sleep_gate) < 0) { +@@ -302,9 +354,10 @@ static int g5p_system_sleep_prepare(struct device *dev) + !b->irq_ready ? "irq-not-ready" : + !b->r73b ? "no-r73b" : + READ_ONCE(b->r73b->early_power_abort) ? "power-abort" : +- g5p_sleep_busy(d, &drains); ++ g5p_sleep_busy_list(g5p_sleep_busy_mask(d), why, sizeof(why)); + if (busy) { +- strscpy(why, busy); ++ if (busy != why) ++ strscpy(why, busy); + goto unlock; + } + /* No submitters, queued work or IRQ-side ALIVE action can publish now. */ +@@ -389,7 +442,7 @@ static __printf(3, 4) int g5p_sleep_why(struct g5p_drm *d, int ret, + * draining firmware events. Accepted or in-flight work is not a refusal, + * since prepare drains it, except under the never-park envelope, where the + * drained firmware still cannot certify. A busy state no drain clears +- * (g5p_sleep_busy() without *drains) refuses here exactly as prepare would, ++ * (G5P_SB_STATIC) refuses here exactly as prepare would, + * so "ready" is never answered for it. Register samples are taken only + * when the GPU is already idle and the core lock is free, with the same + * reads as g5p_sleep_certificate(); a sampled GPC 0 falls through to it. +@@ -399,9 +452,9 @@ static int g5p_system_sleep_check(struct device *dev) + struct g5p_diag *g = dev_get_drvdata(dev); + struct g5p_drm *d; + struct g5p_boot *b; +- u32 cons, prod, state, mode = 0, r8 = 0; +- const char *busy; +- bool never_park, drains; ++ u32 cons, prod, state, mode = 0, r8 = 0, held; ++ char list[G5P_SB_LIST]; ++ bool never_park; + int ret = 0; + + if (!g || !g->event_cookie) +@@ -435,13 +488,14 @@ static int g5p_system_sleep_check(struct device *dev) + ret = g5p_sleep_why(d, -ENODEV, "not-powered"); + goto unlock; + } +- busy = g5p_sleep_busy(d, &drains); +- if (busy) { ++ held = g5p_sleep_busy_mask(d); ++ if (held) { + /* Prepare drains work in flight; it cannot drain the rest. */ +- if (never_park || !drains) ++ if (never_park || !g5p_sleep_busy_drains(held)) + ret = g5p_sleep_why(d, -EBUSY, "%s%s", + never_park ? "never-park/" : "busy/", +- busy); ++ g5p_sleep_busy_list(held, list, ++ sizeof(list))); + goto unlock; + } + for (u32 i = 0; i < G5P_TD_NRING; i++) { +@@ -498,7 +552,7 @@ static int g5p_system_suspend(struct device *dev) + struct g5p_render **drop, *r; + u32 slots[G5P_RU_USER_SLOTS], held, count = 0, n = 0, state, mode, r8; + const char *busy, *step = "entry"; +- bool drains; ++ char list[G5P_SB_LIST]; + u64 t0; + int ret; + +@@ -513,7 +567,7 @@ static int g5p_system_suspend(struct device *dev) + READ_ONCE(b->event_failed) ? "event-failed" : + READ_ONCE(b->stopped) ? "stopped" : + READ_ONCE(b->r73b->early_power_abort) ? "power-abort" : +- g5p_sleep_busy(d, &drains); ++ g5p_sleep_busy_list(g5p_sleep_busy_mask(d), list, sizeof(list)); + if (!busy && !g5p_sleep_rings(g)) + busy = "rings"; + if (!busy && !g5p_sleep_certificate(g, false)) { diff --git a/kernel/patches/drivers/drm/0039-drm-apple-g5p-answer-the-readiness-query-with-prepar.patch b/kernel/patches/drivers/drm/0039-drm-apple-g5p-answer-the-readiness-query-with-prepar.patch new file mode 100644 index 00000000..686cfe1d --- /dev/null +++ b/kernel/patches/drivers/drm/0039-drm-apple-g5p-answer-the-readiness-query-with-prepar.patch @@ -0,0 +1,50 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:15:36 +0100 +Subject: [PATCH] drm/apple-g5p: answer the readiness query with prepare's + criterion + +The readiness query answered "ready" for any drainable busy state, since +prepare drains work in flight. A drainable state with no job in device +test is not drained by anything: on J71t with the GPU parked and no +client in the driver, /sys/power/suspend_ready read "ready" while every +attempt drained for 1 s and refused. + +A drainable state now counts as in flight only while an accepted job is +not yet signalled (r8_queued); otherwise the query refuses it as +"busy/", as prepare's state check will after its drain. Prepare +itself is unchanged. + +Assisted-by: LLM +--- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 10 +++++++--- + 1 file changed, 7 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index e06b48430..40cf32e4f 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -442,8 +442,11 @@ static __printf(3, 4) int g5p_sleep_why(struct g5p_drm *d, int ret, + * draining firmware events. Accepted or in-flight work is not a refusal, + * since prepare drains it, except under the never-park envelope, where the + * drained firmware still cannot certify. A busy state no drain clears +- * (G5P_SB_STATIC) refuses here exactly as prepare would, +- * so "ready" is never answered for it. Register samples are taken only ++ * (G5P_SB_STATIC) refuses here exactly as prepare would, and so does a ++ * drainable one with no job in flight (r8_queued: accepted, not yet ++ * signalled) to retire it: prepare's drain would wait 1 s on it and refuse. ++ * So "ready" is never answered for either. A close item still queued with ++ * no job may answer busy for a moment; prepare drains it. Register samples are taken only + * when the GPU is already idle and the core lock is free, with the same + * reads as g5p_sleep_certificate(); a sampled GPC 0 falls through to it. + */ +@@ -491,7 +494,8 @@ static int g5p_system_sleep_check(struct device *dev) + held = g5p_sleep_busy_mask(d); + if (held) { + /* Prepare drains work in flight; it cannot drain the rest. */ +- if (never_park || !g5p_sleep_busy_drains(held)) ++ if (never_park || !g5p_sleep_busy_drains(held) || ++ !(held & BIT(G5P_SB_R8_QUEUED))) + ret = g5p_sleep_why(d, -EBUSY, "%s%s", + never_park ? "never-park/" : "busy/", + g5p_sleep_busy_list(held, list, diff --git a/kernel/patches/drivers/drm/0040-drm-apple-g5p-take-genpd-s-witnessed-gfx-power-off-a.patch b/kernel/patches/drivers/drm/0040-drm-apple-g5p-take-genpd-s-witnessed-gfx-power-off-a.patch new file mode 100644 index 00000000..54117d5d --- /dev/null +++ b/kernel/patches/drivers/drm/0040-drm-apple-g5p-take-genpd-s-witnessed-gfx-power-off-a.patch @@ -0,0 +1,374 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:11:42 +0100 +Subject: [PATCH] drm/apple-g5p: take genpd's witnessed gfx power-off as the + clean start + +The incoming dirty-start machine trusted a kexec boot's GPU only after +its own bounded gate cycle, or when apple_pmgr.kexec_gpu_cycle=1 was on +the command line and the sticky was_pwrgated bit was set. That bit can +predate the boot, and no code in this tree performs the cycle the token +names, so the token turned a stale history bit into trust. + +Now that a warm boot gates idle domains like a cold one, genpd powers +gfx off once the GPU has probed, long before the first wake, exactly as +on a cold boot. Record genpd's own transitions with a power notifier +(OFF and ON are sent only after a successful transition) and treat a gfx +power-off held for at least the reset's hold in this boot as the clean +start a cold boot has. Without such a witness the bounded cycle runs as +before. The command-line token is no longer read. + +System sleep gets the same witness: genpd gates gfx in suspend_noirq and +ignores a refused power-off, so a successful runtime put proves nothing. +If no power-off of the reset's hold was witnessed since the sleep's put, +resume spends one bounded gate cycle before the firmware boot instead of +starting it over a GPU that kept power. + +genpd's PRE_OFF notification also closes the MMIO licence, as before +every other gfx gate. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/core.inc | 62 ++++++++++++++++++++++ + drivers/gpu/apple-g5p/dirtystart108.h | 39 +++++++------- + drivers/gpu/apple-g5p/dirtystart108.inc | 32 ++--------- + drivers/gpu/drm/apple-g5p/system-sleep.inc | 18 +++++++ + 4 files changed, 102 insertions(+), 49 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +index 9177065d4..20eb5594f 100644 +--- a/drivers/gpu/apple-g5p/core.inc ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -78,10 +79,15 @@ struct g5p_diag { + void __iomem *window[2]; + void __iomem *rgx; + struct regmap *pmgr; ++ struct device_node *gfx_np; /* the gfx PMGR power-state node, or NULL */ ++ /* genpd notifier on gfx: when genpd last powered it off and on (0: never). */ ++ struct notifier_block gfx_nb; ++ u64 gfx_off_ns, gfx_on_ns; /* CLOCK_BOOTTIME */ + /* Serializes commands and all hardware access. */ + struct mutex lock; + atomic_t sleep_gate; /* closed bit plus active host operations */ + bool sleep_reserved, sleep_off, sleep_boot, sleep_failed, sleep_irq_off; ++ u64 sleep_off_ns; /* when this sleep released gfx power (resume's witness window) */ + bool powered; + bool loaded; + +@@ -162,6 +168,11 @@ static u32 g5p_ds_gen(void); + /* The GPU ADT clock/power gate124 resolves to ps-regs[4]+index8*8. + * Read via its existing syscon, sharing the provider mapping without writes. + */ ++static void g5p_gfx_np_put(void *np) ++{ ++ of_node_put(np); ++} ++ + static void g5p_power_observer_init(struct g5p_diag *g) + { + struct device_node *domain, *parent; +@@ -178,6 +189,9 @@ static void g5p_power_observer_init(struct g5p_diag *g) + g->pmgr = syscon_node_to_regmap(parent); + if (IS_ERR(g->pmgr)) + g->pmgr = NULL; ++ else if (!devm_add_action_or_reset(g->dev, g5p_gfx_np_put, ++ of_node_get(domain))) ++ g->gfx_np = domain; + } + of_node_put(parent); + of_node_put(domain); +@@ -217,6 +231,47 @@ static void g5p_power_snapshot(struct g5p_diag *g, const char *stage) + stage, (raw >> 9) & 1, (raw >> 8) & 1); + } + ++/* The bounded reset's hold (DS108 RESET, the sleep fallback): also the shortest genpd off ++ * accepted in its place. ++ */ ++#define G5P_GFX_RESET_MS 10U ++ ++/* ++ * genpd's own record of gfx power, cold and warm alike. PRE_OFF closes the MMIO licence ++ * as before every PMGR gfx gate; OFF and ON are sent only after the transition succeeded. ++ * Atomic context (IRQ-safe genpd lock): RAM only. Boot time, not monotonic: s2idle stops the ++ * monotonic clock, and the off/on of a system sleep land on either side of it. ++ */ ++static int g5p_gfx_notify(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct g5p_diag *g = container_of(nb, struct g5p_diag, gfx_nb); ++ ++ if (action == GENPD_NOTIFY_PRE_OFF) ++ g5p_mmio_gate(g); ++ else if (action == GENPD_NOTIFY_OFF) ++ WRITE_ONCE(g->gfx_off_ns, ktime_get_boottime_ns()); ++ else if (action == GENPD_NOTIFY_ON) ++ WRITE_ONCE(g->gfx_on_ns, ktime_get_boottime_ns()); ++ return NOTIFY_OK; ++} ++ ++static void g5p_gfx_notify_remove(void *dev) ++{ ++ dev_pm_genpd_remove_notifier(dev); ++} ++ ++/* genpd powered gfx off after @since and held it off for at least the reset's hold. */ ++static bool g5p_gfx_gated_since(struct g5p_diag *g, u64 since) ++{ ++ u64 off = READ_ONCE(g->gfx_off_ns), on = READ_ONCE(g->gfx_on_ns); ++ ++ if (!off || off < since) ++ return false; ++ if (on < off) ++ on = ktime_get_boottime_ns(); /* still off */ ++ return on - off >= (u64)G5P_GFX_RESET_MS * NSEC_PER_MSEC; ++} ++ + static int g5p_power(struct g5p_diag *g) + { + int ret; +@@ -583,6 +638,13 @@ static int g5p_probe(struct platform_device *pdev) + if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40))) + return -ENODEV; + pm_runtime_enable(dev); ++ /* Before the provider's sync_state can gate gfx: no witness is lost. devm: the ++ * notifier never outlives g, on a failed probe or after remove. ++ */ ++ g->gfx_nb.notifier_call = g5p_gfx_notify; ++ if (dev_pm_genpd_add_notifier(dev, &g->gfx_nb) || ++ devm_add_action_or_reset(dev, g5p_gfx_notify_remove, dev)) ++ dev_info(dev, "gfx genpd notifier unavailable: no power-off witness\n"); + g5p_td_attach(g); + g5p_ds_hook_probe(g); + ; +diff --git a/drivers/gpu/apple-g5p/dirtystart108.h b/drivers/gpu/apple-g5p/dirtystart108.h +index fcf7432fa..99b7aabd8 100644 +--- a/drivers/gpu/apple-g5p/dirtystart108.h ++++ b/drivers/gpu/apple-g5p/dirtystart108.h +@@ -18,11 +18,13 @@ + * one event per driver step with the step's result and RAM-only inputs, and + * asks the machine at two gates (before release, at the DRM claim). It + * assumes nothing pristine: the origin of the power state is classified from +- * the kexec flag, the requested incoming PMGR cycle and the PMGR word, and ++ * the kexec flag, whether this boot saw gfx gated off and the PMGR word, and + * every step's own result is evidence. + * +- * PROBE origin inputs: booted from kexec, kexec_gpu_cycle requested +- * POWER the PMGR gfx word, read once after the PM reference (PMGR read) ++ * PROBE origin input: booted from kexec ++ * POWER the PMGR gfx word, read once after the PM reference (PMGR read), ++ * and whether genpd held gfx powered off for at least the reset's ++ * hold in this boot (RAM: the glue's genpd notifier) + * RESET the bounded incoming power cycle (g5p_r33_gfx_off(), one attempt + * per boot, only in mode 3, only before the first wrapper wake) + * WAKE the Fender wake result +@@ -42,7 +44,7 @@ + * proven in the current reset generation. Equal to combined107 on every + * bootstrap that succeeds. + * 2 strict: also refuse release and claim for an origin that is not cold +- * and not a witnessed cycle (warm-unproven, warm-uncycled, unknown), ++ * and not a witnessed power-off (warm-uncycled, unknown), + * unless a reset succeeded in this generation + * 3 strict + reset: as 2, and the glue may spend the one RESET attempt + * on such an origin before the wrapper wake (destructive: PMGR write) +@@ -77,9 +79,10 @@ static const char *const g5p_ds_ev_name[G5P_DSE_N] = { + enum g5p_ds_origin { + G5P_DSO_UNKNOWN, /* not probed, or the PMGR word could not be read */ + G5P_DSO_COLD, /* not booted from kexec */ +- G5P_DSO_WARM_CYCLED, /* kexec, incoming cycle requested and witnessed (was_pwrgated) */ +- G5P_DSO_WARM_UNPROVEN, /* kexec, cycle requested, no witness in the PMGR word */ +- G5P_DSO_WARM_UNCYCLED, /* kexec, no incoming cycle requested */ ++ G5P_DSO_WARM_CYCLED, /* kexec, gfx witnessed gated off in this boot */ ++ /* Never produced: the sticky was_pwrgated bit can predate this boot. */ ++ G5P_DSO_WARM_UNPROVEN, ++ G5P_DSO_WARM_UNCYCLED, /* kexec, no power-off witnessed in this boot */ + G5P_DSO_N, + }; + +@@ -122,8 +125,6 @@ static const char *const g5p_ds_gate_name[G5P_DSG_N] = { + + #define G5P_DS_MODE_MAX 3 + #define G5P_DS_GPC_ON 0x01u +-#define G5P_DS_PMGR_PWRGATED \ +- BIT(8) /* sticky was_pwrgated history bit of 0x80340 */ + + #define G5P_DSF_RINGS_BUSY \ + BIT(0) /* a ring not empty at map (data, not a failure) */ +@@ -140,7 +141,7 @@ static const char *const g5p_ds_gate_name[G5P_DSG_N] = { + /* One flat input record; each event reads only its own fields (RAM or step results). */ + struct g5p_ds_in { + bool kexec; /* PROBE: chosen linux,booted-from-kexec */ +- bool cycle_requested; /* PROBE: apple_pmgr.kexec_gpu_cycle=1 on the cmdline */ ++ bool gated; /* POWER: gfx held off by genpd in this boot, for >= the reset hold */ + bool pmgr_ok; /* POWER: the PMGR read succeeded */ + u32 pmgr_raw; /* POWER: the gfx word */ + int ret; /* RESET/WAKE/MAPS/LOAD/RELEASE/ALIVE: the step's return */ +@@ -160,7 +161,7 @@ struct g5p_ds { + u32 seen, ok; /* event bitmasks, current generation */ + u32 flags; + enum g5p_ds_origin origin; +- bool kexec, cycle; /* PROBE inputs, kept for POWER */ ++ bool kexec; /* PROBE input, kept for POWER */ + u32 pmgr_raw; + u32 resets; /* RESET attempts this boot (at most one runs) */ + u32 reset_ok_gen; /* gen in which a RESET succeeded, 0 never */ +@@ -184,17 +185,15 @@ static inline void g5p_ds_fail(struct g5p_ds *d, enum g5p_ds_why why) + d->failed = true; + } + +-static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool cycle, +- bool pmgr_ok, u32 raw) ++static inline enum g5p_ds_origin g5p_ds_origin_of(bool kexec, bool gated, ++ bool pmgr_ok) + { + if (!pmgr_ok) + return G5P_DSO_UNKNOWN; + if (!kexec) + return G5P_DSO_COLD; +- if (!cycle) +- return G5P_DSO_WARM_UNCYCLED; +- return (raw & G5P_DS_PMGR_PWRGATED) ? G5P_DSO_WARM_CYCLED : +- G5P_DSO_WARM_UNPROVEN; ++ /* Powered off for the reset's hold in this boot: the state a cold boot starts from. */ ++ return gated ? G5P_DSO_WARM_CYCLED : G5P_DSO_WARM_UNCYCLED; + } + + /* The origin is trusted (cold, witnessed cycle) or a reset succeeded in this generation. */ +@@ -269,9 +268,7 @@ static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, + d->gen = 1; + d->origin = G5P_DSO_UNKNOWN; + a = in->kexec; +- b = in->cycle_requested; + d->kexec = in->kexec; +- d->cycle = in->cycle_requested; + d->seen |= bit; + d->ok |= bit; + break; +@@ -280,12 +277,12 @@ static inline int g5p_ds_event(struct g5p_ds *d, enum g5p_ds_ev e, + g5p_ds_fail(d, G5P_DSW_PROTOCOL); + break; + } +- d->origin = g5p_ds_origin_of(d->kexec, d->cycle, in->pmgr_ok, +- in->pmgr_raw); ++ d->origin = g5p_ds_origin_of(d->kexec, in->gated, in->pmgr_ok); + if (!in->pmgr_ok) + d->flags |= G5P_DSF_PMGR_UNREAD; + d->pmgr_raw = in->pmgr_ok ? in->pmgr_raw : 0; + a = in->pmgr_ok; ++ b = in->gated; + d->seen |= bit; + d->ok |= bit; + break; +diff --git a/drivers/gpu/apple-g5p/dirtystart108.inc b/drivers/gpu/apple-g5p/dirtystart108.inc +index be023e7e7..5a1e226d8 100644 +--- a/drivers/gpu/apple-g5p/dirtystart108.inc ++++ b/drivers/gpu/apple-g5p/dirtystart108.inc +@@ -10,7 +10,8 @@ + * Register access added by this file: one PMGR read of the gfx word after the + * PM reference (POWER, the same read g5p_power_snapshot() makes), and in mode 3 + * only one bounded PMGR cycle through g5p_r33_gfx_off() before the first +- * wrapper wake. Every other input is RAM or a step's own return value. ++ * wrapper wake, unless genpd already held gfx off for as long in this boot. ++ * Every other input is RAM or a step's own return value. + */ + #include + #include "dirtystart108.h" +@@ -54,31 +55,6 @@ static u32 g5p_ds_gen(void) + return gen; + } + +-/* The last apple_pmgr.kexec_gpu_cycle= token of the chosen bootargs (the one the kernel obeys). */ +-static bool g5p_ds_cycle_requested(struct device_node *chosen) +-{ +- static const char key[] = "apple_pmgr.kexec_gpu_cycle="; +- const char *args, *p, *last = NULL; +- char tok[8]; +- bool v = false; +- size_t n; +- +- if (!chosen || of_property_read_string(chosen, "bootargs", &args)) +- return false; +- for (p = strstr(args, key); p; p = strstr(p + 1, key)) +- if (p == args || p[-1] == ' ') +- last = p; +- if (!last) +- return false; +- last += sizeof(key) - 1; +- n = strcspn(last, " "); +- if (!n || n >= sizeof(tok)) +- return false; +- memcpy(tok, last, n); +- tok[n] = 0; +- return !kstrtobool(tok, &v) && v; +-} +- + static void g5p_ds_hook_probe(struct g5p_diag *g) + { + struct device_node *chosen = of_find_node_by_path("/chosen"); +@@ -86,7 +62,6 @@ static void g5p_ds_hook_probe(struct g5p_diag *g) + + in.kexec = chosen && + of_property_read_bool(chosen, "linux,booted-from-kexec"); +- in.cycle_requested = g5p_ds_cycle_requested(chosen); + of_node_put(chosen); + /* Retained warm state needs strict origin checks and the bounded reset. */ + WRITE_ONCE(g5p_ds_boot_mode, in.kexec ? 3 : 1); +@@ -101,6 +76,7 @@ static void g5p_ds_hook_power(struct g5p_diag *g) + struct g5p_ds_in in = {}; + u32 raw = 0; + ++ in.gated = g5p_gfx_gated_since(g, 0); + in.pmgr_ok = g->pmgr && !regmap_read(g->pmgr, 0x80340, &raw); + in.pmgr_raw = raw; + g5p_ds_feed(g, G5P_DSE_POWER, &in); +@@ -112,7 +88,7 @@ static void g5p_ds_hook_power(struct g5p_diag *g) + */ + static int g5p_ds_hook_reset(struct g5p_diag *g) + { +- u32 ms = clamp_t(u32, (10), 1, G5P_R33_GFX_OFF_MAX_MS); ++ u32 ms = clamp_t(u32, G5P_GFX_RESET_MS, 1, G5P_R33_GFX_OFF_MAX_MS); + struct g5p_ds_in in = {}; + unsigned long flags; + bool want; +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 40cf32e4f..64bb87905 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -648,6 +648,7 @@ static int g5p_system_suspend(struct device *dev) + g5p_mmio_gate(g); + g->fender_ready = false; + step = "platform-off"; ++ g->sleep_off_ns = ktime_get_boottime_ns(); /* the notifier's clock */ + ret = pm_runtime_put_sync_suspend(dev); + g->powered = false; /* put consumed our reference, including on error */ + if (ret < 0) +@@ -691,6 +692,23 @@ static int g5p_system_resume(struct device *dev) + goto failed; + g->powered = true; + g->sleep_boot = true; ++ /* genpd gates gfx in suspend_noirq and ignores a refused power-off, so the PM calls' ++ * return proves nothing. Its notifier does: without a witnessed off of the reset's ++ * hold since this sleep's put, spend one bounded gate cycle before the boot instead, ++ * so the firmware never starts over a GPU that kept power. MMIO is still gated. ++ */ ++ step = "platform-cycle"; ++ if (g5p_gfx_gated_since(g, g->sleep_off_ns)) { ++ dev_info(dev, "GPU-PM platform off witnessed ms=%llu\n", ++ (READ_ONCE(g->gfx_on_ns) - READ_ONCE(g->gfx_off_ns)) / ++ NSEC_PER_MSEC); ++ } else { ++ dev_info(dev, "GPU-PM platform off not witnessed: one bounded gate cycle\n"); ++ g5p_mmio_gate(g); ++ ret = g5p_r33_gfx_off(dev, g->pmgr, G5P_GFX_RESET_MS); ++ if (ret) ++ goto failed; ++ } + /* A witnessed platform off/on invalidates DS108's old ALIVE generation. + * Retained maps/image are validated, never allocated or reinitialised. + */ diff --git a/kernel/patches/drivers/drm/0041-drm-apple-g5p-refuse-system-sleep-only-while-pmgr-st.patch b/kernel/patches/drivers/drm/0041-drm-apple-g5p-refuse-system-sleep-only-while-pmgr-st.patch new file mode 100644 index 00000000..f624bdf1 --- /dev/null +++ b/kernel/patches/drivers/drm/0041-drm-apple-g5p-refuse-system-sleep-only-while-pmgr-st.patch @@ -0,0 +1,86 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:12:17 +0100 +Subject: [PATCH] drm/apple-g5p: refuse system sleep only while pmgr still + holds gfx + +The refusal added for kexec boots keyed on the dirty-start mode: every +warm boot refused GPU sleep. pmgr-pwrstate now ends a warm boot's hold +at the gfx provider's sync_state, after which genpd gates gfx in sleep +as on a cold boot, and the resume witnesses that power-off. + +Ask pmgr-pwrstate instead. The refusal remains, under the reason +"gfx-power-held", for the one case where the hold is still in place: a +consumer of the gfx domain that never probed. Cold boots never hold, so +they never refuse for it. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/core.inc | 1 + + drivers/gpu/drm/apple-g5p/system-sleep.inc | 25 +++++++++++----------- + 2 files changed, 13 insertions(+), 13 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +index 20eb5594f..72fc1ff4c 100644 +--- a/drivers/gpu/apple-g5p/core.inc ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + #include + #include + #include +diff --git a/drivers/gpu/drm/apple-g5p/system-sleep.inc b/drivers/gpu/drm/apple-g5p/system-sleep.inc +index 64bb87905..10c68dbc2 100644 +--- a/drivers/gpu/drm/apple-g5p/system-sleep.inc ++++ b/drivers/gpu/drm/apple-g5p/system-sleep.inc +@@ -150,17 +150,16 @@ static void g5p_sleep_sample(struct g5p_diag *g, u32 *state, u32 *mode, u32 *r8) + } + + /* +- * A kexec boot (DS108 mode 3) keeps every S8000 PMGR power state on for the +- * whole boot (pmgr-pwrstate keep_on, "retain the tethered warm boot state"): +- * genpd power-off of gfx returns -EBUSY. Sleep would then stop the firmware +- * without the platform off, and resume would boot it over a GPU that was +- * never power-cycled: a dirty start, which DS108 admits only through its one +- * bounded reset at probe. Refuse before anything is shut, until the warm boot +- * can power the GPU off like a cold one. RAM only. ++ * pmgr-pwrstate holds a kexec boot's domains on until the provider's ++ * sync_state, i.e. until every consumer has probed; after that gfx is gated ++ * in sleep as on a cold boot. While the hold is still in place (a consumer of ++ * gfx never probed) genpd's power-off would return -EBUSY and the GPU would ++ * sleep powered: refuse before anything is shut. Cold boots never hold. No ++ * GPU or PMGR register access. + */ +-static bool g5p_sleep_gfx_power_held(void) ++static bool g5p_sleep_gfx_power_held(struct g5p_diag *g) + { +- return g5p_ds_mode() >= 3; ++ return g->gfx_np && apple_pmgr_ps_power_held(g->gfx_np) != 0; + } + + /* R1 only, as Apple's isQuiescent/isStableOff. Never infer bus retirement +@@ -284,8 +283,8 @@ static int g5p_system_sleep_prepare(struct device *dev) + return -EBUSY; + } + d = g->event_cookie; +- if (g5p_sleep_gfx_power_held()) { +- dev_info(dev, "GPU-PM sleep refused: stage=entry why=warm-gfx-power-held\n"); ++ if (g5p_sleep_gfx_power_held(g)) { ++ dev_info(dev, "GPU-PM sleep refused: stage=entry why=gfx-power-held\n"); + return -EBUSY; + } + if (READ_ONCE(g->sleep_reserved)) { +@@ -481,8 +480,8 @@ static int g5p_system_sleep_check(struct device *dev) + if (!READ_ONCE(b->alive) || READ_ONCE(b->stopped) || !b->released || + !b->irq_ready) + return g5p_sleep_why(d, -ENODEV, "not-booted"); +- if (g5p_sleep_gfx_power_held()) +- return g5p_sleep_why(d, -EBUSY, "warm-gfx-power-held"); ++ if (g5p_sleep_gfx_power_held(g)) ++ return g5p_sleep_why(d, -EBUSY, "gfx-power-held"); + never_park = g5p_sleep_never_park(b); + if (!mutex_trylock(&g->lock)) + return never_park ? g5p_sleep_why(d, -EBUSY, "never-park/locked") : diff --git a/kernel/patches/drivers/drm/0042-drm-apple-g5p-bound-and-serialize-the-full-warm-rebo.patch b/kernel/patches/drivers/drm/0042-drm-apple-g5p-bound-and-serialize-the-full-warm-rebo.patch new file mode 100644 index 00000000..f0210ba9 --- /dev/null +++ b/kernel/patches/drivers/drm/0042-drm-apple-g5p-bound-and-serialize-the-full-warm-rebo.patch @@ -0,0 +1,323 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 20:07:12 +0100 +Subject: [PATCH] drm/apple-g5p: bound handover and cold-reset instead of + jumping with live DMA + +Keep admission closed and latch only successful handovers. On notifier +failure log at emergency level, request a pstore dump, override warm +reboot mode and emergency restart. The tethered watchdog guard gives the +native cold reset priority over PSCI. Never panic: the userspace warm +guard sets panic_timeout=0. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/boot.inc | 23 ++++- + drivers/gpu/apple-g5p/core.inc | 3 - + drivers/gpu/apple-g5p/teardown.inc | 89 +++++++++++++++---- + .../gpu/drm/apple-g5p/drm-render-sched.inc | 2 + + .../gpu/drm/apple-g5p/drm-render-unbind.inc | 7 +- + drivers/gpu/drm/apple-g5p/drm-render-v3.inc | 11 ++- + 6 files changed, 110 insertions(+), 25 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +index 94c135bea..62c6636b7 100644 +--- a/drivers/gpu/apple-g5p/boot.inc ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -176,6 +176,7 @@ struct g5p_boot { + int fifo_error; + int irq; + atomic_t irqs; ++ atomic_t irq_active; + spinlock_t event_lock; + void __iomem *swirq; + bool maps_ready, armed, copied, released, alive, failed, irq_ready; +@@ -559,6 +560,7 @@ static int g5p_boot_maps(struct g5p_diag *g) + b->compute_slot, b->compute_boot_vm, b->preinstall_context); + spin_lock_init(&b->event_lock); + atomic_set(&b->irqs, 0); ++ atomic_set(&b->irq_active, 0); + ret = g5p_plan_layout(&b->layout); + if (ret) + goto out; +@@ -965,7 +967,7 @@ static irqreturn_t g5p_boot_hardirq(int irq, void *data) + return IRQ_WAKE_THREAD; + } + +-static irqreturn_t g5p_boot_irq(int irq, void *data) ++static irqreturn_t g5p_boot_irq_run(int irq, void *data) + { + struct g5p_diag *g = data; + struct g5p_boot *b = g->boot; +@@ -999,6 +1001,25 @@ static irqreturn_t g5p_boot_irq(int irq, void *data) + return IRQ_HANDLED; + } + ++/* Shutdown masks new IRQs and waits only a bounded time for this RAM latch. ++ * A thread scheduled after freeze must not enter the GPU register path. ++ */ ++static irqreturn_t g5p_boot_irq(int irq, void *data) ++{ ++ struct g5p_diag *g = data; ++ struct g5p_boot *b = g->boot; ++ irqreturn_t ret = IRQ_HANDLED; ++ ++ atomic_inc(&b->irq_active); ++ smp_mb__after_atomic(); ++ if (!READ_ONCE(b->stopped)) ++ ret = g5p_boot_irq_run(irq, data); ++ else if (b->swirq) ++ writel(0x4000, b->swirq + 0x94); ++ atomic_dec(&b->irq_active); ++ return ret; ++} ++ + static int g5p_boot_irq_setup(struct g5p_diag *g) + { + struct g5p_boot *b = g->boot; +diff --git a/drivers/gpu/apple-g5p/core.inc b/drivers/gpu/apple-g5p/core.inc +index 72fc1ff4c..e2df8ca55 100644 +--- a/drivers/gpu/apple-g5p/core.inc ++++ b/drivers/gpu/apple-g5p/core.inc +@@ -588,10 +588,7 @@ static ssize_t command_store(struct device *dev, struct device_attribute *attr, + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + if (sysfs_streq(buf, "td-kexec")) { +- if (!g5p_sleep_enter(g)) +- return -EBUSY; + ret = g5p_td_command(g, buf); +- g5p_sleep_exit(g); + return ret ?: count; + } + ret = g5p_run_phase(g, buf, &ran); +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +index d40cfa26f..8d511b0ba 100644 +--- a/drivers/gpu/apple-g5p/teardown.inc ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -15,6 +15,7 @@ + * unchanged. docs/gpu/experiments/recovery108-driver.md. + */ + #include ++#include + #include + #include + #include "teardown.h" +@@ -438,6 +439,7 @@ struct g5p_kq_ctx { + struct g5p_diag *g; + u32 slots[G5P_RU_USER_SLOTS]; + u32 held; ++ bool irq_busy; + }; + + static void __maybe_unused g5p_kq_register(const struct g5p_kq_frontend *fe, +@@ -488,14 +490,17 @@ static void g5p_kq_inputs(void *ctx, struct g5p_ru_kq_in *in) + const struct g5p_kq_frontend *fe = smp_load_acquire(&g5p_kq_fe); + u32 *ctl; + +- if (!READ_ONCE(g->powered)) ++ if (g5p_r8_is_retired() || !READ_ONCE(g->powered)) + return; /* Already idle: no detach, MMIO or second gate cycle. */ + in->pmgr = !!g->pmgr; +- if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released) || +- READ_ONCE(b->parked)) ++ if (!b || !READ_ONCE(b->maps_ready) || !READ_ONCE(b->released)) + return; + in->booted = true; +- in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed); ++ /* PARK is not a witnessed platform off: it still needs the PMGR cycle, ++ * but a stopped firmware must never be sent another detach command. ++ */ ++ in->failed = READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || ++ READ_ONCE(b->stopped) || READ_ONCE(b->parked); + ctl = (u32 *)g5p_boot_shared(b, 0x6c); + dma_rmb(); + in->cons = READ_ONCE(ctl[0]); +@@ -529,13 +534,24 @@ static void g5p_kq_freeze(void *ctx) + */ + static void g5p_kq_irq_off(void *ctx) + { +- struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; ++ struct g5p_kq_ctx *c = ctx; ++ struct g5p_diag *g = c->g; + struct g5p_boot *b = g->boot; ++ u64 deadline = ktime_get_ns() + 20 * NSEC_PER_MSEC; + + g5p_td_op_aic_ack(g); + g5p_td_op_irq_mask(g); +- if (b && b->irq_ready && b->irq > 0) +- synchronize_irq(b->irq); ++ /* Pairs with the thread's active latch and stopped check. No unbounded ++ * synchronize_irq(): a stuck handler must not hold outgoing shutdown. ++ */ ++ smp_mb(); ++ while (b && atomic_read(&b->irq_active)) { ++ if (ktime_get_ns() >= deadline) { ++ c->irq_busy = true; ++ break; ++ } ++ usleep_range(100, 200); ++ } + } + + /* PMGR only: the bounded gate cycle of the r33/dirty-start path (refuses a gate that is not +@@ -545,6 +561,8 @@ static int g5p_kq_cycle(void *ctx) + { + struct g5p_diag *g = ((struct g5p_kq_ctx *)ctx)->g; + ++ if (((struct g5p_kq_ctx *)ctx)->irq_busy) ++ return -EBUSY; + g5p_mmio_gate(g); + return g5p_r33_gfx_off(g->dev, g->pmgr, G5P_KQ_OFF_MS); + } +@@ -563,29 +581,42 @@ static int g5p_kq_run(struct g5p_diag *g, const char *why) + }; + struct g5p_ru_kq_out out; + struct g5p_kq_ctx *c; ++ int ret; + u64 t0 = ktime_get_ns(); ++ u64 deadline = t0 + (u64)G5P_KQ_LOCK_MS * NSEC_PER_MSEC; + ++ if (atomic_read(&g5p_kq_ran) == 1) ++ return -EALREADY; /* Only a witnessed successful handover is latched. */ ++ if (g5p_r8_is_retired()) { ++ dev_emerg(g->dev, "GPU-KQ why=%s retired; no MMIO\n", why); ++ return -ENODEV; ++ } + if (READ_ONCE(g->sleep_failed)) { + dev_emerg(g->dev, + "GPU-KQ frozen after sleep failure; no MMIO\n"); + return -EIO; + } +- if (atomic_xchg(&g5p_kq_ran, 1)) { +- dev_emerg( +- g->dev, +- "GPU-KQ why=%s already ran this boot; nothing done end=kq\n", +- why); +- return -EALREADY; ++ /* Match sleep prepare: close new admission even while users are inside. ++ * A timeout still runs freeze/IRQ mask and the command-lock-safe plan. ++ */ ++ atomic_or(INT_MIN, &g->sleep_gate); ++ while (atomic_read(&g->sleep_gate) & INT_MAX) { ++ if (ktime_get_ns() >= deadline) { ++ dev_emerg(g->dev, "GPU-KQ why=%s sleep users busy; forcing quiesce\n", why); ++ break; ++ } ++ msleep(5); + } ++ if (atomic_cmpxchg(&g5p_kq_ran, 0, -1)) ++ return atomic_read(&g5p_kq_ran) == 1 ? -EALREADY : -EBUSY; + c = kzalloc(sizeof(*c), GFP_KERNEL); + if (!c) { +- dev_emerg(g->dev, "GPU-KQ why=%s ret=%d end=kq\n", why, +- -ENOMEM); ++ atomic_set(&g5p_kq_ran, 0); + return -ENOMEM; + } + c->g = g; + o.ctx = c; +- g5p_ru_kq_run(&o, min((G5P_RU_KQ_FULL), G5P_RU_KQ_FULL), &out); ++ g5p_ru_kq_run(&o, G5P_RU_KQ_FULL, &out); + dev_emerg( + g->dev, + "GPU-KQ why=%s mode=%u inside=%u locked=%u booted=%u failed=%u wedged=%u " +@@ -599,7 +630,11 @@ static int g5p_kq_run(struct g5p_diag *g, const char *why) + out.detach_ret, out.froze, out.cycle_ret, + (ktime_get_ns() - t0) / NSEC_PER_MSEC); + kfree(c); +- return out.detach_ret ?: out.cycle_ret; ++ ret = out.cycle_ret ?: ++ (out.in.booted && !(out.plan & G5P_RU_KQ_CYCLE) ? -EBUSY : 0); ++ /* A successful cycle supersedes a failed detach; DMA is then stopped. */ ++ atomic_set(&g5p_kq_ran, ret ? 0 : 1); ++ return ret; + } + + /* sysfs `command`: td-capture / td-final / td-kexec. Runs before the command mutex, like +@@ -637,7 +672,25 @@ static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, + action != SYS_POWER_OFF) + return NOTIFY_DONE; + /* The quiesce needs a live firmware for its detach, so it runs before the ladder freezes. */ +- g5p_kq_run(g, "reboot-notifier"); ++ { ++ int ret = g5p_kq_run(g, "reboot-notifier"); ++ ++ dev_emerg(g->dev, "GPU-KQ notifier outcome=%d\n", ret); ++ /* The warm guard sets panic_timeout=0. Never panic or return to ++ * kexec with live DMA: persist the reason, override the guard, ++ * and take the native watchdog's cold-reset path. ++ */ ++ if (ret && ret != -EALREADY) { ++ dev_emerg(g->dev, ++ "GPU-KQ unsafe handover ret=%d; forcing cold watchdog reset to iOS\n", ret); ++ /* PANIC dump reason reaches default pstore filters without ++ * entering panic() or consulting panic_timeout. ++ */ ++ kmsg_dump_desc(KMSG_DUMP_PANIC, "GPU-KQ cold-reset fallback (no panic)"); ++ WRITE_ONCE(reboot_mode, REBOOT_COLD); ++ emergency_restart(); ++ } ++ } + { + return NOTIFY_DONE; + } +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +index c2463af31..2b6f3bb9e 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-sched.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-sched.inc +@@ -181,6 +181,8 @@ static void g5p_rsched_work(struct work_struct *work) + g5p_render_close_deferred(d); + continue; + } ++ if (atomic_read(&d->core->sleep_gate) < 0) ++ break; + /* A lost job blocks the channel: hold every job until the drain is decided, and + * until a recovery the drain or the executor asked for has run. + */ +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +index 92d1a57f7..0bc1786b1 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-unbind.inc +@@ -66,7 +66,12 @@ static int g5p_rv3_unbind_slots(struct g5p_drm *d, const u32 *slots, u32 n, + return -EBUSY; + } + /* Invalidate PM-record reuse before any unbind is published. */ +- mutex_lock(&rd->lock); ++ if (!strcmp(why, "kexec")) { ++ if (!mutex_trylock(&rd->lock)) ++ return -EBUSY; ++ } else { ++ mutex_lock(&rd->lock); ++ } + { + struct g5p_render *r; + +diff --git a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +index 70b4ba2fb..d3369a2ff 100644 +--- a/drivers/gpu/drm/apple-g5p/drm-render-v3.inc ++++ b/drivers/gpu/drm/apple-g5p/drm-render-v3.inc +@@ -1156,8 +1156,9 @@ static void g5p_rv3_dc_kick(struct g5p_diag *g, u32 start, u32 target, + READ_ONCE(b->control_complete), + g5p_rv3_wk_ds108(b), + READ_ONCE(g->reset_transition)); +- if (READ_ONCE(b->failed) || READ_ONCE(b->event_failed) || +- READ_ONCE(b->stopped) || g5p_ru_term_failed(&g5p_rv3_term) || ++ if (g5p_r8_is_retired() || READ_ONCE(b->failed) || ++ READ_ONCE(b->event_failed) || READ_ONCE(b->stopped) || ++ g5p_ru_term_failed(&g5p_rv3_term) || + !g5p_wk_power_ready(power) || g5p_rv3_wk_r74f_busy(b->r73b) || + cons != start || READ_ONCE(ctl[4]) != target) + goto out; +@@ -1224,6 +1225,12 @@ static int g5p_rv3_dc_wait(struct g5p_diag *g, u32 target, u64 *consumed_ns) + case G5P_RU_BIND_CONSUMED: + *consumed_ns = ktime_get_ns(); + return 0; ++ case G5P_RU_BIND_PROTO: ++ return -EPROTO; ++ case G5P_RU_BIND_TIMEOUT: ++ return -ETIMEDOUT; ++ case G5P_RU_BIND_PENDING: ++ break; + } + usleep_range(100, 200); + } diff --git a/kernel/patches/drivers/drm/0043-drm-apple-g5p-ring-a-phase-s-doorbell-unless-its-own-stamp.patch b/kernel/patches/drivers/drm/0043-drm-apple-g5p-ring-a-phase-s-doorbell-unless-its-own-stamp.patch new file mode 100644 index 00000000..4e9be273 --- /dev/null +++ b/kernel/patches/drivers/drm/0043-drm-apple-g5p-ring-a-phase-s-doorbell-unless-its-own-stamp.patch @@ -0,0 +1,72 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:42:25 +0100 +Subject: [PATCH] drm/apple-g5p: ring a phase's doorbell unless its own stamp + retired it + +The publish-time doorbell of a render phase was skipped when the +engine's completion-event count had moved since the job was armed. +The count is taken when the job starts, before its TA is published, and +with two jobs in flight the previous job's 3D completes during this +job's TA: its completion event moved the count, the 3D of this job was +taken for already retired, and its doorbell was never rung. The +firmware then did not consume the 3D command until something else woke +its scheduler; when nothing did, the event stream stayed silent for a +second and the job and its context were lost ("3d-silent"). Scenes +with long fragment phases hit it: glmark2 terrain lost its context in +two of about eleven runs. + +Apple's TA and 3D submit paths ring the doorbell after every ring entry +without any retirement check, and a doorbell for an entry the firmware +already consumed adds no work. Skip it only when the phase's own stamp +words show the phase retired; keep the event count as evidence. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/render-user-wake.h | 12 ++++++++---- + 1 file changed, 8 insertions(+), 4 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/render-user-wake.h b/drivers/gpu/apple-g5p/render-user-wake.h +index c8fd0aa..a4650d5 100644 +--- a/drivers/gpu/apple-g5p/render-user-wake.h ++++ b/drivers/gpu/apple-g5p/render-user-wake.h +@@ -29,7 +29,7 @@ + * repaired, never a power arm), dma_wmb(), R0+0xb00 = 0x10 | engine. At most one attempt + * per phase. Never after a timeout or fault (the terminal latch's sticky `failed`), never + * with the boot failed or stopped, never unpowered, never with an r74f bell in flight, +- * never once the phase shows any sign of retirement, never on a ring that is not exactly ++ * never once the phase's own stamp shows it retired, never on a ring that is not exactly + * this publish. A consumer already at the target does not cancel (Apple never checks + * it); it is recorded. The natural TA completion (terminal kind COMPLETE, the epoch's + * first note) is not a failure: the factor's sequence is the one licensed exception to the +@@ -90,7 +90,7 @@ enum g5p_wk_did { + G5P_WK_SKIP_POWER, /* one of G5P_WK_PWR_REQUIRED is absent */ + G5P_WK_SKIP_R74F, /* an r74f bell is published, open or firing */ + G5P_WK_SKIP_RING, /* the ring is not exactly this publish */ +- G5P_WK_SKIP_RETIRED, /* a stamp == seq or a completion event since the publish */ ++ G5P_WK_SKIP_RETIRED, /* a stamp == seq: this phase retired */ + G5P_WK_SKIP_FENCE, /* R1+0x14 read back outside 7: no further register */ + G5P_WK_SKIP_GPC, /* GPC not active: no 0xb00 */ + G5P_WK_DIDS, +@@ -120,7 +120,11 @@ struct g5p_wk_in { + u32 start, target, prod, + cons; /* ring: producer before this publish, our target, now */ + u32 stamp[2]; /* this engine's stamp words */ +- u32 type1; /* this engine's completion events decoded since the phase armed */ ++ /* This engine's completion events decoded since the job armed: evidence only. With two ++ * jobs in flight the previous job's completion lands here too, so it cannot tell that ++ * this phase retired; the stamps can. ++ */ ++ u32 type1; + }; + + /* The register access of the factor, in order. The kernel binds them to writel/readl on the +@@ -221,7 +225,7 @@ static inline u32 g5p_wk_guard(const struct g5p_wk_in *in) + in->cons > 255 || ((in->target - in->start) & 255) != 1 || + ((in->cons - in->start) & 255) > 1) + return G5P_WK_SKIP_RING; +- if (in->stamp[0] == in->seq || in->stamp[1] == in->seq || in->type1) ++ if (in->stamp[0] == in->seq || in->stamp[1] == in->seq) + return G5P_WK_SKIP_RETIRED; + return 0; + } diff --git a/kernel/patches/drivers/drm/0044-drm-apple-g5p-wait-out-a-power-on-request-that-ALIVE.patch b/kernel/patches/drivers/drm/0044-drm-apple-g5p-wait-out-a-power-on-request-that-ALIVE.patch new file mode 100644 index 00000000..46b72250 --- /dev/null +++ b/kernel/patches/drivers/drm/0044-drm-apple-g5p-wait-out-a-power-on-request-that-ALIVE.patch @@ -0,0 +1,255 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 15:30:00 +0100 +Subject: [PATCH] drm/apple-g5p: wait out a power-on request that ALIVE + overtook + +The firmware boots without waiting for the GPU cores' power-on, so its +ALIVE message can arrive while the power-on request is still in +flight. The request register then still has its busy bit (bit 4) set +next to the target state, e.g. 0x14 for state 4. After ALIVE, only the +first-ALIVE handler may read the power registers, so the release poll +stops there. It already handed an unread state to that handler, but it +counted a busy read as a failed power-on. That latched a power abort and +left the GPU unusable for the boot, which then fell back to software +rendering. On J71t this happened twice in a day of boots, once cold and +once warm. The last reads were 0x14 after 43 and 52 polls, while good +power-ons settle after 13 to 54 polls. + +Apple's setGPCPerformanceState and setGPUCorePower wait for bit 4 to +clear before anything else and allow up to 30 s. When the last read +shows a request toward a powered state still in flight and the poll +has time left, defer the decision to the first-ALIVE handler as for +an unread state. The handler waits for bit 4 for the rest of the +power-on bound, never less than its own limit. Its settled read decides +before any GPU register write. + +So no power register is read beyond what the release poll was already +allowed. A poll that ran out its bound, an all-ones read, a parking +request (0x10) or a settled state that is not powered still abort as +before. The settle loop now stops re-reading once register access is +revoked or the boot has failed or stopped. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/boot-r73b.inc | 2 ++ + drivers/gpu/apple-g5p/boot.inc | 20 +++++++---- + drivers/gpu/apple-g5p/lifecycle.inc | 24 +++++++++---- + drivers/gpu/apple-g5p/r74e112.h | 52 ++++++++++++++++++++++++++--- + 4 files changed, 80 insertions(+), 18 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/boot-r73b.inc b/drivers/gpu/apple-g5p/boot-r73b.inc +index 3948aab..d47d8d5 100644 +--- a/drivers/gpu/apple-g5p/boot-r73b.inc ++++ b/drivers/gpu/apple-g5p/boot-r73b.inc +@@ -287,6 +287,8 @@ struct g5p_r73b { + atomic_t early_proof; + u32 early_proof_gen, early_proof_gen_now, early_proof_why; + s32 early_defer_ret; ++ /* The owner's bit-4 wait for a deferred proof (g5p_px_settle_us()). */ ++ u32 early_proof_settle_us; + /* boot_alive_settle_us: limit, first in-flight value seen, re-reads, time spent. */ + u32 settle_us, settle_first, settle_samples; + u64 settle_ns; +diff --git a/drivers/gpu/apple-g5p/boot.inc b/drivers/gpu/apple-g5p/boot.inc +index 836a1d4..585e699 100644 +--- a/drivers/gpu/apple-g5p/boot.inc ++++ b/drivers/gpu/apple-g5p/boot.inc +@@ -1235,6 +1235,7 @@ static int g5p_r74e_fender_start(struct g5p_diag *g) + bool on_ran = false; + u32 samples = + 0; /* wake112: the reads of the poll that ran last (the printed raw) */ ++ u32 on_bound_us = b->perf_poweron ? 10000 : 1800, on_us; + int ret; + + r->early_power_start_ns = ktime_get_mono_fast_ns(); +@@ -1252,8 +1253,7 @@ static int g5p_r74e_fender_start(struct g5p_diag *g) + if (!ret) { + on_ran = true; + /* poweron: the first request above pstate 1, a voltage step; a longer bound */ +- ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, +- b->perf_poweron ? 10000 : 1800, ++ ret = g5p_r74e_power_poll(g, b->perf_poweron ?: 1, on_bound_us, + &r->early_on_ns, &samples); + if (b->perf_poweron) + dev_emerg( +@@ -1262,12 +1262,17 @@ static int g5p_r74e_fender_start(struct g5p_diag *g) + b->perf_poweron, r->early_power_raw, + r->early_on_ns, ret); + } +- /* wake112: only ALIVE preempting the on-poll before its first read defers; the +- * first-ALIVE owner owes the proof. A sampled on-poll (0x11, ...) aborts as before. ++ /* ALIVE preempting the on-poll before its first read, or while its last read ++ * shows a request still in flight with time left, defers; the first-ALIVE owner owes ++ * the proof and waits out the rest of the bound. Any other sampled preemption aborts. + */ +- latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples); ++ on_us = (u32)min_t(u64, div_u64(r->early_on_ns, NSEC_PER_USEC), U32_MAX); ++ latch = g5p_px_poll(on_ran, ret, ret == -EHOSTDOWN, samples, ++ r->early_power_raw, on_us, on_bound_us); + if (latch == G5P_PXP_DEFER) { + r->early_defer_ret = ret; ++ r->early_proof_settle_us = g5p_px_settle_us( ++ samples, on_us, on_bound_us, r->settle_us); + r->early_proof_gen = g5p_ds_gen(); + ret = 0; + atomic_set_release(&r->early_proof, G5P_PX_DEFERRED); +@@ -1288,9 +1293,10 @@ static int g5p_r74e_fender_start(struct g5p_diag *g) + if (latch == G5P_PXP_DEFER) + dev_emerg( + g->dev, +- "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u end=r74e112\n", ++ "R74E112 defer gen=%u on_ret=%d raw=%08x alive=%u settle_us=%u end=r74e112\n", + r->early_proof_gen, r->early_defer_ret, +- r->early_power_raw, READ_ONCE(b->alive)); ++ r->early_power_raw, READ_ONCE(b->alive), ++ r->early_proof_settle_us); + return ret; + } + +diff --git a/drivers/gpu/apple-g5p/lifecycle.inc b/drivers/gpu/apple-g5p/lifecycle.inc +index 5195849..164d538 100644 +--- a/drivers/gpu/apple-g5p/lifecycle.inc ++++ b/drivers/gpu/apple-g5p/lifecycle.inc +@@ -174,20 +174,27 @@ static int g5p_r73e_prepare(struct g5p_diag *g) + + /* Apple's GPC precondition (AGXMetaFender_setGPCPerformanceState, ...6ca4ccc): a request in + * device test shows R1+0x1000 bit 4; wait for it to clear before judging the state. R1 reads only, +- * bounded by boot_alive_settle_us; with 0 the first read stands, as before. ++ * bounded by limit_us (boot_alive_settle_us, or a deferred proof's rest of the power-on bound); ++ * with 0 the first read stands, as before. Each re-read first rechecks that R1 is still ++ * licensed and the boot neither failed nor stopped; if not, the busy word stands and refuses. + */ +-static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw) ++static u32 g5p_r74e_settled(struct g5p_diag *g, struct g5p_r73b *r, u32 raw, ++ u32 limit_us) + { ++ struct g5p_boot *b = g->boot; + u64 start; + +- if (!r->settle_us || !(raw & 0x10)) ++ if (!limit_us || !(raw & 0x10)) + return raw; + if (!r->settle_first) + r->settle_first = raw; + start = ktime_get_mono_fast_ns(); + while ((raw & 0x10) && ktime_get_mono_fast_ns() - start < +- (u64)r->settle_us * NSEC_PER_USEC) { ++ (u64)limit_us * NSEC_PER_USEC) { + udelay(10); ++ if (!g5p_r8_r1_allowed() || READ_ONCE(b->stopped) || ++ READ_ONCE(b->failed)) ++ break; + raw = readl(g->window[1] + 0x1000); + r->settle_samples++; + } +@@ -216,7 +223,10 @@ static int g5p_r73e_irq_submit(struct g5p_diag *g) + return -EPROTO; + if (gate == G5P_PXG_PROVE && !g5p_r74e_take(g)) + return -EHOSTDOWN; +- r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ r->early_state = g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000), ++ gate == G5P_PXG_PROVE ? ++ r->early_proof_settle_us : ++ r->settle_us); + if (gate == G5P_PXG_PROVE) + g5p_r74e_settle(r, r->early_state); + if (!g5p_gpc_ok(r->early_state)) +@@ -241,8 +251,8 @@ static int g5p_r73e_irq_submit(struct g5p_diag *g) + raw = readl(g->window[1] + 0x14); + if (raw & ~7U) + return -EPROTO; +- r->early_after_state = +- g5p_r74e_settled(g, r, readl(g->window[1] + 0x1000)); ++ r->early_after_state = g5p_r74e_settled( ++ g, r, readl(g->window[1] + 0x1000), r->settle_us); + if (!g5p_gpc_ok(r->early_after_state)) + return -EHOSTDOWN; + dev_emerg( +diff --git a/drivers/gpu/apple-g5p/r74e112.h b/drivers/gpu/apple-g5p/r74e112.h +index cacac73..4a0b44f 100644 +--- a/drivers/gpu/apple-g5p/r74e112.h ++++ b/drivers/gpu/apple-g5p/r74e112.h +@@ -25,6 +25,17 @@ + * before ALIVE (0x11 in rounds 74e..79b, or any other word) and every other R74E failure + * (off-poll, timeout, GPC1 write refused) abort exactly as before. + * ++ * One sampled case defers as well: the last read shows a power-state request still in ++ * flight (bit 4 set, a nonzero target, no other bit) and ALIVE arrived while the poll ++ * still had time left. The firmware boots without waiting for the GPC, so ALIVE can land ++ * while the request is still in flight (0x14 after 43 and 52 reads, cold and warm; good ++ * power-ons settle after 13 to 54 reads). Apple's ++ * setGPCPerformanceState and setGPUCorePower wait for bit 4 to clear before anything ++ * else, for up to 30 s. The owner continues the poll for the rest of its bound ++ * (g5p_px_settle_us()): no R1 read beyond what the release poll was allowed, and no ++ * register write before the state field settles. A poll that ran out its bound, an ++ * all-ones read, a parking request (0x10) or a settled wrong state still abort. ++ * + * States (r73b->early_proof): + * NONE no deferral: the poll decided, or the release has not run + * DEFERRED ALIVE preempted the on-poll; one owner read is owed (generation recorded) +@@ -38,6 +49,8 @@ + + #define G5P_PX_STATE_MASK 0x1fU + #define G5P_PX_STATE_ON 0x01U ++#define G5P_PX_REQ_BUSY 0x10U /* a power-state request is in flight */ ++#define G5P_PX_REQ_TARGET 0x0fU + + enum g5p_px { + G5P_PX_NONE, +@@ -74,20 +87,51 @@ enum g5p_pxp { + G5P_PXP_ABORT, /* anything else: power abort (cold) */ + }; + ++/* A request toward a powered state still in flight, and nothing else: 0x11..0x1f. */ ++static inline bool g5p_px_in_flight(u32 raw) ++{ ++ return (raw & ~G5P_PX_STATE_MASK) == 0 && (raw & G5P_PX_REQ_BUSY) && ++ (raw & G5P_PX_REQ_TARGET); ++} ++ + /* + * on_ran: the core request, the off-poll and the GPC1 write succeeded and the on-poll + * ran. ret: the last step's result. preempted: the on-poll stopped on ALIVE. +- * samples: the on-poll's R1+0x1000 reads. Only a preemption with no read defers (no +- * abort, no release_ready); a sampled one saw the state and aborts. ++ * samples: the on-poll's R1+0x1000 reads; raw: the last of them. elapsed_us, bound_us: ++ * the on-poll's time and its bound. A preemption defers (no abort, no release_ready) ++ * when the poll read nothing, or when its last read is a request still in flight and the ++ * bound has time left; any other sampled preemption saw a wrong state and aborts. + */ + static inline enum g5p_pxp g5p_px_poll(bool on_ran, int ret, bool preempted, +- u32 samples) ++ u32 samples, u32 raw, u32 elapsed_us, ++ u32 bound_us) + { + if (!on_ran) + return G5P_PXP_ABORT; + if (!ret) + return G5P_PXP_READY; +- return preempted && !samples ? G5P_PXP_DEFER : G5P_PXP_ABORT; ++ if (!preempted) ++ return G5P_PXP_ABORT; ++ if (!samples) ++ return G5P_PXP_DEFER; ++ return g5p_px_in_flight(raw) && elapsed_us < bound_us ? G5P_PXP_DEFER : ++ G5P_PXP_ABORT; ++} ++ ++/* ++ * How long the owner may wait for bit 4 to clear before its read decides. An unsampled ++ * deferral keeps the owner's own limit (settle_us); a sampled in-flight one continues the ++ * on-poll for the rest of its bound, never less than settle_us. ++ */ ++static inline u32 g5p_px_settle_us(u32 samples, u32 elapsed_us, u32 bound_us, ++ u32 settle_us) ++{ ++ u32 left; ++ ++ if (!samples || elapsed_us >= bound_us) ++ return settle_us; ++ left = bound_us - elapsed_us; ++ return left > settle_us ? left : settle_us; + } + + /* The first-ALIVE owner's admission. */ diff --git a/kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch b/kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch new file mode 100644 index 00000000..47656153 --- /dev/null +++ b/kernel/patches/drivers/gpio/0006-gpio-apple-d2365-refuse-sleep-when-the-wake-mask-wri.patch @@ -0,0 +1,33 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] gpio: apple-d2365: refuse sleep when the wake mask write + fails + +apple_pmic_gpio_suspend() warned and went on when writing the event wake +mask failed. The system then slept with wake sources the PMIC would not +report, and could not be woken by them. Return the error so that the +suspend is aborted, and drop the IRQ lock on that path. + +Assisted-by: LLM +--- + drivers/gpio/gpio-apple-d2365.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpio/gpio-apple-d2365.c b/drivers/gpio/gpio-apple-d2365.c +index ba66e8eab..c516dfe30 100644 +--- a/drivers/gpio/gpio-apple-d2365.c ++++ b/drivers/gpio/gpio-apple-d2365.c +@@ -588,8 +588,10 @@ static int apple_pmic_gpio_suspend(struct device *dev) + mutex_lock(&gpio->irq_lock); + /* Only wake-enabled events may pull the line while asleep. */ + ret = apple_pmic_event_sync(gpio, gpio->wake); +- if (ret) +- dev_warn(dev, "event wake mask write failed: %d\n", ret); ++ if (ret) { ++ mutex_unlock(&gpio->irq_lock); ++ return ret; ++ } + gpio->suspended = true; + mutex_unlock(&gpio->irq_lock); + /* A thread that passed the gate before it closed has finished now. */ diff --git a/kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch b/kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch new file mode 100644 index 00000000..24f22212 --- /dev/null +++ b/kernel/patches/drivers/gpio/0007-dt-bindings-gpio-apple-d2365-pmic-gpio-describe-wake.patch @@ -0,0 +1,69 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:27:23 +0000 +Subject: [PATCH] dt-bindings: gpio: apple,d2365-pmic-gpio: describe wake-only + events + +Add apple,wake-events: GPIO events (0x30 plus the line) that end system +sleep and have no interrupt consumer, such as the iPad 5 Smart Cover +hall sensors on lines 0 and 10. A hall sensor that is unmasked while the +system is awake drives the PMU line on every change of the magnet field, +so these events are unmasked only for the duration of a sleep and cannot +be mapped as interrupts. Only the D2365 has the event controller they +need. + +Assisted-by: LLM +--- + .../bindings/gpio/apple,d2365-pmic-gpio.yaml | 18 ++++++++++++++++++ + 1 file changed, 18 insertions(+) + +diff --git a/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml b/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml +index c1cae13ab..6241bfc50 100644 +--- a/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml ++++ b/Documentation/devicetree/bindings/gpio/apple,d2365-pmic-gpio.yaml +@@ -54,6 +54,21 @@ properties: + maxItems: 1 + description: PMU event output of the PMIC, active low. + ++ apple,wake-events: ++ $ref: /schemas/types.yaml#/definitions/uint32-array ++ minItems: 1 ++ maxItems: 24 ++ items: ++ minimum: 0x30 ++ maximum: 0x47 ++ description: | ++ GPIO events (0x30 plus the line number) that end system sleep and have ++ no interrupt consumer, such as the iPad 5 Smart Cover hall sensors on ++ lines 0 and 10 (events 0x30 and 0x3a). They stay masked while the ++ system is awake, where the line is read through the GPIO instead, and ++ are unmasked only while it sleeps: a hall sensor drives the PMU line on ++ every change of the magnet field. They cannot be mapped as interrupts. ++ + required: + - compatible + - gpio-controller +@@ -61,6 +76,7 @@ required: + + dependencies: + interrupt-controller: ["#interrupt-cells"] ++ apple,wake-events: [interrupt-controller] + + allOf: + - if: +@@ -74,6 +90,7 @@ allOf: + interrupt-controller: false + "#interrupt-cells": false + interrupts: false ++ apple,wake-events: false + + additionalProperties: false + +@@ -92,6 +109,7 @@ examples: + #interrupt-cells = <2>; + interrupt-controller; + interrupts = <38 IRQ_TYPE_LEVEL_LOW>; ++ apple,wake-events = <0x30 0x3a>; + }; + }; + ... diff --git a/kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch b/kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch new file mode 100644 index 00000000..67b97c85 --- /dev/null +++ b/kernel/patches/drivers/gpio/0008-gpio-apple-d2365-wake-from-sleep-on-listed-GPIO-even.patch @@ -0,0 +1,446 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:27:23 +0000 +Subject: [PATCH] gpio: apple-d2365: wake from sleep on listed GPIO events + +The Smart Cover halls cannot be interrupt consumers: unmasked while the +system is awake, they keep the PMU line asserted (the 2026-11-07 storm +with IRQ gpio-keys). The switches stay polled; make the halls wake-only +instead. + +Events in apple,wake-events get no interrupt mapping and stay masked +while awake. Every event latches whether masked or not, so ->suspend +stops the event thread and reads the block before it unmasks them, or +the cover close that started the suspend would end it at once. Consumer +events in that read are dispatched as the thread would have, and a wake +source among them ends the transition. ->suspend then unmasks the wake +and wake-only events and takes a wake reference on the PMU line, which +no child interrupt holds for these events. Any failure there masks the +halls again before returning, because ->resume will not run. ->resume +masks them and, while the thread is still gated, reads the latched +events and the line levels as the wake reason, then dispatches the +consumers. + +The read of the event block is factored out of the thread and shared; +the line interrupt's consumers and the storm accounting stay in the +thread. At probe, each wake line's boot configuration is logged next to +the rule the iOS sleep path applies (config byte & 0xc1 == 1). + +Assisted-by: LLM +--- + drivers/gpio/gpio-apple-d2365.c | 258 +++++++++++++++++++++++++++++--- + 1 file changed, 235 insertions(+), 23 deletions(-) + +diff --git a/drivers/gpio/gpio-apple-d2365.c b/drivers/gpio/gpio-apple-d2365.c +index c516dfe30..604d4cba3 100644 +--- a/drivers/gpio/gpio-apple-d2365.c ++++ b/drivers/gpio/gpio-apple-d2365.c +@@ -11,6 +11,12 @@ + * One more interrupt, numbered after the last event, is the line itself: its + * consumers run before the block is read, for a consumer that must act + * within the shortest time after an assertion (charger withdrawal). ++ * ++ * GPIO events listed in apple,wake-events have no consumer: they stay ++ * masked while the system is awake and are unmasked only while it sleeps. ++ * An event latches whether it is masked or not, so the block is read ++ * before they are unmasked, or a move from before the suspend (the cover ++ * close that started it) would end the sleep at once. + */ + + #include +@@ -72,6 +78,10 @@ + #define D2365_EVENT_MASK 0x1c0 + #define D2365_EVENT_BYTES 30 + #define D2365_GPIO_EVENT 0x30 ++/* iOS keeps a GPIO event unmasked in sleep if its config byte & 0xc1 == 1. */ ++#define D2365_GPIO_SLEEP_RULE_MASK 0xc1 ++#define D2365_GPIO_SLEEP_RULE_VALUE 0x01 ++#define APPLE_PMIC_INPUT_BYTES 3 + + #define APPLE_PMIC_EVENT_BYTES_MAX 30 + #define APPLE_PMIC_EVENTS_MAX (APPLE_PMIC_EVENT_BYTES_MAX * 8) +@@ -116,6 +126,8 @@ struct apple_pmic_gpio { + struct delayed_work irq_retry; + DECLARE_BITMAP(enabled, APPLE_PMIC_IRQS_MAX); /* unmasked while awake */ + DECLARE_BITMAP(wake, APPLE_PMIC_EVENTS_MAX); /* unmasked while asleep */ ++ DECLARE_BITMAP(wake_only, APPLE_PMIC_EVENTS_MAX); /* apple,wake-events */ ++ u8 armed_level[APPLE_PMIC_INPUT_BYTES]; + u8 mask[APPLE_PMIC_EVENT_BYTES_MAX]; /* as last written */ + u8 boot_mask[APPLE_PMIC_EVENT_BYTES_MAX]; + int wake_delta; /* parent wake references to apply */ +@@ -126,6 +138,8 @@ struct apple_pmic_gpio { + bool irq_held; + bool removing; + bool shutting_down; ++ bool wake_only_armed; /* wake_only unmasked for this sleep */ ++ bool parent_wake_armed; /* our own wake reference on the line */ + }; + + static unsigned int d2207_gpio_config_reg(unsigned int offset) +@@ -285,6 +299,9 @@ static int apple_pmic_irq_map(struct irq_domain *domain, unsigned int irq, + + if (hwirq > gpio->nevents) + return -EINVAL; ++ /* Unmasked while awake, a hall sensor holds the PMU line. */ ++ if (hwirq < gpio->nevents && test_bit(hwirq, gpio->wake_only)) ++ return -EINVAL; + + irq_set_chip_data(irq, gpio); + irq_set_chip_and_handler(irq, &apple_pmic_irq_chip, handle_simple_irq); +@@ -346,7 +363,8 @@ static bool apple_pmic_event_gate(struct apple_pmic_gpio *gpio) + * suspend_device_irqs() must abort the transition, as the armed + * interrupt does afterwards. + */ +- if (!bitmap_empty(gpio->wake, gpio->nevents)) ++ if (!bitmap_empty(gpio->wake, gpio->nevents) || ++ gpio->wake_only_armed) + pm_system_wakeup(); + apple_pmic_irq_hold(gpio, false); + return false; +@@ -363,6 +381,41 @@ static void apple_pmic_dispatch(struct apple_pmic_gpio *gpio, + handle_nested_irq(child); + } + ++/* ++ * Read (and so clear) the event block: @latched gets every set event, ++ * @pending the ones with a consumer. Caller holds irq_lock. ++ */ ++static int apple_pmic_event_read(struct apple_pmic_gpio *gpio, ++ unsigned long *latched, unsigned long *pending) ++{ ++ const struct apple_pmic_event_info *ev = gpio->info->events; ++ u8 events[APPLE_PMIC_EVENT_BYTES_MAX]; ++ unsigned int hwirq; ++ int ret; ++ ++ bitmap_zero(latched, APPLE_PMIC_EVENTS_MAX); ++ bitmap_zero(pending, APPLE_PMIC_EVENTS_MAX); ++ ret = regmap_bulk_read(gpio->regmap, ev->status_base, events, ev->bytes); ++ if (ret) ++ return ret; ++ ++ for (hwirq = 0; hwirq < gpio->nevents; hwirq++) ++ if (events[hwirq / 8] & BIT(hwirq % 8)) ++ __set_bit(hwirq, latched); ++ bitmap_and(pending, latched, gpio->enabled, gpio->nevents); ++ return 0; ++} ++ ++/* Consumers may mask their interrupt, which takes the bus lock. */ ++static void apple_pmic_dispatch_events(struct apple_pmic_gpio *gpio, ++ const unsigned long *pending) ++{ ++ unsigned int hwirq; ++ ++ for_each_set_bit(hwirq, pending, gpio->nevents) ++ apple_pmic_dispatch(gpio, hwirq); ++} ++ + /* + * Per assertion of the PMU line: first the consumers of the line itself, + * then one read of the event block; every set event with a consumer is then +@@ -377,10 +430,8 @@ static void apple_pmic_dispatch(struct apple_pmic_gpio *gpio, + static irqreturn_t apple_pmic_event_thread(int irq, void *data) + { + struct apple_pmic_gpio *gpio = data; +- const struct apple_pmic_event_info *ev = gpio->info->events; ++ DECLARE_BITMAP(latched, APPLE_PMIC_EVENTS_MAX); + DECLARE_BITMAP(pending, APPLE_PMIC_EVENTS_MAX); +- u8 events[APPLE_PMIC_EVENT_BYTES_MAX]; +- unsigned int hwirq; + int ret; + + bitmap_zero(pending, APPLE_PMIC_EVENTS_MAX); +@@ -408,15 +459,10 @@ static irqreturn_t apple_pmic_event_thread(int irq, void *data) + } + + /* Reading the events clears them and releases the line. */ +- ret = regmap_bulk_read(gpio->regmap, ev->status_base, events, ev->bytes); ++ ret = apple_pmic_event_read(gpio, latched, pending); + if (ret) + goto hold; + gpio->retries = 0; +- +- for (hwirq = 0; hwirq < gpio->nevents; hwirq++) +- if (events[hwirq / 8] & BIT(hwirq % 8) && +- test_bit(hwirq, gpio->enabled)) +- __set_bit(hwirq, pending); + mutex_unlock(&gpio->irq_lock); + goto dispatch; + +@@ -429,9 +475,7 @@ static irqreturn_t apple_pmic_event_thread(int irq, void *data) + mutex_unlock(&gpio->irq_lock); + + dispatch: +- /* Consumers may mask their interrupt, which takes the bus lock. */ +- for_each_set_bit(hwirq, pending, gpio->nevents) +- apple_pmic_dispatch(gpio, hwirq); ++ apple_pmic_dispatch_events(gpio, pending); + return IRQ_HANDLED; + + unlock: +@@ -481,6 +525,59 @@ static void apple_pmic_event_stop(void *data) + free_irq(gpio->parent_irq, gpio); + } + ++/* What the boot firmware left on a wake-only line, next to iOS's sleep rule. */ ++static void apple_pmic_wake_report(struct apple_pmic_gpio *gpio, ++ unsigned int event) ++{ ++ const struct apple_pmic_gpio_info *info = gpio->info; ++ struct device *dev = gpio->chip.parent; ++ unsigned int line = event - info->events->gpio_event; ++ unsigned int reg = info->config_reg(line); ++ unsigned int cfg, cfg_hi, level; ++ ++ if (regmap_read(gpio->regmap, reg, &cfg) || ++ regmap_read(gpio->regmap, reg + 1, &cfg_hi) || ++ regmap_read(gpio->regmap, info->input_base + line / 8, &level)) { ++ dev_warn(dev, "wake event 0x%02x: cannot read line %u\n", ++ event, line); ++ return; ++ } ++ dev_info(dev, ++ "wake event 0x%02x: line %u config 0x%02x/0x%02x level %u, mask at boot %s, iOS keeps it %s in sleep\n", ++ event, line, cfg, cfg_hi, !!(level & BIT(line % 8)), ++ gpio->boot_mask[event / 8] & BIT(event % 8) ? "masked" : "open", ++ (cfg & D2365_GPIO_SLEEP_RULE_MASK) == D2365_GPIO_SLEEP_RULE_VALUE ? ++ "unmasked" : "masked"); ++} ++ ++static void apple_pmic_wake_only_parse(struct apple_pmic_gpio *gpio) ++{ ++ const struct apple_pmic_event_info *ev = gpio->info->events; ++ struct device *dev = gpio->chip.parent; ++ u32 events[APPLE_PMIC_EVENTS_MAX / 8]; ++ int count, i; ++ ++ count = device_property_count_u32(dev, "apple,wake-events"); ++ if (count <= 0) ++ return; ++ if (count > gpio->info->ngpio || ++ device_property_read_u32_array(dev, "apple,wake-events", events, ++ count)) { ++ dev_warn(dev, "ignoring unreadable apple,wake-events\n"); ++ return; ++ } ++ for (i = 0; i < count; i++) { ++ if (events[i] < ev->gpio_event || ++ events[i] >= ev->gpio_event + gpio->info->ngpio || ++ events[i] >= gpio->nevents) { ++ dev_warn(dev, "ignoring wake event 0x%x: not a GPIO event\n", ++ events[i]); ++ continue; ++ } ++ __set_bit(events[i], gpio->wake_only); ++ } ++} ++ + static int apple_pmic_event_probe(struct platform_device *pdev, + struct apple_pmic_gpio *gpio) + { +@@ -496,6 +593,7 @@ static int apple_pmic_event_probe(struct platform_device *pdev, + if (gpio->parent_irq < 0) + return gpio->parent_irq; + gpio->nevents = ev->bytes * 8; ++ apple_pmic_wake_only_parse(gpio); + + ret = devm_mutex_init(dev, &gpio->irq_lock); + if (ret) +@@ -540,7 +638,19 @@ static int apple_pmic_event_probe(struct platform_device *pdev, + if (ret) + return dev_err_probe(dev, ret, + "failed to request the PMU interrupt\n"); +- return devm_add_action_or_reset(dev, apple_pmic_event_stop, gpio); ++ ret = devm_add_action_or_reset(dev, apple_pmic_event_stop, gpio); ++ if (ret) ++ return ret; ++ ++ if (!bitmap_empty(gpio->wake_only, gpio->nevents)) { ++ unsigned int event; ++ ++ for_each_set_bit(event, gpio->wake_only, gpio->nevents) ++ apple_pmic_wake_report(gpio, event); ++ dev_info(dev, "wake-only events %*pbl on IRQ %d\n", ++ gpio->nevents, gpio->wake_only, gpio->parent_irq); ++ } ++ return 0; + } + + static struct platform_driver d2365_gpio_driver; +@@ -576,32 +686,108 @@ void apple_d2365_gpio_quiesce(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_d2365_gpio_quiesce); + ++/* ++ * Drop what latched while awake before the wake-only events are unmasked, ++ * with the event thread stopped. Consumer events in that read are run as ++ * the thread would have run them; a wake source among them ends this ++ * transition, as it would once the line is armed. ++ */ ++static void apple_pmic_wake_only_drop(struct apple_pmic_gpio *gpio) ++{ ++ struct device *dev = gpio->chip.parent; ++ const struct apple_pmic_gpio_info *info = gpio->info; ++ DECLARE_BITMAP(latched, APPLE_PMIC_EVENTS_MAX); ++ DECLARE_BITMAP(pending, APPLE_PMIC_EVENTS_MAX); ++ int ret; ++ ++ mutex_lock(&gpio->irq_lock); ++ ret = apple_pmic_event_read(gpio, latched, pending); ++ if (regmap_bulk_read(gpio->regmap, info->input_base, gpio->armed_level, ++ APPLE_PMIC_INPUT_BYTES)) ++ memset(gpio->armed_level, 0xff, sizeof(gpio->armed_level)); ++ mutex_unlock(&gpio->irq_lock); ++ if (ret) { ++ /* Unknown: as the thread does, every consumer may have an event. */ ++ dev_warn(dev, "cannot drop the latched events: %d\n", ret); ++ bitmap_copy(pending, gpio->enabled, gpio->nevents); ++ } ++ if (bitmap_intersects(pending, gpio->wake, gpio->nevents)) ++ pm_system_wakeup(); ++ apple_pmic_dispatch_events(gpio, pending); ++} ++ + static int apple_pmic_gpio_suspend(struct device *dev) + { + struct apple_pmic_gpio *gpio = dev_get_drvdata(dev); ++ DECLARE_BITMAP(armed, APPLE_PMIC_EVENTS_MAX); ++ bool wake_only = false; + int ret; + + if (!gpio->domain) + return 0; + + cancel_delayed_work_sync(&gpio->irq_retry); ++ if (!bitmap_empty(gpio->wake_only, gpio->nevents)) { ++ wake_only = true; ++ disable_irq(gpio->parent_irq); ++ apple_pmic_wake_only_drop(gpio); ++ } ++ + mutex_lock(&gpio->irq_lock); + /* Only wake-enabled events may pull the line while asleep. */ +- ret = apple_pmic_event_sync(gpio, gpio->wake); +- if (ret) { +- mutex_unlock(&gpio->irq_lock); +- return ret; +- } ++ bitmap_or(armed, gpio->wake, gpio->wake_only, gpio->nevents); ++ ret = apple_pmic_event_sync(gpio, armed); ++ if (ret) ++ goto unwind; + gpio->suspended = true; ++ gpio->wake_only_armed = wake_only; + mutex_unlock(&gpio->irq_lock); ++ if (wake_only) ++ enable_irq(gpio->parent_irq); + /* A thread that passed the gate before it closed has finished now. */ + synchronize_irq(gpio->parent_irq); ++ ++ if (wake_only) { ++ /* No child interrupt takes a wake reference for these. */ ++ ret = irq_set_irq_wake(gpio->parent_irq, 1); ++ if (ret) { ++ dev_err(dev, "cannot arm the PMU interrupt for wake: %d\n", ++ ret); ++ mutex_lock(&gpio->irq_lock); ++ gpio->suspended = false; ++ gpio->wake_only_armed = false; ++ goto unwind_enabled; ++ } ++ gpio->parent_wake_armed = true; ++ dev_info(dev, "wake events %*pbl armed, GPIO levels %*phN\n", ++ gpio->nevents, gpio->wake_only, APPLE_PMIC_INPUT_BYTES, ++ gpio->armed_level); ++ } + return 0; ++ ++unwind: ++ /* resume() will not run for this device: mask the halls again here. */ ++ if (wake_only) ++ enable_irq(gpio->parent_irq); ++unwind_enabled: ++ if (apple_pmic_event_sync(gpio, gpio->enabled)) ++ dev_err(dev, "event mask restore failed\n"); ++ /* A line held while gated is not re-enabled by a resume of ours. */ ++ if (gpio->irq_held) { ++ gpio->irq_held = false; ++ enable_irq(gpio->parent_irq); ++ } ++ mutex_unlock(&gpio->irq_lock); ++ return ret; + } + + static int apple_pmic_gpio_resume(struct device *dev) + { + struct apple_pmic_gpio *gpio = dev_get_drvdata(dev); ++ DECLARE_BITMAP(latched, APPLE_PMIC_EVENTS_MAX); ++ DECLARE_BITMAP(pending, APPLE_PMIC_EVENTS_MAX); ++ u8 level[APPLE_PMIC_INPUT_BYTES] = {}; ++ bool armed, read = false; + int ret; + + if (!gpio->domain) +@@ -609,17 +795,43 @@ static int apple_pmic_gpio_resume(struct device *dev) + + /* Our I2C ancestors have completed their resume callbacks. */ + mutex_lock(&gpio->irq_lock); +- gpio->suspended = false; +- gpio->retries = 0; + ret = apple_pmic_event_sync(gpio, gpio->enabled); + if (ret) + dev_warn(dev, "event mask write failed: %d\n", ret); +- /* An event that arrived while gated is still latched. */ ++ armed = gpio->wake_only_armed; ++ gpio->wake_only_armed = false; ++ bitmap_zero(pending, APPLE_PMIC_EVENTS_MAX); ++ if (armed) { ++ /* Still gated: the thread cannot take the wake reason first. */ ++ regmap_bulk_read(gpio->regmap, gpio->info->input_base, level, ++ sizeof(level)); ++ read = !apple_pmic_event_read(gpio, latched, pending); ++ } ++ gpio->suspended = false; ++ gpio->retries = 0; ++ /* An event that arrived while gated is still latched, unless read here. */ + if (gpio->irq_held) { + gpio->irq_held = false; + enable_irq(gpio->parent_irq); + } + mutex_unlock(&gpio->irq_lock); ++ ++ if (armed) { ++ if (read) { ++ bitmap_and(latched, latched, gpio->wake_only, gpio->nevents); ++ dev_info(dev, "wake events disarmed, latched %*pbl, GPIO levels %*phN (armed at %*phN)\n", ++ gpio->nevents, latched, APPLE_PMIC_INPUT_BYTES, ++ level, APPLE_PMIC_INPUT_BYTES, gpio->armed_level); ++ } else { ++ /* Unknown: every consumer may have an event. */ ++ bitmap_copy(pending, gpio->enabled, gpio->nevents); ++ } ++ if (gpio->parent_wake_armed) { ++ irq_set_irq_wake(gpio->parent_irq, 0); ++ gpio->parent_wake_armed = false; ++ } ++ apple_pmic_dispatch_events(gpio, pending); ++ } + return 0; + } + +@@ -728,13 +940,13 @@ static int apple_pmic_gpio_probe(struct platform_device *pdev) + gpio->regmap = dev_get_regmap(dev->parent, NULL); + if (!gpio->regmap) + return dev_err_probe(dev, -ENODEV, "parent regmap unavailable\n"); ++ gpio->chip.parent = dev; + + ret = apple_pmic_event_probe(pdev, gpio); + if (ret) + return ret; + + gpio->chip.label = gpio->info->label; +- gpio->chip.parent = dev; + gpio->chip.owner = THIS_MODULE; + gpio->chip.base = -1; + gpio->chip.ngpio = gpio->info->ngpio; diff --git a/kernel/patches/drivers/gpu/0014-drm-apple-g5p-cycle-the-gfx-gate-with-auto-PM-on-as-.patch b/kernel/patches/drivers/gpu/0014-drm-apple-g5p-cycle-the-gfx-gate-with-auto-PM-on-as-.patch new file mode 100644 index 00000000..ffd0d50a --- /dev/null +++ b/kernel/patches/drivers/gpu/0014-drm-apple-g5p-cycle-the-gfx-gate-with-auto-PM-on-as-.patch @@ -0,0 +1,92 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:08:39 +0100 +Subject: [PATCH] drm/apple-g5p: cycle the gfx gate with auto-PM on as + pmgr-pwrstate does + +The bounded gfx gate cycle (the incoming dirty-start reset, the in-boot +recovery and the kexec quiesce) refused any gate with auto-PM set. +pmgr-pwrstate sets auto-PM whenever genpd powers gfx on, which is every +cold boot: there the cycle was always refused, so a cold boot could not +recover the GPU in place, and once a warm boot gates idle domains like a +cold one it could not either. + +Do what pmgr-pwrstate does for every transition it makes: clear auto-PM +together with the off target, and set it again only after the on target +is reached. The minimum-state and reset refusals are unchanged. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/power-r33.inc | 27 +++++++++++++++++++-------- + 1 file changed, 19 insertions(+), 8 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/power-r33.inc b/drivers/gpu/apple-g5p/power-r33.inc +index ce8a18f3c..4edc3b4c8 100644 +--- a/drivers/gpu/apple-g5p/power-r33.inc ++++ b/drivers/gpu/apple-g5p/power-r33.inc +@@ -3,10 +3,14 @@ + * R0+0x418/0x41c (no reset) keeps the previous boot's value through it. At the + * first Fender wake, before any GPU MMIO and before any GPU IRQ is requested, + * hold the same PMGR gate (0x80340, gate124) off for boot_gfx_off_ms, then +- * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset, +- * auto). Every exit leaves the gate requested on. ++ * restore target 0xf. The refusal rules are pmgr-pwrstate's own (min, reset). ++ * Auto-PM, which pmgr-pwrstate turns on whenever genpd powers gfx (every cold ++ * boot), is handled as pmgr-pwrstate handles it: cleared with the off target, ++ * set again once the on target is reached. Every exit leaves the gate ++ * requested on. + */ + #define G5P_R33_GFX_PS 0x80340U ++#define G5P_R33_GFX_AUTO BIT(28) + #define G5P_R33_GFX_OFF_MAX_MS 2000U + #define G5P_R33_GFX_POLL_US 10000U + +@@ -15,16 +19,15 @@ static inline bool g5p_r33_gfx_on(u32 raw) + return (raw & 0xf) == 0xf && ((raw >> 4) & 0xf) == 0xf; + } + +-/* Only a plain, fully-on, software-controlled gate may be cycled. */ ++/* Only a plain, fully-on gate with no floor and no reset may be cycled. */ + static inline bool g5p_r33_gfx_cyclable(u32 raw) + { +- return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31) && +- !((raw >> 28) & 1); ++ return g5p_r33_gfx_on(raw) && !((raw >> 16) & 0xf) && !(raw >> 31); + } + + static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) + { +- u32 before, held, v = 0, after = 0; ++ u32 before, held, v = 0, after = 0, autopm; + u64 t0; + int ret, on; + +@@ -38,11 +41,13 @@ static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) + if (!g5p_r33_gfx_cyclable(before)) { + dev_info( + dev, +- "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset/auto or not on\n", ++ "POWER gfx-off refused ms=%u before=%08x; pmgr min/reset or not on\n", + ms, before); + return -EBUSY; + } +- ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, 0xf, 0); ++ /* pmgr-pwrstate apple_pmgr_ps_set(): auto-PM off with the target write. */ ++ autopm = before & G5P_R33_GFX_AUTO; ++ ret = regmap_update_bits(pmgr, G5P_R33_GFX_PS, autopm | 0xf, 0); + if (!ret) + ret = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, v, + !((v >> 4) & 0xf), 10, +@@ -59,6 +64,12 @@ static int g5p_r33_gfx_off(struct device *dev, struct regmap *pmgr, u32 ms) + on = regmap_read_poll_timeout(pmgr, G5P_R33_GFX_PS, after, + g5p_r33_gfx_on(after), 10, + G5P_R33_GFX_POLL_US); ++ /* ... and back on only once the on target is reached, as there. */ ++ if (!on && autopm) { ++ on = regmap_update_bits(pmgr, G5P_R33_GFX_PS, autopm, autopm); ++ if (!on) ++ on = regmap_read(pmgr, G5P_R33_GFX_PS, &after); ++ } + dev_info( + dev, + "POWER gfx-off ms=%u held_us=%u before=%08x end=%08x after=%08x off_ret=%d " diff --git a/kernel/patches/drivers/gpu/0015-drm-apple-g5p-accept-an-already-completed-kexec-hand.patch b/kernel/patches/drivers/gpu/0015-drm-apple-g5p-accept-an-already-completed-kexec-hand.patch new file mode 100644 index 00000000..0b719245 --- /dev/null +++ b/kernel/patches/drivers/gpu/0015-drm-apple-g5p-accept-an-already-completed-kexec-hand.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:32 +0100 +Subject: [PATCH] drm/apple-g5p: accept an already completed kexec handover + +Report a second td-kexec as success so quiesce-gpu removes its pending +receipt. Other sysfs errors remain failures. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/teardown.inc | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +index 8d511b0ba..051ddedb8 100644 +--- a/drivers/gpu/apple-g5p/teardown.inc ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -650,8 +650,11 @@ static int g5p_td_command(struct g5p_diag *g, const char *command) + ret = g5p_td_capture(g, "sysfs"); + else if (sysfs_streq(command, "td-final")) + ret = g5p_td_final(g, "sysfs"); +- else if (sysfs_streq(command, "td-kexec")) ++ else if (sysfs_streq(command, "td-kexec")) { + ret = g5p_kq_run(g, "sysfs"); ++ if (ret == -EALREADY) ++ return 0; /* The witnessed handover is already complete. */ ++ } + else + return -EINVAL; + return ret == -EAGAIN ? 0 : ret; diff --git a/kernel/patches/drivers/gpu/0016-drm-apple-g5p-preserve-halt-and-poweroff-after-quies.patch b/kernel/patches/drivers/gpu/0016-drm-apple-g5p-preserve-halt-and-poweroff-after-quies.patch new file mode 100644 index 00000000..16c27cf1 --- /dev/null +++ b/kernel/patches/drivers/gpu/0016-drm-apple-g5p-preserve-halt-and-poweroff-after-quies.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:32 +0100 +Subject: [PATCH] drm/apple-g5p: preserve halt and poweroff after quiesce + failure + +Only restart can expose a new kernel to retained DMA. Never turn halt or +poweroff into a cold reboot. A fully retired instance is already handed +over and needs no MMIO or second cycle. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/teardown.inc | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +index 051ddedb8..93eb34f79 100644 +--- a/drivers/gpu/apple-g5p/teardown.inc ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -589,7 +589,7 @@ static int g5p_kq_run(struct g5p_diag *g, const char *why) + return -EALREADY; /* Only a witnessed successful handover is latched. */ + if (g5p_r8_is_retired()) { + dev_emerg(g->dev, "GPU-KQ why=%s retired; no MMIO\n", why); +- return -ENODEV; ++ return -EALREADY; /* td-final retired this instance; no DMA remains. */ + } + if (READ_ONCE(g->sleep_failed)) { + dev_emerg(g->dev, +@@ -683,7 +683,7 @@ static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, + * kexec with live DMA: persist the reason, override the guard, + * and take the native watchdog's cold-reset path. + */ +- if (ret && ret != -EALREADY) { ++ if (action == SYS_RESTART && ret && ret != -EALREADY) { + dev_emerg(g->dev, + "GPU-KQ unsafe handover ret=%d; forcing cold watchdog reset to iOS\n", ret); + /* PANIC dump reason reaches default pstore filters without diff --git a/kernel/patches/drivers/gpu/0017-drm-apple-g5p-flush-consoles-before-emergency-cold-r.patch b/kernel/patches/drivers/gpu/0017-drm-apple-g5p-flush-consoles-before-emergency-cold-r.patch new file mode 100644 index 00000000..3f668f14 --- /dev/null +++ b/kernel/patches/drivers/gpu/0017-drm-apple-g5p-flush-consoles-before-emergency-cold-r.patch @@ -0,0 +1,38 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:33 +0100 +Subject: [PATCH] drm/apple-g5p: flush consoles before emergency cold reset + +Use the shared cold escape to give printk threads a bounded opportunity +to deliver evidence before IRQs and other CPUs stop. A pstore dump alone +is not recoverable evidence across an iOS boot. + +Assisted-by: LLM +--- + drivers/gpu/apple-g5p/teardown.inc | 9 +++------ + 1 file changed, 3 insertions(+), 6 deletions(-) + +diff --git a/drivers/gpu/apple-g5p/teardown.inc b/drivers/gpu/apple-g5p/teardown.inc +index 93eb34f79..252c3e488 100644 +--- a/drivers/gpu/apple-g5p/teardown.inc ++++ b/drivers/gpu/apple-g5p/teardown.inc +@@ -1,3 +1,5 @@ ++#include ++ + // SPDX-License-Identifier: GPL-2.0-only OR MIT + /* + * Kernel glue for the GPU teardown state machine (teardown.h). +@@ -686,12 +688,7 @@ static int g5p_td_reboot_notify(struct notifier_block *nb, unsigned long action, + if (action == SYS_RESTART && ret && ret != -EALREADY) { + dev_emerg(g->dev, + "GPU-KQ unsafe handover ret=%d; forcing cold watchdog reset to iOS\n", ret); +- /* PANIC dump reason reaches default pstore filters without +- * entering panic() or consulting panic_timeout. +- */ +- kmsg_dump_desc(KMSG_DUMP_PANIC, "GPU-KQ cold-reset fallback (no panic)"); +- WRITE_ONCE(reboot_mode, REBOOT_COLD); +- emergency_restart(); ++ apple_cold_reset("GPU-KQ cold-reset fallback (no panic)"); + } + } + { diff --git a/kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..0bf54215 --- /dev/null +++ b/kernel/patches/drivers/iio/0005-iio-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,771 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] iio: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/iio/accel/Kconfig | 2 +- + drivers/iio/accel/apple_aop_accel.c | 50 +++- + drivers/iio/gyro/Kconfig | 12 + + drivers/iio/gyro/Makefile | 1 + + drivers/iio/gyro/apple_aop_gyro.c | 291 ++++++++++++++++++++++ + drivers/iio/pressure/Kconfig | 12 + + drivers/iio/pressure/Makefile | 1 + + drivers/iio/pressure/apple_aop_pressure.c | 257 +++++++++++++++++++ + 8 files changed, 621 insertions(+), 5 deletions(-) + create mode 100644 drivers/iio/gyro/apple_aop_gyro.c + create mode 100644 drivers/iio/pressure/apple_aop_pressure.c + +diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig +index b01598194..ed328a590 100644 +--- a/drivers/iio/accel/Kconfig ++++ b/drivers/iio/accel/Kconfig +@@ -213,7 +213,7 @@ config APPLE_AOP_ACCEL + select IIO_KFIFO_BUF + help + Say yes here to get support for the accelerometer that the Always-On +- Processor (AOP) of Apple A10 iPads and iPhones serves. ++ Processor (AOP) of Apple A9 and A10 iPads and iPhones serves. + + To compile this driver as a module, choose M here: the module will be + called apple_aop_accel. +diff --git a/drivers/iio/accel/apple_aop_accel.c b/drivers/iio/accel/apple_aop_accel.c +index 96c901361..e1b0e136d 100644 +--- a/drivers/iio/accel/apple_aop_accel.c ++++ b/drivers/iio/accel/apple_aop_accel.c +@@ -9,9 +9,13 @@ + * + * 0 u32 sequence + * 4 u16 reserved +- * 6 s32 x, y, z in 1/65536 g, in the device frame (x right, y up, +- * z out of the screen; +1 g opposite to gravity) ++ * 6 s32 x, y, z in 1/65536 g + * 18 u32 timestamp ++ * ++ * The T8010's firmware reports in the device frame (x right, y up, z out of ++ * the screen; +1 g opposite to gravity). The S8000's firmware first needs the ++ * unit's factory calibration and reports in the sensor's frame; the board's ++ * mount-matrix maps it. + */ + + #include +@@ -21,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -41,8 +46,16 @@ + /* 9.80665 m/s^2 / 65536 */ + #define AOP_ACCEL_SCALE_NANO 149637 + ++struct apple_aop_accel_hw { ++ const struct apple_aop_calibration *calibration; ++ unsigned int num_calibration; ++ /* No separate raw sample stream (raw interval 0) */ ++ bool no_raw_stream; ++}; ++ + struct apple_aop_accel { + struct device *dev; ++ const struct apple_aop_accel_hw *hw; + const struct apple_aop_service *service; + struct iio_mount_matrix orientation; + +@@ -75,7 +88,9 @@ static int apple_aop_accel_set_interval(struct apple_aop_accel *accel, u32 inter + int i, ret; + + for (i = 0; i < ARRAY_SIZE(tags); i++) { +- ret = apple_aop_accel_set_property(accel, tags[i], interval_us); ++ ret = apple_aop_accel_set_property(accel, tags[i], ++ tags[i] == AOP_PROP_RAW_INTERVAL && ++ accel->hw->no_raw_stream ? 0 : interval_us); + if (ret) + return ret; + } +@@ -230,6 +245,7 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return -ENOMEM; + accel = iio_priv(indio_dev); + accel->dev = dev; ++ accel->hw = device_get_match_data(dev); + accel->service = service; + mutex_init(&accel->lock); + spin_lock_init(&accel->sample_lock); +@@ -252,6 +268,15 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (accel->hw->num_calibration) { ++ ret = apple_aop_calibrate(service, dev, accel->hw->calibration, ++ accel->hw->num_calibration); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to calibrate\n"); ++ } ++ } ++ + ret = apple_aop_accel_set_interval(accel, AOP_ACCEL_DEFAULT_US); + if (ret) { + apple_aop_unlisten(service); +@@ -265,8 +290,25 @@ static int apple_aop_accel_probe(struct platform_device *pdev) + return devm_iio_device_register(dev, indio_dev); + } + ++/* Offset at low temperature, sensitivity (either), interrupt calibration */ ++static const struct apple_aop_calibration apple_aop_accel_s8000_calibration[] = { ++ { "apple,low-temp-accel-offset", 0x1a, 1 }, ++ { "apple,accel-sensitivity-calibration", 0x0b, 2 }, ++ { "apple,accel-range-sensitivity-cal", 0x27, 2 }, ++ { "apple,accel-interrupt-calibration", 0x19, 3 }, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_s8000 = { ++ .calibration = apple_aop_accel_s8000_calibration, ++ .num_calibration = ARRAY_SIZE(apple_aop_accel_s8000_calibration), ++ .no_raw_stream = true, ++}; ++ ++static const struct apple_aop_accel_hw apple_aop_accel_t8010 = { }; ++ + static const struct of_device_id apple_aop_accel_of_match[] = { +- { .compatible = "apple,t8010-aop-accel" }, ++ { .compatible = "apple,s8000-aop-accel", .data = &apple_aop_accel_s8000 }, ++ { .compatible = "apple,t8010-aop-accel", .data = &apple_aop_accel_t8010 }, + { } + }; + MODULE_DEVICE_TABLE(of, apple_aop_accel_of_match); +diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig +index 3e193ee0f..8f506a65d 100644 +--- a/drivers/iio/gyro/Kconfig ++++ b/drivers/iio/gyro/Kconfig +@@ -63,6 +63,18 @@ config ADXRS450 + This driver can also be built as a module. If so, the module + will be called adxrs450. + ++config APPLE_AOP_GYRO ++ tristate "Apple AOP gyroscope" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the gyroscope that the Always-On ++ Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_gyro. ++ + config BMG160 + tristate "BOSCH BMG160 Gyro Sensor" + depends on (I2C || SPI_MASTER) +diff --git a/drivers/iio/gyro/Makefile b/drivers/iio/gyro/Makefile +index 0319b397d..212bbb814 100644 +--- a/drivers/iio/gyro/Makefile ++++ b/drivers/iio/gyro/Makefile +@@ -10,6 +10,7 @@ obj-$(CONFIG_ADIS16136) += adis16136.o + obj-$(CONFIG_ADIS16260) += adis16260.o + obj-$(CONFIG_ADXRS290) += adxrs290.o + obj-$(CONFIG_ADXRS450) += adxrs450.o ++obj-$(CONFIG_APPLE_AOP_GYRO) += apple_aop_gyro.o + obj-$(CONFIG_BMG160) += bmg160_core.o + obj-$(CONFIG_BMG160_I2C) += bmg160_i2c.o + obj-$(CONFIG_BMG160_SPI) += bmg160_spi.o +diff --git a/drivers/iio/gyro/apple_aop_gyro.c b/drivers/iio/gyro/apple_aop_gyro.c +new file mode 100644 +index 000000000..edbda1f43 +--- /dev/null ++++ b/drivers/iio/gyro/apple_aop_gyro.c +@@ -0,0 +1,291 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Gyroscope served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the gyroscope itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one motion report (EPIC ++ * subtype 0xc1) per report interval, laid out like the accelerometer's: ++ * ++ * 0 u16 sequence ++ * 2 u32 status ++ * 6 s32 x, y, z in 1/65536 degree per second ++ * 18 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_GYRO 0xc1 ++#define AOP_GYRO_LEN 22 ++#define AOP_GYRO_XYZ 6 ++ ++#define AOP_GYRO_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_GYRO_MIN_US 10000 /* 100 Hz */ ++#define AOP_GYRO_MAX_US 1000000 /* 1 Hz */ ++ ++/* pi / 180 rad/s / 65536 = 1171270 / 2^42 rad/s */ ++#define AOP_GYRO_SCALE 1171270 ++#define AOP_GYRO_SCALE_LOG2 42 ++ ++struct apple_aop_gyro { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ struct iio_mount_matrix orientation; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ s32 sample[3]; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(s32, scan, 3); ++}; ++ ++static int apple_aop_gyro_set_u32(struct apple_aop_gyro *gyro, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(gyro->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_gyro_set_interval(struct apple_aop_gyro *gyro, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ gyro->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_gyro_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int i; ++ ++ if (subtype != AOP_REPORT_GYRO || len < AOP_GYRO_LEN) ++ return; ++ ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ for (i = 0; i < 3; i++) ++ gyro->sample[i] = get_unaligned_le32(data + AOP_GYRO_XYZ + 4 * i); ++ gyro->have_sample = true; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ memcpy(gyro->scan, gyro->sample, sizeof(gyro->sample)); ++ iio_push_to_buffers_with_ts(indio_dev, gyro->scan, sizeof(gyro->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_gyro_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&gyro->sample_lock, flags); ++ *val = gyro->sample[chan->scan_index]; ++ ret = gyro->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&gyro->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ *val = AOP_GYRO_SCALE; ++ *val2 = AOP_GYRO_SCALE_LOG2; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&gyro->lock); ++ *val = USEC_PER_SEC / gyro->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % gyro->interval_us) * USEC_PER_SEC / ++ gyro->interval_us; ++ mutex_unlock(&gyro->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_gyro_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_GYRO_MIN_US, AOP_GYRO_MAX_US); ++ ++ mutex_lock(&gyro->lock); ++ ret = apple_aop_gyro_set_interval(gyro, interval); ++ mutex_unlock(&gyro->lock); ++ return ret; ++} ++ ++static const struct iio_mount_matrix * ++apple_aop_gyro_get_mount_matrix(const struct iio_dev *indio_dev, ++ const struct iio_chan_spec *chan) ++{ ++ struct apple_aop_gyro *gyro = iio_priv(indio_dev); ++ ++ return &gyro->orientation; ++} ++ ++static const struct iio_chan_spec_ext_info apple_aop_gyro_ext_info[] = { ++ IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, apple_aop_gyro_get_mount_matrix), ++ { } ++}; ++ ++#define APPLE_AOP_GYRO_CHANNEL(axis, index) { \ ++ .type = IIO_ANGL_VEL, \ ++ .modified = 1, \ ++ .channel2 = IIO_MOD_##axis, \ ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ ++ .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \ ++ .scan_index = index, \ ++ .scan_type = { \ ++ .sign = 's', \ ++ .realbits = 32, \ ++ .storagebits = 32, \ ++ .endianness = IIO_CPU, \ ++ }, \ ++ .ext_info = apple_aop_gyro_ext_info, \ ++} ++ ++static const struct iio_chan_spec apple_aop_gyro_channels[] = { ++ APPLE_AOP_GYRO_CHANNEL(X, 0), ++ APPLE_AOP_GYRO_CHANNEL(Y, 1), ++ APPLE_AOP_GYRO_CHANNEL(Z, 2), ++ IIO_CHAN_SOFT_TIMESTAMP(3), ++}; ++ ++static const struct iio_info apple_aop_gyro_info = { ++ .read_raw = apple_aop_gyro_read_raw, ++ .write_raw = apple_aop_gyro_write_raw, ++}; ++ ++/* Sensitivity and interrupt calibration make it complete */ ++static const struct apple_aop_calibration apple_aop_gyro_calibration[] = { ++ { "apple,gyro-sensitivity-calibration", 0x0b, 1 }, ++ { "apple,gyro-interrupt-calibration", 0x19, 2 }, ++ { "apple,gyro-range-sensitivity-cal", 0x27, 0 }, ++ { "apple,gyro-trim-calibration", 0x20, 0 }, ++}; ++ ++static void apple_aop_gyro_stop(void *data) ++{ ++ struct apple_aop_gyro *gyro = data; ++ ++ apple_aop_gyro_set_u32(gyro, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(gyro->service); ++} ++ ++static int apple_aop_gyro_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_gyro *gyro; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*gyro)); ++ if (!indio_dev) ++ return -ENOMEM; ++ gyro = iio_priv(indio_dev); ++ gyro->dev = dev; ++ gyro->service = service; ++ mutex_init(&gyro->lock); ++ spin_lock_init(&gyro->sample_lock); ++ ++ ret = iio_read_mount_matrix(dev, &gyro->orientation); ++ if (ret) ++ return ret; ++ ++ indio_dev->name = "apple-aop-gyro"; ++ indio_dev->info = &apple_aop_gyro_info; ++ indio_dev->channels = apple_aop_gyro_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_gyro_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_gyro_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_gyro_calibration, ++ ARRAY_SIZE(apple_aop_gyro_calibration)); ++ if (!ret) ++ ret = apple_aop_gyro_set_interval(gyro, AOP_GYRO_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_gyro_stop, gyro); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_gyro_of_match[] = { ++ { .compatible = "apple,s8000-aop-gyro" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_gyro_of_match); ++ ++static struct platform_driver apple_aop_gyro_driver = { ++ .driver = { ++ .name = "apple-aop-gyro", ++ .of_match_table = apple_aop_gyro_of_match, ++ }, ++ .probe = apple_aop_gyro_probe, ++}; ++module_platform_driver(apple_aop_gyro_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP gyroscope driver"); ++MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig +index 838a8340c..18535ad99 100644 +--- a/drivers/iio/pressure/Kconfig ++++ b/drivers/iio/pressure/Kconfig +@@ -384,4 +384,16 @@ config ADP810 + To compile this driver as a module, choose M here: the module will + be called adp810 + ++config APPLE_AOP_PRESSURE ++ tristate "Apple AOP barometric pressure sensor" ++ depends on APPLE_AOP_T8010 ++ select IIO_BUFFER ++ select IIO_KFIFO_BUF ++ help ++ Say yes here to get support for the barometric pressure sensor that ++ the Always-On Processor (AOP) of Apple A9 iPads and iPhones serves. ++ ++ To compile this driver as a module, choose M here: the module will be ++ called apple_aop_pressure. ++ + endmenu +diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile +index bc0d11a20..bf3e27ba9 100644 +--- a/drivers/iio/pressure/Makefile ++++ b/drivers/iio/pressure/Makefile +@@ -9,6 +9,7 @@ obj-$(CONFIG_ABP2030PA) += abp2030pa.o + obj-$(CONFIG_ABP2030PA_I2C) += abp2030pa_i2c.o + obj-$(CONFIG_ABP2030PA_SPI) += abp2030pa_spi.o + obj-$(CONFIG_ADP810) += adp810.o ++obj-$(CONFIG_APPLE_AOP_PRESSURE) += apple_aop_pressure.o + obj-$(CONFIG_ROHM_BM1390) += rohm-bm1390.o + obj-$(CONFIG_BMP280) += bmp280.o + bmp280-objs := bmp280-core.o bmp280-regmap.o +diff --git a/drivers/iio/pressure/apple_aop_pressure.c b/drivers/iio/pressure/apple_aop_pressure.c +new file mode 100644 +index 000000000..8cd983c57 +--- /dev/null ++++ b/drivers/iio/pressure/apple_aop_pressure.c +@@ -0,0 +1,257 @@ ++// SPDX-License-Identifier: GPL-2.0-only OR MIT ++/* ++ * Barometric pressure sensor served by the Apple Always-On Processor (AOP) ++ * ++ * The AOP firmware samples the pressure sensor itself. Once it has the unit's ++ * factory calibration and its batch, raw and report intervals (EPIC ++ * set-property tags 0x16, 0x5c and 0), it sends one report (EPIC subtype ++ * 0xc4) per report interval: ++ * ++ * 0 u16 sequence ++ * 2 u32 pressure in 1/65536 kPa ++ * 6 s32 temperature (unit unknown, not exported) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AOP_PROP_REPORT_INTERVAL 0x00 ++#define AOP_PROP_BATCH_INTERVAL 0x16 ++#define AOP_PROP_RAW_INTERVAL 0x5c ++ ++#define AOP_REPORT_PRESSURE 0xc4 ++#define AOP_PRESSURE_LEN 10 ++#define AOP_PRESSURE_VALUE 2 ++ ++#define AOP_PRESSURE_DEFAULT_US 100000 /* 10 Hz */ ++#define AOP_PRESSURE_MIN_US 10000 /* 100 Hz */ ++#define AOP_PRESSURE_MAX_US 1000000 /* 1 Hz */ ++ ++struct apple_aop_pressure { ++ struct device *dev; ++ const struct apple_aop_service *service; ++ ++ struct mutex lock; /* interval */ ++ u32 interval_us; ++ ++ spinlock_t sample_lock; /* sample, have_sample */ ++ u32 sample; ++ bool have_sample; ++ ++ IIO_DECLARE_BUFFER_WITH_TS(u32, scan, 1); ++}; ++ ++static int apple_aop_pressure_set_u32(struct apple_aop_pressure *pressure, u32 key, u32 value) ++{ ++ __le32 le = cpu_to_le32(value); ++ ++ return apple_aop_set_property(pressure->service, key, &le, sizeof(le)); ++} ++ ++/* No separate raw sample stream: raw interval 0. */ ++static int apple_aop_pressure_set_interval(struct apple_aop_pressure *pressure, u32 interval_us) ++{ ++ int ret; ++ ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_BATCH_INTERVAL, interval_us); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_RAW_INTERVAL, 0); ++ if (!ret) ++ ret = apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, interval_us); ++ if (!ret) ++ pressure->interval_us = interval_us; ++ return ret; ++} ++ ++static void apple_aop_pressure_report(void *cookie, u16 subtype, const void *data, size_t len) ++{ ++ struct iio_dev *indio_dev = cookie; ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ ++ if (subtype != AOP_REPORT_PRESSURE || len < AOP_PRESSURE_LEN) ++ return; ++ ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ pressure->sample = get_unaligned_le32(data + AOP_PRESSURE_VALUE); ++ pressure->have_sample = true; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ ++ if (iio_buffer_enabled(indio_dev)) { ++ pressure->scan[0] = pressure->sample; ++ iio_push_to_buffers_with_ts(indio_dev, pressure->scan, sizeof(pressure->scan), ++ iio_get_time_ns(indio_dev)); ++ } ++} ++ ++static int apple_aop_pressure_read_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int *val, int *val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ unsigned long flags; ++ int ret; ++ ++ switch (mask) { ++ case IIO_CHAN_INFO_RAW: ++ spin_lock_irqsave(&pressure->sample_lock, flags); ++ *val = pressure->sample; ++ ret = pressure->have_sample ? IIO_VAL_INT : -EAGAIN; ++ spin_unlock_irqrestore(&pressure->sample_lock, flags); ++ return ret; ++ case IIO_CHAN_INFO_SCALE: ++ /* kPa */ ++ *val = 1; ++ *val2 = 16; ++ return IIO_VAL_FRACTIONAL_LOG2; ++ case IIO_CHAN_INFO_SAMP_FREQ: ++ mutex_lock(&pressure->lock); ++ *val = USEC_PER_SEC / pressure->interval_us; ++ *val2 = (u64)(USEC_PER_SEC % pressure->interval_us) * USEC_PER_SEC / ++ pressure->interval_us; ++ mutex_unlock(&pressure->lock); ++ return IIO_VAL_INT_PLUS_MICRO; ++ default: ++ return -EINVAL; ++ } ++} ++ ++static int apple_aop_pressure_write_raw(struct iio_dev *indio_dev, ++ struct iio_chan_spec const *chan, ++ int val, int val2, long mask) ++{ ++ struct apple_aop_pressure *pressure = iio_priv(indio_dev); ++ u64 micro_hz; ++ u32 interval; ++ int ret; ++ ++ if (mask != IIO_CHAN_INFO_SAMP_FREQ) ++ return -EINVAL; ++ if (val < 0 || val2 < 0) ++ return -EINVAL; ++ ++ micro_hz = (u64)val * USEC_PER_SEC + val2; ++ if (!micro_hz) ++ return -EINVAL; ++ interval = clamp_t(u64, div64_u64((u64)USEC_PER_SEC * USEC_PER_SEC, micro_hz), ++ AOP_PRESSURE_MIN_US, AOP_PRESSURE_MAX_US); ++ ++ mutex_lock(&pressure->lock); ++ ret = apple_aop_pressure_set_interval(pressure, interval); ++ mutex_unlock(&pressure->lock); ++ return ret; ++} ++ ++static const struct iio_chan_spec apple_aop_pressure_channels[] = { ++ { ++ .type = IIO_PRESSURE, ++ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), ++ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), ++ .scan_index = 0, ++ .scan_type = { ++ .sign = 'u', ++ .realbits = 32, ++ .storagebits = 32, ++ .endianness = IIO_CPU, ++ }, ++ }, ++ IIO_CHAN_SOFT_TIMESTAMP(1), ++}; ++ ++static const struct iio_info apple_aop_pressure_info = { ++ .read_raw = apple_aop_pressure_read_raw, ++ .write_raw = apple_aop_pressure_write_raw, ++}; ++ ++/* The offset or the acceleration sensitivity makes it complete */ ++static const struct apple_aop_calibration apple_aop_pressure_calibration[] = { ++ { "apple,pressure-offset-calibration", 0x0b, 1 }, ++ { "apple,pressure-accel-sensitivity", 0xd3, 1 }, ++ { "apple,pressure-heater-resistance", 0xd4, 0 }, ++}; ++ ++static void apple_aop_pressure_stop(void *data) ++{ ++ struct apple_aop_pressure *pressure = data; ++ ++ apple_aop_pressure_set_u32(pressure, AOP_PROP_REPORT_INTERVAL, 0); ++ apple_aop_unlisten(pressure->service); ++} ++ ++static int apple_aop_pressure_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ const struct apple_aop_service *service = dev_get_platdata(dev); ++ struct apple_aop_pressure *pressure; ++ struct iio_dev *indio_dev; ++ int ret; ++ ++ if (!service) ++ return -ENODEV; ++ ++ indio_dev = devm_iio_device_alloc(dev, sizeof(*pressure)); ++ if (!indio_dev) ++ return -ENOMEM; ++ pressure = iio_priv(indio_dev); ++ pressure->dev = dev; ++ pressure->service = service; ++ mutex_init(&pressure->lock); ++ spin_lock_init(&pressure->sample_lock); ++ ++ indio_dev->name = "apple-aop-pressure"; ++ indio_dev->info = &apple_aop_pressure_info; ++ indio_dev->channels = apple_aop_pressure_channels; ++ indio_dev->num_channels = ARRAY_SIZE(apple_aop_pressure_channels); ++ indio_dev->modes = INDIO_DIRECT_MODE; ++ ++ ret = devm_iio_kfifo_buffer_setup(dev, indio_dev, NULL); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_listen(service, apple_aop_pressure_report, indio_dev); ++ if (ret) ++ return ret; ++ ++ ret = apple_aop_calibrate(service, dev, apple_aop_pressure_calibration, ++ ARRAY_SIZE(apple_aop_pressure_calibration)); ++ if (!ret) ++ ret = apple_aop_pressure_set_interval(pressure, AOP_PRESSURE_DEFAULT_US); ++ if (ret) { ++ apple_aop_unlisten(service); ++ return dev_err_probe(dev, ret, "failed to start reports\n"); ++ } ++ ++ ret = devm_add_action_or_reset(dev, apple_aop_pressure_stop, pressure); ++ if (ret) ++ return ret; ++ ++ return devm_iio_device_register(dev, indio_dev); ++} ++ ++static const struct of_device_id apple_aop_pressure_of_match[] = { ++ { .compatible = "apple,s8000-aop-pressure" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_aop_pressure_of_match); ++ ++static struct platform_driver apple_aop_pressure_driver = { ++ .driver = { ++ .name = "apple-aop-pressure", ++ .of_match_table = apple_aop_pressure_of_match, ++ }, ++ .probe = apple_aop_pressure_probe, ++}; ++module_platform_driver(apple_aop_pressure_driver); ++ ++MODULE_AUTHOR("Berkay Kösebay "); ++MODULE_DESCRIPTION("Apple AOP barometric pressure sensor driver"); ++MODULE_LICENSE("Dual MIT/GPL"); diff --git a/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch b/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch new file mode 100644 index 00000000..88cd9ce1 --- /dev/null +++ b/kernel/patches/drivers/input/0030-Input-apple-z2-stop-the-touch-controller-on-shutdown.patch @@ -0,0 +1,48 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] Input: apple-z2: stop the touch controller on shutdown + +A kexec hands the next kernel an S8000 SPI bus that the touch controller +may still be using: its interrupt thread can run and the SPI core can +still hold its messages. + +Add a shutdown callback that stops the controller and waits for its +interrupt, also when the threaded handler has already masked itself. SPI +children shut down before their controller, so the SPI driver can then +drain its queue. + +Assisted-by: LLM +--- + drivers/input/touchscreen/apple_z2.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/drivers/input/touchscreen/apple_z2.c b/drivers/input/touchscreen/apple_z2.c +index 92a04edcd..5dcf3579d 100644 +--- a/drivers/input/touchscreen/apple_z2.c ++++ b/drivers/input/touchscreen/apple_z2.c +@@ -4599,6 +4599,16 @@ static struct spi_device_id apple_z2_of_id[] = { + }; + MODULE_DEVICE_TABLE(spi, apple_z2_of_id); + ++/* Children stop before the SPI controller drains its core queue. */ ++static void apple_z2_shutdown(struct spi_device *spi) ++{ ++ struct apple_z2 *z2 = spi_get_drvdata(spi); ++ ++ apple_z2_stop(z2); ++ /* Also wait when the IRQ thread already masked itself. */ ++ synchronize_irq(spi->irq); ++} ++ + static struct spi_driver apple_z2_driver = { + .driver = { + .name = "apple-z2", +@@ -4609,6 +4619,7 @@ static struct spi_driver apple_z2_driver = { + .id_table = apple_z2_of_id, + .probe = apple_z2_probe, + .remove = apple_z2_remove, ++ .shutdown = apple_z2_shutdown, + }; + + module_spi_driver(apple_z2_driver); diff --git a/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch b/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch new file mode 100644 index 00000000..e442e50e --- /dev/null +++ b/kernel/patches/drivers/iommu/0011-dt-bindings-iommu-keep-the-certified-ANS-kexec-hando.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: iommu: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/iommu/apple,dart.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml +index df72722fc..adcbd21fc 100644 +--- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml ++++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml +@@ -21,6 +21,10 @@ description: |+ + fault conditions. + + properties: ++ apple,ans-dart-handoff: ++ type: boolean ++ description: Allow the storage endpoint to retire and rebuild this DART stream during kexec. ++ + compatible: + oneOf: + - enum: diff --git a/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch b/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch new file mode 100644 index 00000000..e5204e73 --- /dev/null +++ b/kernel/patches/drivers/iommu/0012-iommu-apple-dart-keep-the-certified-ANS-kexec-handof.patch @@ -0,0 +1,56 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] iommu/apple-dart: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + drivers/iommu/apple-dart.c | 24 ++++++++++++++++++++++++ + 1 file changed, 24 insertions(+) + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 1ee6add55..5aff4bc3e 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -35,6 +35,7 @@ + #include + + #include ++#include + + #include "dma-iommu.h" + +@@ -1009,6 +1010,29 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++int apple_ans_dart_stop(struct device *dev) ++{ ++ struct iommu_domain *domain = iommu_get_domain_for_dev(dev); ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ int i; ++ ++ if (!domain || !cfg || ++ !cfg->stream_maps[0].dart || ++ !of_property_read_bool(cfg->stream_maps[0].dart->dev->of_node, ++ "apple,ans-dart-handoff") || ++ (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) ++ return -ENODEV; ++ for_each_stream_map(i, cfg, map) { ++ if (map->dart->hw->type != DART_S5L8960X) ++ return -EOPNOTSUPP; ++ apple_dart_hw_disable_dma(map); ++ } ++ dev_info(dev, "Apple ANS: DART stopped for handoff\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_stop); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { diff --git a/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch b/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch new file mode 100644 index 00000000..711841f3 --- /dev/null +++ b/kernel/patches/drivers/iommu/0013-iommu-apple-dart-save-and-rebuild-translations-aroun.patch @@ -0,0 +1,97 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] iommu/apple-dart: save and rebuild translations around + certified ANS stop + +apple_ans_dart_stop() stops the ANS DART before a kexec handoff. System +sleep needs the same stop and its reverse. + +Save the DART's translation state when apple_ans_dart_stop() stops it, +and add apple_ans_dart_start() to restore it. The regular suspend +callback skips a DART stopped this way, which would otherwise save its +state a second time, and resume clears the flag. The NVMe sleep path +uses the pair to stop the ANS translations before sleep and rebuild them +on resume. + +Assisted-by: LLM +--- + drivers/iommu/apple-dart.c | 29 +++++++++++++++++++++++++++++ + include/linux/apple-ans.h | 2 ++ + 2 files changed, 31 insertions(+) + +diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c +index 5aff4bc3e..5c28d4e4b 100644 +--- a/drivers/iommu/apple-dart.c ++++ b/drivers/iommu/apple-dart.c +@@ -217,6 +217,7 @@ enum dart_type { + * @iommu: iommu core device + */ + struct apple_dart { ++ bool ans_stopped; + struct device *dev; + const struct apple_dart_hw *hw; + +@@ -1010,6 +1011,9 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, + return dart->hw->invalidate_tlb(stream_map); + } + ++static __maybe_unused int apple_dart_suspend(struct device *dev); ++static __maybe_unused int apple_dart_resume(struct device *dev); ++ + int apple_ans_dart_stop(struct device *dev) + { + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); +@@ -1026,6 +1030,8 @@ int apple_ans_dart_stop(struct device *dev) + for_each_stream_map(i, cfg, map) { + if (map->dart->hw->type != DART_S5L8960X) + return -EOPNOTSUPP; ++ apple_dart_suspend(map->dart->dev); ++ map->dart->ans_stopped = true; + apple_dart_hw_disable_dma(map); + } + dev_info(dev, "Apple ANS: DART stopped for handoff\n"); +@@ -1033,6 +1039,25 @@ int apple_ans_dart_stop(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_ans_dart_stop); + ++int apple_ans_dart_start(struct device *dev) ++{ ++ struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); ++ struct apple_dart_stream_map *map; ++ int i, ret; ++ ++ if (!cfg) ++ return -ENODEV; ++ for_each_stream_map(i, cfg, map) { ++ if (!map->dart->ans_stopped) ++ continue; ++ ret = apple_dart_resume(map->dart->dev); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_dart_start); ++ + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, + struct apple_dart_master_cfg *cfg) + { +@@ -2017,6 +2042,9 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + ++ if (dart->ans_stopped) ++ return 0; ++ + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = dart->hw->read_tcr(dart, sid); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) +@@ -2046,6 +2074,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) + dart->hw->write_tcr(dart, sid, dart->save_tcr[sid]); + } + ++ dart->ans_stopped = false; + return 0; + } + diff --git a/kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch b/kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch new file mode 100644 index 00000000..f98a0dbf --- /dev/null +++ b/kernel/patches/drivers/mailbox/0001-dt-bindings-mailbox-select-certified-AOP-reload-thro.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: mailbox: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. +Describe the compatible and retained-reload property. + +Assisted-by: LLM +--- + .../devicetree/bindings/mailbox/apple,mailbox.yaml | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +index ba035ff80..8821623d3 100644 +--- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml ++++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +@@ -22,6 +22,12 @@ description: + properties: + compatible: + oneOf: ++ - description: ++ The same A7 IOP mailbox on the A9. ++ items: ++ - const: apple,s8000-a7iop-mailbox ++ - const: apple,t8010-a7iop-mailbox ++ + - description: + ASC mailboxes are the most common variant found on the M1 used + for example for the display controller, the system management diff --git a/kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch b/kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch new file mode 100644 index 00000000..15940583 --- /dev/null +++ b/kernel/patches/drivers/media/0006-media-apple-vxd-stop-the-A9-firmware-for-system-slee.patch @@ -0,0 +1,330 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 17:23:32 +0000 +Subject: [PATCH] media: apple-vxd: stop the A9 firmware for system sleep + +The DEVA firmware boots on the first open of the video node and stays +resident until the driver is unbound. udev and media daemons open the +node at boot, so the firmware is resident in practice. Over s2idle the +decoder power domain goes off and the resident firmware does not survive +it, and the J71t sleep precheck refuses any bound decoder without a +reserved sleep admission. + +Follow AppleD5500 instead, which stops the core and frees the firmware +when its last client closes (terminateDeviceInternal, fffffff00628a604: +clear the ready magic, write 1 to 0x200, known state v1, free, power +down). Register a sleep admission provider for the A9 variant. Its +prepare does no MMIO: it refuses while a file is open and otherwise +closes the open gate until complete, after resume or any abort. The +system suspend callback then stops the firmware in that order and frees +what it used; the next open boots it again, as the first open after +probe does. If the core does not reach its known state, every buffer it +may still read is kept and the sleep is refused. + +The T8010 variant registers no provider and keeps its behaviour. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 63 ++++++++++++++- + .../media/platform/apple/vxd/apple-vxd-fw.c | 77 +++++++++++++++++++ + .../media/platform/apple/vxd/apple-vxd-regs.h | 6 ++ + .../media/platform/apple/vxd/apple-vxd-v4l2.c | 9 ++- + drivers/media/platform/apple/vxd/apple-vxd.h | 12 +++ + 5 files changed, 164 insertions(+), 3 deletions(-) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index ab7515fee..31a024233 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -245,7 +245,51 @@ static int apple_vxd_runtime_suspend(struct device *dev) + return 0; + } + ++/* ++ * Sleep admission: no MMIO. Refuse while a file is open (its contexts and ++ * any queued pictures belong to the running firmware); otherwise close the ++ * open gate until complete, after resume or any abort. ++ */ ++static int apple_vxd_sleep_prepare(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ int ret = 0; ++ ++ mutex_lock(&vxd->power_lock); ++ if (vxd->users) ++ ret = -EBUSY; ++ else ++ vxd->sleep_gate = true; ++ mutex_unlock(&vxd->power_lock); ++ if (ret) ++ dev_info(dev, "system sleep refused: decoder in use\n"); ++ return ret; ++} ++ ++static void apple_vxd_sleep_complete(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ ++ mutex_lock(&vxd->power_lock); ++ vxd->sleep_gate = false; ++ mutex_unlock(&vxd->power_lock); ++} ++ ++static int apple_vxd_suspend(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ ++ if (!vxd->variant->sleep_stop) ++ return 0; ++ ++ /* No open file: nothing can queue a picture, let the last one finish. */ ++ flush_work(&vxd->job_done_work); ++ cancel_delayed_work_sync(&vxd->watchdog_work); ++ return apple_vxd_fw_sleep_stop(vxd); ++} ++ + static const struct dev_pm_ops apple_vxd_pm_ops = { ++ .suspend = pm_sleep_ptr(apple_vxd_suspend), + RUNTIME_PM_OPS(apple_vxd_runtime_suspend, apple_vxd_runtime_resume, NULL) + }; + +@@ -254,6 +298,7 @@ static const struct apple_vxd_variant s8000_vxd_variant = { + .num_gart_pts = 512, /* "setting 2GB mapping" [E] */ + /* HEVCSupported = ((rev - 7) < 4) && ((0xD >> (rev - 7)) & 1) [E] */ + .hevc_supported = true, ++ .sleep_stop = true, + }; + + /* The T8010 core has two pixel pipes; it reports them in CORE_ID. */ +@@ -353,14 +398,27 @@ static int apple_vxd_probe(struct platform_device *pdev) + if (ret) + goto err_disable_runtime_pm; + ++ /* Before the video node exists: nothing can open it unadmitted. */ ++ if (variant->sleep_stop) { ++ vxd->sleep_admission.dev = &pdev->dev; ++ vxd->sleep_admission.prepare = apple_vxd_sleep_prepare; ++ vxd->sleep_admission.complete = apple_vxd_sleep_complete; ++ ret = register_pm_suspend_admission(&vxd->sleep_admission); ++ if (ret) ++ goto err_gart_fini; ++ } ++ + ret = apple_vxd_h264_register(vxd); + if (!ret) + ret = apple_vxd_v4l2_init(vxd); + if (ret) +- goto err_gart_fini; ++ goto err_admission; + + return 0; + ++err_admission: ++ if (variant->sleep_stop) ++ unregister_pm_suspend_admission(&vxd->sleep_admission); + err_gart_fini: + apple_vxd_gart_fini(vxd); + err_disable_runtime_pm: +@@ -373,6 +431,9 @@ static void apple_vxd_remove(struct platform_device *pdev) + { + struct apple_vxd_dev *vxd = platform_get_drvdata(pdev); + ++ /* Before any teardown; also waits out a transition in progress. */ ++ if (vxd->variant->sleep_stop) ++ unregister_pm_suspend_admission(&vxd->sleep_admission); + cancel_delayed_work_sync(&vxd->watchdog_work); + apple_vxd_v4l2_cleanup(vxd); + apple_vxd_fw_cleanup(vxd); +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-fw.c b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +index 753ca4944..ede737efd 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-fw.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +@@ -435,6 +435,83 @@ int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd) + return ret; + } + ++/* ++ * One reference per open file. A sleep that has been admitted closes the ++ * gate first, so an open between admission and the device's suspend ++ * callback fails instead of booting the firmware behind it. ++ */ ++int apple_vxd_fw_open(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ if (vxd->sleep_gate) { ++ mutex_unlock(&vxd->power_lock); ++ return -EBUSY; ++ } ++ vxd->users++; ++ mutex_unlock(&vxd->power_lock); ++ ++ ret = apple_vxd_boot_firmware(vxd); ++ if (ret) ++ apple_vxd_fw_close(vxd); ++ return ret; ++} ++ ++void apple_vxd_fw_close(struct apple_vxd_dev *vxd) ++{ ++ mutex_lock(&vxd->power_lock); ++ if (!WARN_ON(!vxd->users)) ++ vxd->users--; ++ mutex_unlock(&vxd->power_lock); ++} ++ ++/* ++ * System sleep, after admission and with no open file: stop the MTX in ++ * AppleD5500's terminate order (clear the ready magic, the 0x200 write, ++ * known state v1), then free what the running firmware used. The next ++ * open boots it again, as the first open after probe does. If the core ++ * does not reach its known state, keep every buffer it may still read ++ * and refuse the sleep; the decoder then stays failed (-EIO) until rebind. ++ */ ++int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ if (!vxd->sleep_gate || vxd->users) { ++ mutex_unlock(&vxd->power_lock); ++ return -EBUSY; ++ } ++ if (!vxd->fw_loaded) { ++ mutex_unlock(&vxd->power_lock); ++ return 0; ++ } ++ ++ apple_vxd_mbox_stop(vxd); ++ vxd->hw_ready = false; ++ vxd_write(vxd, VXD_REG_FW_READY_MAGIC, 0); ++ vxd_write(vxd, VXD_REG_VECT_CFG, VXD_VECT_CFG_TERMINATE); ++ ret = vxd_known_state_v1(vxd, "Terminate"); ++ if (ret) { ++ dev_err(vxd->dev, "firmware stop for sleep failed: %d\n", ret); ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++ } ++ ++ apple_vxd_cmdbufs_fini(vxd); ++ apple_vxd_gart_unmap(vxd, &vxd->fw_map); ++ dma_free_coherent(vxd->dev, vxd->fw_size, vxd->fw_vaddr, vxd->fw_dma); ++ vxd->fw_vaddr = NULL; ++ vxd->fw_loaded = false; ++ mutex_unlock(&vxd->power_lock); ++ ++ /* The reference boot took; the PM core holds its own until complete. */ ++ pm_runtime_put_noidle(vxd->dev); ++ dev_info(vxd->dev, "firmware stopped for system sleep\n"); ++ return 0; ++} ++ + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd) + { + bool was_loaded; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-regs.h b/drivers/media/platform/apple/vxd/apple-vxd-regs.h +index edeccb8e0..fdbe48903 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-regs.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd-regs.h +@@ -89,6 +89,12 @@ + + /* VEC block and entropy decoder pipes */ + #define VXD_REG_VECT_CFG 0x0200 ++/* ++ * AppleD5500's terminate path (fffffff00628c474 -> fffffff00629972c [E]) ++ * writes 1 here after clearing the ready magic and before known state v1. ++ * Nothing beyond that order is known about the write. ++ */ ++#define VXD_VECT_CFG_TERMINATE 0x00000001 + #define VXD_REG_VECT_CFG2 0x0208 + #define VXD_VECT_CFG2_MSGS_MINUS1 0x000000c7 /* core[0xb4] - 1 */ + #define VXD_REG_ENTDEC_0_SIZE 0x0340 +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +index 23c051f1d..e9467232d 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +@@ -885,13 +885,15 @@ static int apple_vxd_open(struct file *filp) + * can be read, and keeps running (and the block powered) until the + * driver is unbound. Later opens find it running. + */ +- ret = apple_vxd_boot_firmware(vxd); ++ ret = apple_vxd_fw_open(vxd); + if (ret) + return ret; + + ctx = kzalloc_obj(*ctx); +- if (!ctx) ++ if (!ctx) { ++ apple_vxd_fw_close(vxd); + return -ENOMEM; ++ } + + ctx->vxd = vxd; + v4l2_fh_init(&ctx->fh, vdev); +@@ -900,6 +902,7 @@ static int apple_vxd_open(struct file *filp) + if (ret) { + v4l2_fh_exit(&ctx->fh); + kfree(ctx); ++ apple_vxd_fw_close(vxd); + return ret; + } + +@@ -910,11 +913,13 @@ static int apple_vxd_open(struct file *filp) + static int apple_vxd_release(struct file *filp) + { + struct apple_vxd_ctx *ctx = file_to_vxd_ctx(filp); ++ struct apple_vxd_dev *vxd = ctx->vxd; + + v4l2_fh_del(&ctx->fh, filp); + apple_vxd_ctx_fini(ctx); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); ++ apple_vxd_fw_close(vxd); + + return 0; + } +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index bba04f75a..81fef0f1a 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -66,6 +67,11 @@ struct apple_vxd_variant { + u32 num_gart_pts; /* mapped page tables (512 = 2 GiB) */ + bool hevc_supported; + bool has_integration; /* second reg range, see VXD_INTEG_CTRL */ ++ /* ++ * Stop the firmware for system sleep the way AppleD5500 does when its ++ * last client closes, and boot it again on the next open. ++ */ ++ bool sleep_stop; + }; + + /* +@@ -284,6 +290,9 @@ struct apple_vxd_dev { + /* Core state. */ + struct mutex power_lock; /* boot vs. runtime suspend */ + bool hw_ready; /* firmware took its init message */ ++ unsigned int users; /* open files, under power_lock */ ++ bool sleep_gate; /* sleep admitted: no new open */ ++ struct pm_suspend_admission sleep_admission; + u32 core_id; + u32 core_rev; + u32 num_entropy_pipes; +@@ -448,6 +457,9 @@ void apple_vxd_dev_buf_free(struct apple_vxd_dev *vxd, + + /* fw.c */ + int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_open(struct apple_vxd_dev *vxd); ++void apple_vxd_fw_close(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd); + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd); + + /* mbox.c */ diff --git a/kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch b/kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch new file mode 100644 index 00000000..fdae4cdb --- /dev/null +++ b/kernel/patches/drivers/media/0007-media-apple-vxd-answer-the-suspend-readiness-query.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 18:11:36 +0000 +Subject: [PATCH] media: apple-vxd: answer the suspend readiness query + +The A9 decoder's sleep admission refuses while a file is open and +otherwise closes its open gate until complete. Provide the matching +check: report -EBUSY while a file is open, without touching the gate. + +Assisted-by: LLM +--- + drivers/media/platform/apple/vxd/apple-vxd-core.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index 31a024233..dff75927d 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -266,6 +266,18 @@ static int apple_vxd_sleep_prepare(struct device *dev) + return ret; + } + ++/* Readiness query: what prepare would decide, without closing the gate. */ ++static int apple_vxd_sleep_check(struct device *dev) ++{ ++ struct apple_vxd_dev *vxd = dev_get_drvdata(dev); ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ ret = vxd->users ? -EBUSY : 0; ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++} ++ + static void apple_vxd_sleep_complete(struct device *dev) + { + struct apple_vxd_dev *vxd = dev_get_drvdata(dev); +@@ -403,6 +415,7 @@ static int apple_vxd_probe(struct platform_device *pdev) + vxd->sleep_admission.dev = &pdev->dev; + vxd->sleep_admission.prepare = apple_vxd_sleep_prepare; + vxd->sleep_admission.complete = apple_vxd_sleep_complete; ++ vxd->sleep_admission.check = apple_vxd_sleep_check; + ret = register_pm_suspend_admission(&vxd->sleep_admission); + if (ret) + goto err_gart_fini; diff --git a/kernel/patches/drivers/media/0008-media-apple-vxd-Prepare-for-a-second-coded-format.patch b/kernel/patches/drivers/media/0008-media-apple-vxd-Prepare-for-a-second-coded-format.patch new file mode 100644 index 00000000..98193b10 --- /dev/null +++ b/kernel/patches/drivers/media/0008-media-apple-vxd-Prepare-for-a-second-coded-format.patch @@ -0,0 +1,252 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 21:24:51 +0100 +Subject: [PATCH] media: apple-vxd: Prepare for a second coded format + +The coded-format table was the H.264 unit's own array, installed whole +by apple_vxd_h264_register(). Make it a table of pointers in the device +that each codec unit appends to with apple_vxd_register_coded_fmt(), so +a second codec can sit next to H.264 without the core knowing about it. +The first unit registered stays the default OUTPUT format. + +Add apple_vxd_buf_aux_cpu() and apple_vxd_buf_aux_size() next to +apple_vxd_buf_aux_va(): a codec that keeps per-picture state in the +CAPTURE side buffer needs to read and write it. + +apple_vxd_cmdbuf_pair_chunk_end() had no user. Make it match the HEVC +per-picture chunk it was written for: AppleD5500 (0xfffffff0062b7ab0) +zeroes 0x40 - (pairs & 7) * 8 bytes past the write pointer and neither +counts them in the packet's byte count nor advances over them. + +H.264 output is unchanged: its host command-stream compare still matches +the reference generator on all 76 pictures. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-cmd.c | 25 +++++------ + .../media/platform/apple/vxd/apple-vxd-h264.c | 10 +---- + .../media/platform/apple/vxd/apple-vxd-v4l2.c | 42 +++++++++++++++---- + drivers/media/platform/apple/vxd/apple-vxd.h | 17 ++++++-- + 4 files changed, 62 insertions(+), 32 deletions(-) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-cmd.c b/drivers/media/platform/apple/vxd/apple-vxd-cmd.c +index 03340ed33..fa93cd965 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-cmd.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-cmd.c +@@ -318,29 +318,30 @@ int apple_vxd_cmdbuf_chunk_end_exact(struct apple_vxd_cmdbuf *cb, + } + + /* +- * Close a chunk made only of 0x000f0000 pairs (the per-picture kind-4 +- * chunk, 0x62b7ab0): pad by 0x40 - (pairs & 7) * 8 bytes, a full 64 when +- * the pair count is a multiple of eight, and report the byte count +- * including that pad (the kext measures it after the pad; the message +- * sends it as a pair count, >> 3). ++ * Close a chunk made only of 0x000f0000 pairs (the HEVC per-picture kind-4 ++ * chunk, 0x62b7ab0): report its byte count (the message sends it as a pair ++ * count, >> 3), then zero 0x40 - (pairs & 7) * 8 bytes past it, a full 64 ++ * when the pair count is a multiple of eight. The kext writes that pad ++ * past its write pointer and neither counts it nor advances over it, so ++ * this must be the buffer's last chunk. + */ + int apple_vxd_cmdbuf_pair_chunk_end(struct apple_vxd_cmdbuf *cb, u32 *offset, + u32 *bytes) + { + size_t pad; +- int ret; + + if (cb->group_open || cb->used - cb->chunk_start != cb->npairs * 8) + cb->error = true; +- if (cb->error) ++ pad = VXD_CMD_CHUNK_ALIGN - (cb->npairs & 7) * 8; ++ if (cb->error || pad > cb->size - cb->used) { ++ cb->error = true; + return -ENOSPC; ++ } + + *offset = cb->chunk_start; +- pad = VXD_CMD_CHUNK_ALIGN - (cb->npairs & 7) * 8; +- ret = vxd_cmd_pad(cb, pad); +- if (!ret) +- *bytes = cb->used - *offset; +- return ret; ++ *bytes = cb->used - cb->chunk_start; ++ memset((u8 *)cb->vaddr + cb->used, 0, pad); ++ return 0; + } + + /* +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264.c b/drivers/media/platform/apple/vxd/apple-vxd-h264.c +index 726df7525..ca289e11d 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-h264.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-h264.c +@@ -557,14 +557,8 @@ static const struct apple_vxd_coded_fmt_desc apple_vxd_h264_fmt = { + .decoded_fmts = apple_vxd_h264_decoded_fmts, + }; + +-static const struct apple_vxd_coded_fmt_desc apple_vxd_coded_fmts[] = { +- apple_vxd_h264_fmt, +-}; +- +-/* Fills the device's coded-format table, before any V4L2 node exists. */ ++/* Adds H.264 to the device's coded formats, before any V4L2 node exists. */ + int apple_vxd_h264_register(struct apple_vxd_dev *vxd) + { +- vxd->coded_fmts = apple_vxd_coded_fmts; +- vxd->num_coded_fmts = ARRAY_SIZE(apple_vxd_coded_fmts); +- return 0; ++ return apple_vxd_register_coded_fmt(vxd, &apple_vxd_h264_fmt); + } +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +index e9467232d..c5e429df7 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-v4l2.c +@@ -97,8 +97,8 @@ apple_vxd_find_coded_fmt_desc(struct apple_vxd_ctx *ctx, u32 fourcc) + unsigned int i; + + for (i = 0; i < vxd->num_coded_fmts; i++) +- if (vxd->coded_fmts[i].fourcc == fourcc) +- return &vxd->coded_fmts[i]; ++ if (vxd->coded_fmts[i]->fourcc == fourcc) ++ return vxd->coded_fmts[i]; + return NULL; + } + +@@ -107,7 +107,7 @@ static void apple_vxd_reset_coded_fmt(struct apple_vxd_ctx *ctx) + struct v4l2_format *f = &ctx->coded_fmt; + + /* apple_vxd_ctx_init() checked vxd->coded_fmts before calling us. */ +- ctx->coded_fmt_desc = &ctx->vxd->coded_fmts[0]; ++ ctx->coded_fmt_desc = ctx->vxd->coded_fmts[0]; + apple_vxd_reset_fmt(ctx, f, ctx->coded_fmt_desc->fourcc); + f->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + f->fmt.pix_mp.width = ctx->coded_fmt_desc->frmsize.min_width; +@@ -208,12 +208,12 @@ static int apple_vxd_init_ctrls(struct apple_vxd_ctx *ctx) + int ret; + + for (i = 0; i < vxd->num_coded_fmts; i++) +- nctrls += vxd->coded_fmts[i].ctrls->num_ctrls; ++ nctrls += vxd->coded_fmts[i]->ctrls->num_ctrls; + + v4l2_ctrl_handler_init(&ctx->ctrl_hdl, nctrls); + + for (i = 0; i < vxd->num_coded_fmts; i++) { +- ret = apple_vxd_add_ctrls(ctx, vxd->coded_fmts[i].ctrls); ++ ret = apple_vxd_add_ctrls(ctx, vxd->coded_fmts[i]->ctrls); + if (ret) + goto err_free_handler; + } +@@ -297,7 +297,7 @@ static int apple_vxd_do_try_output_fmt(struct apple_vxd_ctx *ctx, + + desc = apple_vxd_find_coded_fmt_desc(ctx, pix_mp->pixelformat); + if (!desc) { +- desc = &ctx->vxd->coded_fmts[0]; ++ desc = ctx->vxd->coded_fmts[0]; + pix_mp->pixelformat = desc->fourcc; + } + +@@ -404,7 +404,7 @@ static int apple_vxd_do_enum_output_fmt(struct apple_vxd_ctx *ctx, + { + if (f->index >= ctx->vxd->num_coded_fmts) + return -EINVAL; +- f->pixelformat = ctx->vxd->coded_fmts[f->index].fourcc; ++ f->pixelformat = ctx->vxd->coded_fmts[f->index]->fourcc; + return 0; + } + +@@ -650,6 +650,18 @@ u32 apple_vxd_buf_aux_va(struct vb2_v4l2_buffer *vbuf) + return vxd_buf(vbuf)->aux.va; + } + ++/* CPU address of a CAPTURE buffer's side buffer, NULL if it has none. */ ++void *apple_vxd_buf_aux_cpu(struct vb2_v4l2_buffer *vbuf) ++{ ++ return vxd_buf(vbuf)->aux.va ? vxd_buf(vbuf)->aux.mem.vaddr : NULL; ++} ++ ++/* Size of a CAPTURE buffer's side buffer, 0 if it has none. */ ++size_t apple_vxd_buf_aux_size(struct vb2_v4l2_buffer *vbuf) ++{ ++ return vxd_buf(vbuf)->aux.va ? vxd_buf(vbuf)->aux.mem.size : 0; ++} ++ + static void apple_vxd_buf_queue(struct vb2_buffer *vb) + { + struct apple_vxd_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); +@@ -935,11 +947,25 @@ static const struct v4l2_file_operations apple_vxd_fops = { + + /* --- registration ------------------------------------------------------ */ + ++/* ++ * Add one coded format to the device's table. The codec units call this ++ * during probe, before any V4L2 node exists; the first one added is the ++ * default OUTPUT format. ++ */ ++int apple_vxd_register_coded_fmt(struct apple_vxd_dev *vxd, ++ const struct apple_vxd_coded_fmt_desc *desc) ++{ ++ if (vxd->num_coded_fmts >= ARRAY_SIZE(vxd->coded_fmts)) ++ return -ENOSPC; ++ vxd->coded_fmts[vxd->num_coded_fmts++] = desc; ++ return 0; ++} ++ + int apple_vxd_v4l2_init(struct apple_vxd_dev *vxd) + { + int ret; + +- if (!vxd->coded_fmts || !vxd->num_coded_fmts) { ++ if (!vxd->num_coded_fmts) { + dev_err(vxd->dev, "v4l2: no coded formats registered, not exposing a node\n"); + return -ENODEV; + } +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index 81fef0f1a..f7eab496c 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -10,7 +10,8 @@ + * - host<->firmware mailbox: ring geometry, 0x80 init message, push, + * IRQ drain, fences (apple-vxd-mbox.c) + * - command buffers and their encoders (apple-vxd-cmdbuf.c, -cmd.c) +- * - the V4L2 stateless H.264 decoder (apple-vxd-v4l2.c, -h264*.c) ++ * - the V4L2 stateless node (apple-vxd-v4l2.c) and its codec units: ++ * H.264 (apple-vxd-h264*.c) + * + * [E] marks facts taken from the kext. + */ +@@ -61,6 +62,9 @@ enum apple_vxd_image_fmt { + APPLE_VXD_IMG_FMT_420_8BIT, + }; + ++/* One per codec unit. */ ++#define APPLE_VXD_MAX_CODED_FMTS 2 ++ + /* SoC variant data (compatible table). */ + struct apple_vxd_variant { + unsigned int vxd_version; /* ADT "vxd-version" */ +@@ -351,10 +355,11 @@ struct apple_vxd_dev { + struct v4l2_m2m_dev *m2m_dev; + struct mutex vdev_lock; /* serializes ioctls */ + /* +- * Coded formats, filled by apple_vxd_h264_register() during probe +- * before any V4L2 node exists. ++ * Coded formats, one per codec unit, added with ++ * apple_vxd_register_coded_fmt() during probe before any V4L2 node ++ * exists. + */ +- const struct apple_vxd_coded_fmt_desc *coded_fmts; ++ const struct apple_vxd_coded_fmt_desc *coded_fmts[APPLE_VXD_MAX_CODED_FMTS]; + unsigned int num_coded_fmts; + }; + +@@ -495,6 +500,10 @@ int apple_vxd_packet_msg(struct apple_vxd_dev *vxd, u32 *pkt, u16 fence, + /* v4l2.c */ + u32 apple_vxd_buf_va(struct vb2_v4l2_buffer *vbuf); + u32 apple_vxd_buf_aux_va(struct vb2_v4l2_buffer *vbuf); ++void *apple_vxd_buf_aux_cpu(struct vb2_v4l2_buffer *vbuf); ++size_t apple_vxd_buf_aux_size(struct vb2_v4l2_buffer *vbuf); ++int apple_vxd_register_coded_fmt(struct apple_vxd_dev *vxd, ++ const struct apple_vxd_coded_fmt_desc *desc); + int apple_vxd_v4l2_init(struct apple_vxd_dev *vxd); + void apple_vxd_v4l2_cleanup(struct apple_vxd_dev *vxd); + extern const struct v4l2_ctrl_ops apple_vxd_ctrl_ops; diff --git a/kernel/patches/drivers/media/0009-media-apple-vxd-Add-the-HEVC-command-stream-builder.patch b/kernel/patches/drivers/media/0009-media-apple-vxd-Add-the-HEVC-command-stream-builder.patch new file mode 100644 index 00000000..50028532 --- /dev/null +++ b/kernel/patches/drivers/media/0009-media-apple-vxd-Add-the-HEVC-command-stream-builder.patch @@ -0,0 +1,1116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 21:25:14 +0100 +Subject: [PATCH] media: apple-vxd: Add the HEVC command-stream builder + +AppleD5500 decodes HEVC with a register program of its own, distinct +from the AVC one: per slice segment a chunk with the 0x90000000 +descriptor, the whole slice NAL by DMA ({0xb0000001, header bits, bytes} +and {0xa.., address}), the 0x30xx/0x34xx command registers and a slice +structure the core copies into the picture's slice parameter buffer +(0xf), each with a kind-1 packet (mailbox 0x81); then one chunk of +single register writes for the picture, the 0x50xx/0x54xx entropy and +0x70xx/0x71xx render registers with the output and reference surfaces +and the scaling lists, with a kind-4 packet (mailbox 0x88) that ends the +job. The layouts come from the iOS 16.7.10 kext (picture path +0xfffffff0062960a0, per slice 0x62b37d4, per picture 0x62b7ab0) and are +listed register by register in the comments. + +This builds that program from the V4L2 HEVC controls. It is pure code, +like the H.264 builder, so it also builds on the host: a harness feeds +it x265 and VideoToolbox test streams and compares every command word, +slice structure and packet against an independent generator written from +the kext. + +Supported: 8-bit 4:2:0, I/P/B, multiple and dependent slice segments, +tiles, WPP, long-term references, weighted prediction, PCM and scaling +lists. Refused: other chroma formats and bit depths (the Main 10 output +layout is not established), more than 15 references. + +The 15 hardware reference slots are named by the V4L2 DPB index; the +kext numbers them in its own DPB order, which never leaves the host. + +Nothing calls the builder yet. + +Assisted-by: LLM +--- + drivers/media/platform/apple/vxd/Makefile | 2 +- + .../platform/apple/vxd/apple-vxd-hevc-cmd.c | 940 ++++++++++++++++++ + .../platform/apple/vxd/apple-vxd-hevc-cmd.h | 109 ++ + 3 files changed, 1050 insertions(+), 1 deletion(-) + create mode 100644 drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.c + create mode 100644 drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.h + +diff --git a/drivers/media/platform/apple/vxd/Makefile b/drivers/media/platform/apple/vxd/Makefile +index 7f692f8bf..0877a60ae 100644 +--- a/drivers/media/platform/apple/vxd/Makefile ++++ b/drivers/media/platform/apple/vxd/Makefile +@@ -2,6 +2,6 @@ + apple-vxd-y := apple-vxd-core.o apple-vxd-gart.o apple-vxd-fw.o \ + apple-vxd-mbox.o apple-vxd-cmd.o apple-vxd-cmdbuf.o \ + apple-vxd-v4l2.o apple-vxd-h264.o apple-vxd-h264-parse.o \ +- apple-vxd-h264-cmd.o ++ apple-vxd-h264-cmd.o apple-vxd-hevc-cmd.o + + obj-$(CONFIG_VIDEO_APPLE_VXD) += apple-vxd.o +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.c b/drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.c +new file mode 100644 +index 000000000..00c9067cd +--- /dev/null ++++ b/drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.c +@@ -0,0 +1,940 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Apple VXD - HEVC (H.265) command-stream builder ++ * ++ * The HEVC path of the kext (picture path 0xfffffff0062960a0; per slice ++ * 0x62b37d4, per picture 0x62b7ab0; docs/periph/vdec-hevc-20261110.md in ++ * the hardware bring-up records has the word-level map). Per picture: ++ * ++ * per slice segment, one chunk and one kind-1 packet (mailbox 0x81): ++ * 0x90000000 + 9 words slice descriptor ++ * 0xb + 0xa the whole slice NAL from its header byte; ++ * the core skips the header (RBSP bits) ++ * 0x1 register runs CMD block 0x30xx / 0x34xx ++ * 0x3090 / 0x3094 slice structure destination / lines, ++ * written after the structure is built ++ * 0xf slice structure transfer (independent ++ * segments): built in a CPU buffer, copied ++ * by the core into the slice parameter buffer ++ * 0x60000000 ++ * then zero padding to the next 64 bytes, not counted ++ * then one chunk of single register writes (0x000f0000 form) and one ++ * kind-4 packet (mailbox 0x88): entropy registers 0x50xx/0x54xx, the ++ * scaling lists 0x5800.., and the render registers 0x70xx/0x71xx with ++ * the output and reference surfaces. The 0x88 message ends the job. ++ * ++ * Reference slots: the hardware has 15, and every list in the stream ++ * (0x7100 surfaces, 0x5438 long-term bits, 0x5440 POC deltas, the slice ++ * structures' reference indices) names them by number. Apple numbers them ++ * in its own DPB order; this driver uses the V4L2 DPB index. ++ * ++ * Supported: 8-bit 4:2:0 (Main, Main Still Picture), I/P/B, multiple ++ * slices and dependent slice segments, tiles, WPP, long-term references, ++ * weighted prediction, PCM, scaling lists. Refused: other chroma formats ++ * and bit depths (the output layout is not established), range extensions, ++ * more than 15 references. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-vxd-hevc-cmd.h" ++ ++/* Slice-level CMD block (MSVDX command registers). */ ++#define HEVC_CMD_SP_DEST 0x3090 /* slice structure address */ ++#define HEVC_CMD_SP_LINES 0x3094 /* its size in 64-byte lines - 1 */ ++#define HEVC_CMD_FRAGMENT 0x3098 ++#define HEVC_CMD_FORMAT 0x30a0 ++#define HEVC_CMD_SIZE 0x30a8 ++#define HEVC_CMD_SLICE_POS 0x30b0 ++#define HEVC_CMD_RESET 0x3080 ++#define HEVC_CMD_RESET_ZERO 0x3120 /* x4 */ ++#define HEVC_CMD_GENC_SIZE 0x3100 /* x4, interleaved with: */ ++#define HEVC_CMD_GENC_BASE 0x3110 ++#define HEVC_CMD_SPS0 0x3400 ++#define HEVC_CMD_SPS1 0x3404 ++#define HEVC_CMD_PPS0 0x3408 ++#define HEVC_CMD_SEGMENT 0x3410 ++#define HEVC_CMD_SLICE0 0x3414 ++#define HEVC_CMD_TILE_COLS 0x3440 /* x12 */ ++#define HEVC_CMD_TILE_ROWS 0x3480 /* x12 */ ++/* Entropy block. */ ++#define HEVC_ENT_CONTROL 0x5000 ++#define HEVC_ENT_SIZE8 0x5010 ++#define HEVC_ENT_FRAGMENT 0x5030 ++#define HEVC_ENT_GENC_SIZE 0x5040 ++#define HEVC_ENT_GENC_BASE 0x5050 ++#define HEVC_ENT_ABOVE 0x50c0 ++#define HEVC_ENT_SPS0 0x5400 /* .. 0x540c */ ++#define HEVC_ENT_COL_MV 0x5430 ++#define HEVC_ENT_CUR_MV 0x5434 ++#define HEVC_ENT_REFS 0x5438 ++#define HEVC_ENT_COL_REFS 0x543c ++#define HEVC_ENT_POC_DELTA 0x5440 /* x4 */ ++#define HEVC_ENT_COL_POC_DELTA 0x5450 /* x4 */ ++#define HEVC_ENT_TILE_COLS 0x5480 /* x12 */ ++#define HEVC_ENT_TILE_ROWS 0x54c0 /* x12 */ ++#define HEVC_ENT_SCALING 0x5800 ++/* Render block. */ ++#define HEVC_REN_DST 0x700c /* Y, UV */ ++#define HEVC_REN_INTRA 0x7018 ++#define HEVC_REN_CACHE 0x701c ++#define HEVC_REN_WEIGHT_DENOM 0x7024 ++#define HEVC_REN_LAYOUT 0x703c ++#define HEVC_REN_STRIDE 0x7040 ++#define HEVC_REN_PIC_CONFIG 0x70a0 ++#define HEVC_REN_CHROMA_QP 0x70a4 ++#define HEVC_REN_SIZE 0x70b0 ++#define HEVC_REN_SIZE8 0x70b4 ++#define HEVC_REN_REFS 0x7100 /* Y, UV per slot */ ++#define HEVC_REN_CSTRIDE 0x71ac ++#define HEVC_REN_FORMAT 0x71b4 ++#define HEVC_REN_AUXLINE 0x71ec ++#define HEVC_REN_WEIGHTS(l) (0x7200 + 0x100 * (l)) ++#define HEVC_REN_OFFSETS(l) (0x7280 + 0x100 * (l)) ++#define HEVC_REN_7474 0x7474 ++ ++#define HEVC_F_ADDR 0x10 /* 0x000f0000 flag on addresses */ ++ ++/* Largest slice structure: B header, 1 + 2 x 2 x 15 weight pairs, 0x7474 x 2. */ ++#define HEVC_STRUCT_MAX (48 + 8 * 63 + 0x38) ++ ++/* The linear-output layout word (0x62b8b0c with tiling off, aux line on). */ ++#define HEVC_LAYOUT_LINEAR 0x400 ++ ++enum { ++ HEVC_B = V4L2_HEVC_SLICE_TYPE_B, ++ HEVC_P = V4L2_HEVC_SLICE_TYPE_P, ++ HEVC_I = V4L2_HEVC_SLICE_TYPE_I, ++}; ++ ++/* The per-slice parameter block (the kext's P[]), V4L2 names. */ ++struct hevc_slice { ++ bool dependent; ++ u32 type; /* B 0, P 1, I 2 */ ++ u32 address; ++ bool sao_luma, sao_chroma, temporal_mvp, mvd_l1_zero, cabac_init; ++ bool col_from_l0, dbk_disabled, lf_across_slices; ++ u32 l0, l1; /* num_ref_idx_lX_active_minus1 */ ++ u32 col_ref_idx; ++ u32 merge_cand; /* MaxNumMergeCand */ ++ int qp; /* SliceQpY */ ++ int cb_qp, cr_qp, beta, tc; ++ u8 slot[2][15]; /* hardware slot of RefPicListX[i], else 0 */ ++ bool weighted; ++}; ++ ++/* Picture-level state carried across the slices of one picture. */ ++struct hevc_pic { ++ const struct apple_vxd_hevc_frame *f; ++ u32 ctb_mb; /* CtbSizeY / 16 */ ++ u32 w_ctb, h_ctb; /* PicWidth/HeightInCtbsY */ ++ u32 w, h; /* luma size rounded to the minimum CB */ ++ u32 qp_bd; /* QpBdOffsetY */ ++ u32 tile_cols[12], tile_rows[12]; ++ u32 seg_x, seg_y; /* last independent segment, in MBs */ ++ u32 min_type; /* minimum slice type so far (C[13c470]) */ ++ int min_type_q; /* over independent segments, -1 none */ ++ u32 cache_div; /* Q+0x84 */ ++ u32 side_used, transfers, dest; ++ u32 prev_dest, prev_lines; ++ int col_slot; /* -1 none */ ++ bool col_invalid; ++ size_t last_desc; /* word index of the last slice descriptor */ ++}; ++ ++#define SPS_F(sps, n) (!!((sps)->flags & V4L2_HEVC_SPS_FLAG_##n)) ++#define PPS_F(pps, n) (!!((pps)->flags & V4L2_HEVC_PPS_FLAG_##n)) ++#define SLICE_F(s, n) (!!((s)->flags & V4L2_HEVC_SLICE_PARAMS_FLAG_##n)) ++ ++u32 apple_vxd_hevc_genc_size(unsigned int width, unsigned int height) ++{ ++ /* (W/16) * 1152.0 * (H/16) * 0.25, page aligned, capped (0x62b18f8) */ ++ u32 v = DIV_ROUND_UP(width, 16) * 288 * DIV_ROUND_UP(height, 16); ++ ++ return min_t(u32, ALIGN(v, 0x1000), 0x6000000); ++} ++ ++static void hevc_aux_sizes(u32 w, u32 h, struct apple_vxd_hevc_aux_layout *l) ++{ ++ /* CHEVCDecoder::AllocatePictureMemory (0x62b87dc) */ ++ u32 mv = (DIV_ROUND_UP(w, 64) * 0x80 + DIV_ROUND_UP(w, 128) * 0x40) * ++ DIV_ROUND_UP(h, 64); ++ u32 frag = DIV_ROUND_UP(h, 64) * 0x600 + 0x3fc0; ++ u32 above = DIV_ROUND_UP(w, 64) * 0xc0; ++ ++ l->mv = 0; ++ l->frag = ALIGN(mv, 64); ++ l->above = ALIGN(l->frag + frag, 64); ++ l->record = ALIGN(l->above + above, 64); ++ l->size = l->record + ALIGN(sizeof(struct apple_vxd_hevc_record), 64); ++} ++ ++static void hevc_coded_size(const struct v4l2_ctrl_hevc_sps *sps, u32 *w, u32 *h) ++{ ++ u32 mcb = sps->log2_min_luma_coding_block_size_minus3 + 3; ++ ++ *w = (sps->pic_width_in_luma_samples >> mcb) << mcb; ++ *h = (sps->pic_height_in_luma_samples >> mcb) << mcb; ++} ++ ++void apple_vxd_hevc_aux_layout(const struct v4l2_ctrl_hevc_sps *sps, ++ struct apple_vxd_hevc_aux_layout *l) ++{ ++ u32 w, h; ++ ++ hevc_coded_size(sps, &w, &h); ++ hevc_aux_sizes(w, h, l); ++} ++ ++/* An upper bound for any stream that fits a @width x @height CAPTURE. */ ++u32 apple_vxd_hevc_aux_size(unsigned int width, unsigned int height) ++{ ++ struct apple_vxd_hevc_aux_layout l; ++ ++ hevc_aux_sizes(width, height, &l); ++ return l.size; ++} ++ ++int apple_vxd_hevc_validate(const struct v4l2_ctrl_hevc_sps *sps, ++ const struct v4l2_ctrl_hevc_pps *pps, ++ const char **why) ++{ ++ u32 ctb_log2 = sps->log2_min_luma_coding_block_size_minus3 + 3 + ++ sps->log2_diff_max_min_luma_coding_block_size; ++ ++ if (sps->chroma_format_idc != 1) ++ *why = "chroma format other than 4:2:0"; ++ else if (sps->bit_depth_luma_minus8 || sps->bit_depth_chroma_minus8) ++ *why = "bit depth above 8 (Main 10 output layout not established)"; ++ else if (SPS_F(sps, SEPARATE_COLOUR_PLANE)) ++ *why = "separate colour planes"; ++ else if (ctb_log2 < 4 || ctb_log2 > 6 || ++ sps->log2_min_luma_coding_block_size_minus3 > 3 || ++ sps->log2_min_luma_transform_block_size_minus2 > 3 || ++ sps->log2_diff_max_min_luma_transform_block_size > 3 || ++ sps->max_transform_hierarchy_depth_inter > 7 || ++ sps->max_transform_hierarchy_depth_intra > 7) ++ *why = "coding or transform block sizes outside the hardware fields"; ++ else if (!sps->pic_width_in_luma_samples || !sps->pic_height_in_luma_samples || ++ sps->pic_width_in_luma_samples > 4096 || ++ sps->pic_height_in_luma_samples > 4096) ++ *why = "picture size outside the hardware fields"; ++ else if (PPS_F(pps, TILES_ENABLED) && ++ (pps->num_tile_columns_minus1 > 19 || pps->num_tile_rows_minus1 > 21)) ++ *why = "tile counts out of range"; ++ else if (pps->diff_cu_qp_delta_depth > 3 || pps->log2_parallel_merge_level_minus2 > 7) ++ *why = "PPS depth fields out of range"; ++ else ++ return 0; ++ return -EINVAL; ++} ++ ++/* --- locating the slices --------------------------------------------- */ ++ ++static bool hevc_is_sc(const u8 *p, size_t i, size_t len) ++{ ++ return i + 3 <= len && !p[i] && !p[i + 1] && p[i + 2] == 1; ++} ++ ++/* ++ * Frame-based Annex-B: the OUTPUT buffer holds the picture's slice NALs, ++ * each behind a start code, in slice order. The i-th slice NAL is the ++ * i-th VCL NAL found. data_byte_offset counts RBSP header bytes, and ++ * clients disagree on its base: FFmpeg and libva-v4l2-request count from ++ * the buffer start, GStreamer from the slice's start code. Either way the ++ * header ends inside this NAL, which tells them apart. ++ */ ++int apple_vxd_hevc_find_nals(const u8 *src, size_t len, ++ const struct v4l2_ctrl_hevc_slice_params *slices, ++ unsigned int nslices, struct apple_vxd_hevc_nal *nals, ++ const char **why) ++{ ++ size_t i = 0, sc, start, end; ++ unsigned int n = 0; ++ ++ while (n < nslices) { ++ u32 hdr; ++ ++ while (i < len && !hevc_is_sc(src, i, len)) ++ i++; ++ if (i >= len) { ++ *why = "fewer slice NALs in the OUTPUT buffer than SLICE_PARAMS"; ++ return -EINVAL; ++ } ++ sc = i; ++ start = i + 3; ++ for (end = start; end < len && !hevc_is_sc(src, end, len); end++) ++ ; ++ i = end; ++ while (end > start && !src[end - 1]) ++ end--; ++ if (end - start < 3 || ((src[start] >> 1) & 0x3f) > 31) ++ continue; /* not a VCL NAL */ ++ ++ if (slices[n].data_byte_offset > start && ++ slices[n].data_byte_offset < end) { ++ hdr = slices[n].data_byte_offset - start; ++ } else if (slices[n].data_byte_offset >= start - sc + 2 && ++ slices[n].data_byte_offset < end - sc) { ++ hdr = slices[n].data_byte_offset - (start - sc); ++ } else { ++ *why = "data_byte_offset outside its slice NAL"; ++ return -EINVAL; ++ } ++ if (hdr < 3) { ++ *why = "slice header shorter than the NAL header"; ++ return -EINVAL; ++ } ++ nals[n].offset = start; ++ nals[n].size = end - start; ++ nals[n].header_bits = 8 * hdr; ++ n++; ++ } ++ return 0; ++} ++ ++/* --- per-picture words (0x62b2bb8 / 0x62b38d0 / 0x62b7a18) ------------ */ ++ ++static u32 hevc_sps0(const struct v4l2_ctrl_hevc_sps *sps) ++{ ++ return sps->max_transform_hierarchy_depth_intra << 28 | ++ sps->max_transform_hierarchy_depth_inter << 24 | ++ (sps->log2_diff_max_min_luma_transform_block_size & 3) << 20 | ++ (sps->log2_min_luma_transform_block_size_minus2 & 3) << 16 | ++ (sps->log2_diff_max_min_luma_coding_block_size & 3) << 12 | ++ (sps->log2_min_luma_coding_block_size_minus3 & 3) << 8 | ++ (sps->bit_depth_chroma_minus8 & 7) << 4 | ++ (sps->bit_depth_luma_minus8 & 7); ++} ++ ++/* PCM fields are zero without PCM, which makes the depths read 1. */ ++static u32 hevc_sps1(const struct v4l2_ctrl_hevc_sps *sps) ++{ ++ bool pcm = SPS_F(sps, PCM_ENABLED); ++ u32 v; ++ ++ v = ((pcm ? sps->pcm_sample_bit_depth_chroma_minus1 & 15 : 0) + 1) << 12 | ++ ((pcm ? sps->pcm_sample_bit_depth_luma_minus1 & 15 : 0) + 1) << 8; ++ if (pcm) ++ v |= (sps->log2_diff_max_min_pcm_luma_coding_block_size & 3) << 6 | ++ (sps->log2_min_pcm_luma_coding_block_size_minus3 & 3) << 4 | ++ SPS_F(sps, PCM_LOOP_FILTER_DISABLED) << 2; ++ return v | pcm << 1 | SPS_F(sps, AMP_ENABLED); ++} ++ ++static u32 hevc_tiles(const struct v4l2_ctrl_hevc_pps *pps) ++{ ++ bool tiles = PPS_F(pps, TILES_ENABLED); ++ ++ return (tiles ? pps->num_tile_rows_minus1 : 0) << 24 | ++ (tiles ? pps->num_tile_columns_minus1 : 0) << 16 | ++ PPS_F(pps, ENTROPY_CODING_SYNC_ENABLED) << 10 | ++ tiles << 9 | PPS_F(pps, TRANSQUANT_BYPASS_ENABLED) << 8; ++} ++ ++static u32 hevc_pic_config(const struct hevc_pic *p) ++{ ++ const struct v4l2_ctrl_hevc_sps *sps = p->f->sps; ++ ++ return (p->min_type == HEVC_I) << 30 | ++ (sps->log2_min_luma_coding_block_size_minus3 + ++ sps->log2_diff_max_min_luma_coding_block_size) << 28 | ++ 0x1c0000 | (sps->bit_depth_chroma_minus8 & 7) << 8 | ++ (sps->bit_depth_luma_minus8 & 7) << 4 | sps->chroma_format_idc; ++} ++ ++/* ++ * Tile boundaries in MBs, two 16-bit halves per word: cumulative explicit ++ * sizes, or the kext's float32 accumulation of PicSizeInCtbs / (n + 1), ++ * truncated per step. The integer form below is the H.265 6.5.1 one, which ++ * the float form equals whenever it is exact (shift by CtbLog2 - 4). ++ */ ++static void hevc_tile_bounds(const struct v4l2_ctrl_hevc_pps *pps, u32 shift, ++ u32 w_ctb, u32 h_ctb, u32 *cols, u32 *rows) ++{ ++ unsigned int k, n; ++ u32 acc; ++ ++ memset(cols, 0, 12 * sizeof(*cols)); ++ memset(rows, 0, 12 * sizeof(*rows)); ++ if (!PPS_F(pps, TILES_ENABLED)) ++ return; ++ ++ n = pps->num_tile_columns_minus1; ++ for (k = 0, acc = 0; k < n; k++) { ++ if (PPS_F(pps, UNIFORM_SPACING)) ++ acc = (k + 1) * w_ctb / (n + 1); ++ else ++ acc += pps->column_width_minus1[k] + 1; ++ cols[k / 2] |= (acc << shift) << (16 * (k & 1)); ++ } ++ n = pps->num_tile_rows_minus1; ++ for (k = 0, acc = 0; k < n; k++) { ++ if (PPS_F(pps, UNIFORM_SPACING)) ++ acc = (k + 1) * h_ctb / (n + 1); ++ else ++ acc += pps->row_height_minus1[k] + 1; ++ rows[k / 2] |= (acc << shift) << (16 * (k & 1)); ++ } ++} ++ ++/* --- per slice -------------------------------------------------------- */ ++ ++static int hevc_slice_params(const struct apple_vxd_hevc_frame *f, ++ unsigned int idx, struct hevc_slice *s, ++ const char **why) ++{ ++ const struct v4l2_ctrl_hevc_slice_params *sp = &f->slices[idx]; ++ const struct v4l2_ctrl_hevc_pps *pps = f->pps; ++ unsigned int i, lst; ++ ++ memset(s, 0, sizeof(*s)); ++ s->address = sp->slice_segment_addr; ++ s->dependent = SLICE_F(sp, DEPENDENT_SLICE_SEGMENT); ++ if (s->dependent) { ++ /* ++ * The kext parses only the address of a dependent segment and ++ * builds its words from a zeroed record (slice type B, no ++ * references, MaxNumMergeCand 5). The core takes the rest ++ * from the independent segment. ++ */ ++ s->merge_cand = 5; ++ s->qp = 26 + pps->init_qp_minus26; ++ return 0; ++ } ++ ++ s->type = sp->slice_type; ++ if (s->type > HEVC_I) { ++ *why = "invalid slice type"; ++ return -EINVAL; ++ } ++ s->sao_luma = SLICE_F(sp, SLICE_SAO_LUMA); ++ s->sao_chroma = SLICE_F(sp, SLICE_SAO_CHROMA); ++ s->temporal_mvp = SLICE_F(sp, SLICE_TEMPORAL_MVP_ENABLED); ++ s->mvd_l1_zero = SLICE_F(sp, MVD_L1_ZERO); ++ s->cabac_init = SLICE_F(sp, CABAC_INIT); ++ /* Only B slices code it; the kext's default is 1. */ ++ s->col_from_l0 = s->type != HEVC_B || SLICE_F(sp, COLLOCATED_FROM_L0); ++ s->dbk_disabled = SLICE_F(sp, SLICE_DEBLOCKING_FILTER_DISABLED); ++ s->lf_across_slices = SLICE_F(sp, SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED); ++ /* I slices carry the PPS defaults, P slices the PPS L1 default. */ ++ s->l0 = s->type == HEVC_I ? pps->num_ref_idx_l0_default_active_minus1 : ++ sp->num_ref_idx_l0_active_minus1; ++ s->l1 = s->type == HEVC_B ? sp->num_ref_idx_l1_active_minus1 : ++ pps->num_ref_idx_l1_default_active_minus1; ++ if (s->l0 > 14 || s->l1 > 14) { ++ *why = "more than 15 active references"; ++ return -EINVAL; ++ } ++ s->col_ref_idx = s->temporal_mvp ? sp->collocated_ref_idx : 0; ++ s->merge_cand = 5 - (s->type == HEVC_I ? 0 : sp->five_minus_max_num_merge_cand); ++ s->qp = 26 + pps->init_qp_minus26 + sp->slice_qp_delta; ++ s->cb_qp = sp->slice_cb_qp_offset; ++ s->cr_qp = sp->slice_cr_qp_offset; ++ s->beta = sp->slice_beta_offset_div2; ++ s->tc = sp->slice_tc_offset_div2; ++ s->weighted = (PPS_F(pps, WEIGHTED_PRED) && s->type == HEVC_P) || ++ (PPS_F(pps, WEIGHTED_BIPRED) && s->type == HEVC_B); ++ ++ for (lst = 0; s->type != HEVC_I && lst < (s->type == HEVC_B ? 2 : 1); lst++) { ++ const u8 *ref = lst ? sp->ref_idx_l1 : sp->ref_idx_l0; ++ u32 n = lst ? s->l1 : s->l0; ++ ++ for (i = 0; i <= n; i++) { ++ if (ref[i] >= f->dp->num_active_dpb_entries) { ++ *why = "a reference index names no DPB entry"; ++ return -EINVAL; ++ } ++ s->slot[lst][i] = ref[i]; ++ } ++ } ++ if (s->type != HEVC_I && s->col_ref_idx > (s->col_from_l0 ? s->l0 : s->l1)) { ++ *why = "collocated_ref_idx out of range"; ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++/* The slice structure and its 0xf transfer (0x62b6010). */ ++static void hevc_slice_struct(struct apple_vxd_cmdbuf *cb, struct hevc_pic *p, ++ const struct hevc_slice *s, ++ const struct v4l2_ctrl_hevc_slice_params *sp, ++ size_t w3090, size_t w3094) ++{ ++ const struct apple_vxd_hevc_frame *f = p->f; ++ const struct v4l2_ctrl_hevc_sps *sps = f->sps; ++ const struct v4l2_ctrl_hevc_pps *pps = f->pps; ++ const struct v4l2_hevc_pred_weight_table *pw = &sp->pred_weight_table; ++ u32 hdr = s->type == HEVC_B ? 48 : s->type == HEVC_P ? 32 : 16; ++ u32 *w = (u32 *)(f->side_cpu + p->side_used); ++ u32 *pairs = w + hdr / 4; ++ u32 npairs = 0, pad, unpadded, transfer, lines, c320, c321; ++ unsigned int i, lst; ++ ++ memset(w, 0, hdr); ++#define PAIR(r, v) \ ++ do { \ ++ pairs[2 * npairs] = VXD_CMD_REG1 | (r); \ ++ pairs[2 * npairs + 1] = (v); \ ++ npairs++; \ ++ } while (0) ++ if (s->weighted) { ++ u32 cden = pw->luma_log2_weight_denom + pw->delta_chroma_log2_weight_denom; ++ ++ PAIR(HEVC_REN_WEIGHT_DENOM, pw->luma_log2_weight_denom | cden << 4); ++ for (lst = 0; lst < (s->type == HEVC_B ? 2 : 1); lst++) { ++ const s8 *dlw = lst ? pw->delta_luma_weight_l1 : pw->delta_luma_weight_l0; ++ const s8 *lo = lst ? pw->luma_offset_l1 : pw->luma_offset_l0; ++ const s8 (*dcw)[2] = lst ? pw->delta_chroma_weight_l1 : ++ pw->delta_chroma_weight_l0; ++ const s8 (*co)[2] = lst ? pw->chroma_offset_l1 : pw->chroma_offset_l0; ++ ++ for (i = 0; i <= (lst ? s->l1 : s->l0); i++) { ++ PAIR(HEVC_REN_WEIGHTS(lst) + 4 * i, ++ ((u32)dlw[i] & 0x1ff) | ((u32)dcw[i][0] & 0x1ff) << 9 | ++ ((u32)dcw[i][1] & 0x1ff) << 18); ++ PAIR(HEVC_REN_OFFSETS(lst) + 4 * i, ++ (u8)lo[i] | (u32)(u8)co[i][0] << 8 | ++ (u32)(u8)co[i][1] << 16); ++ } ++ } ++ } ++ if (f->caps->tail_7474) { ++ PAIR(HEVC_REN_7474, 0); ++ PAIR(HEVC_REN_7474, 0); ++ } ++#undef PAIR ++ ++ c320 = PPS_F(pps, CONSTRAINED_INTRA_PRED) << 7 | ++ SPS_F(sps, STRONG_INTRA_SMOOTHING_ENABLED) << 6 | ++ SPS_F(sps, SAMPLE_ADAPTIVE_OFFSET) << 5 | s->sao_chroma << 3 | ++ s->sao_luma << 2 | ++ (PPS_F(pps, WEIGHTED_BIPRED) && s->type == HEVC_B) << 1 | ++ (PPS_F(pps, WEIGHTED_PRED) && s->type == HEVC_P); ++ c321 = PPS_F(pps, LOOP_FILTER_ACROSS_TILES_ENABLED) << 2 | ++ s->lf_across_slices << 1 | s->dbk_disabled; ++ ++ w[0] = s->type | s->sao_luma << 2 | s->sao_chroma << 3 | s->temporal_mvp << 4 | ++ s->l0 << 8 | s->l1 << 12 | s->mvd_l1_zero << 16 | s->col_from_l0 << 18 | ++ s->merge_cand << 20; ++ w[1] = s->lf_across_slices << 24 | ((p->qp_bd + s->qp) & 0x7f) | ++ ((u32)s->cb_qp & 31) << 8 | ((u32)s->cr_qp & 31) << 16; ++ w[2] = p->seg_y << 10 | p->seg_x | npairs << 20; ++ w[3] = c320 << 16 | c321 << 12 | ((u32)s->tc & 15) << 8 | ++ ((u32)s->beta & 15) << 4 | 8 | (2 - s->type); ++ for (lst = 0; s->type != HEVC_I && lst < (s->type == HEVC_B ? 2 : 1); lst++) ++ for (i = 0; i < 15; i++) ++ w[4 + 4 * lst + i / 4] |= (u32)s->slot[lst][i] << (8 * (i % 4)); ++ ++ pad = (npairs * 0x38 - hdr) & 0x38; ++ unpadded = hdr + 8 * npairs; ++ memset((u8 *)w + unpadded, 0, pad); ++ transfer = unpadded + pad; ++ lines = unpadded - 1 > 0x3bf ? 15 : (unpadded - 1) >> 6; ++ ++ /* Q+0x7c (minimum independent slice type) and Q+0x84 (cache hint). */ ++ if (!s->address || p->min_type_q < 0 || (int)s->type < p->min_type_q) { ++ u32 d = 0; ++ ++ if (s->type == HEVC_P && pps->num_ref_idx_l0_default_active_minus1) ++ d = abs((int)s->slot[0][0] - (int)s->slot[0][1]); ++ else if (s->type == HEVC_B) ++ d = abs((int)s->slot[0][0] - (int)s->slot[1][0]); ++ p->cache_div = max(p->cache_div, d); ++ p->min_type_q = s->type; ++ } ++ ++ if (!p->transfers) ++ p->dest = f->spdest; ++ apple_vxd_cmd_table_xfer(cb, transfer, f->side + p->side_used, p->dest); ++ cb->vaddr[w3090] = p->dest; ++ cb->vaddr[w3094] = lines; ++ p->prev_dest = p->dest; ++ p->prev_lines = lines; ++ p->dest += transfer; ++ p->transfers++; ++ p->side_used += transfer; ++ ++ /* The co-located picture: the first P/B slice decides (L28666). */ ++ if (SPS_F(sps, SPS_TEMPORAL_MVP_ENABLED) && p->col_slot < 0 && ++ !p->col_invalid && s->type != HEVC_I) { ++ u8 slot = s->slot[s->col_from_l0 ? 0 : 1][s->col_ref_idx]; ++ ++ if (f->refs[slot].rec && f->refs[slot].rec->valid) ++ p->col_slot = slot; ++ else ++ p->col_invalid = true; ++ } ++} ++ ++static int hevc_emit_slice(struct apple_vxd_cmdbuf *cb, struct hevc_pic *p, ++ unsigned int idx, const char **why) ++{ ++ const struct apple_vxd_hevc_frame *f = p->f; ++ const struct v4l2_ctrl_hevc_sps *sps = f->sps; ++ const struct v4l2_ctrl_hevc_pps *pps = f->pps; ++ const struct v4l2_ctrl_hevc_slice_params *sp = &f->slices[idx]; ++ const struct apple_vxd_hevc_nal *nal = &f->nals[idx]; ++ u32 hdr[VXD_CMD_SLICE_BLOCK_WORDS - 1] = { }; ++ u32 row, col_mb, row_mb, w3410, w3414; ++ size_t w3090, w3094; ++ struct hevc_slice s; ++ unsigned int k; ++ int ret; ++ ++ ret = hevc_slice_params(f, idx, &s, why); ++ if (ret) ++ return ret; ++ if (!s.dependent && p->side_used + HEVC_STRUCT_MAX > f->side_size) { ++ *why = "slice structure buffer full"; ++ return -ENOSPC; ++ } ++ ++ p->min_type = idx ? min(p->min_type, s.type) : s.type; ++ ++ /* Segment position in MBs (0x62b55f0). */ ++ row = p->w_ctb ? s.address / p->w_ctb : 0; ++ col_mb = (s.address - row * p->w_ctb) * p->ctb_mb; ++ row_mb = row * p->ctb_mb; ++ if (!s.dependent) { ++ p->seg_x = col_mb; ++ p->seg_y = row_mb; ++ } ++ w3410 = s.dependent << 31 | (p->seg_y & 0xfff) << 16 | (p->seg_x & 0xfff); ++ w3414 = ((p->qp_bd + s.qp) & 0x7f) << 24 | s.merge_cand << 20 | ++ s.cabac_init << 17 | s.mvd_l1_zero << 16 | s.l1 << 12 | s.l0 << 8 | ++ s.sao_chroma << 3 | s.sao_luma << 2 | s.type; ++ ++ p->last_desc = cb->used / 4; ++ hdr[3] = row_mb << 16 | col_mb; ++ hdr[4] = (DIV_ROUND_UP(p->w, 16) - 1) | (DIV_ROUND_UP(p->h, 16) - 1) << 16; ++ apple_vxd_cmd_ctrl_alloc(cb, hdr); ++ ++ apple_vxd_cmd_slice_desc(cb, nal->header_bits, nal->size); ++ apple_vxd_cmd_load(cb, f->src + nal->offset, nal->size); ++ ++ apple_vxd_cmd_group_open(cb, 0); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_FRAGMENT, f->frag & ~63U); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_FORMAT, sps->chroma_format_idc << 16 | 0x10c); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SIZE, (p->h - 1) << 16 | (p->w - 1)); ++ apple_vxd_cmd_group_close(cb); ++ apple_vxd_cmd_group_open(cb, 0); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SLICE_POS, row_mb << 16 | col_mb); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SEGMENT, w3410); ++ apple_vxd_cmd_group_close(cb); ++ if (f->first_slice_of_session && !idx) { ++ apple_vxd_cmd_group_open(cb, 0); ++ for (k = 0; k < 4; k++) ++ apple_vxd_cmd_reg(cb, HEVC_CMD_RESET_ZERO + 4 * k, 0); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_RESET, 1); ++ apple_vxd_cmd_group_close(cb); ++ } ++ apple_vxd_cmd_group_open(cb, 0); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SPS0, hevc_sps0(sps)); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SPS1, hevc_sps1(sps)); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_PPS0, hevc_tiles(pps) | ++ (pps->diff_cu_qp_delta_depth & 3) << 4 | ++ PPS_F(pps, CU_QP_DELTA_ENABLED) << 3 | ++ PPS_F(pps, TRANSFORM_SKIP_ENABLED) << 2 | ++ PPS_F(pps, SIGN_DATA_HIDING_ENABLED)); ++ apple_vxd_cmd_group_close(cb); ++ apple_vxd_cmd_group_open(cb, 0); ++ for (k = 0; k < 12; k++) ++ apple_vxd_cmd_reg(cb, HEVC_CMD_TILE_COLS + 4 * k, p->tile_cols[k]); ++ for (k = 0; k < 12; k++) ++ apple_vxd_cmd_reg(cb, HEVC_CMD_TILE_ROWS + 4 * k, p->tile_rows[k]); ++ apple_vxd_cmd_group_close(cb); ++ apple_vxd_cmd_group_open(cb, 0); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SLICE0, w3414); ++ apple_vxd_cmd_group_close(cb); ++ apple_vxd_cmd_group_open(cb, 0); ++ for (k = 0; k < APPLE_VXD_HEVC_GENC_BUFS; k++) { ++ apple_vxd_cmd_reg(cb, HEVC_CMD_GENC_SIZE + 4 * k, f->genc_size & ~0xfffU); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_GENC_BASE + 4 * k, f->genc[k] & ~0xfffU); ++ } ++ apple_vxd_cmd_group_close(cb); ++ ++ /* 0x3090/0x3094 first, filled in once the structure exists. */ ++ apple_vxd_cmd_group_open(cb, 0); ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SP_DEST, 0); ++ w3090 = cb->used / 4 - 1; ++ apple_vxd_cmd_reg(cb, HEVC_CMD_SP_LINES, 0); ++ w3094 = cb->used / 4 - 1; ++ apple_vxd_cmd_group_close(cb); ++ if (cb->error) ++ return -ENOSPC; ++ if (s.dependent) { ++ cb->vaddr[w3090] = p->prev_dest; ++ cb->vaddr[w3094] = p->prev_lines; ++ } else { ++ hevc_slice_struct(cb, p, &s, sp, w3090, w3094); ++ } ++ apple_vxd_cmd_end_slice(cb); ++ return cb->error ? -ENOSPC : 0; ++} ++ ++/* --- per picture: the kind-4 chunk (0x62b7ab0, 0x62b859c) -------------- */ ++ ++/* ++ * Scaling lists: the 8x8 base lists one byte per coefficient, four per ++ * register; register i takes coefficients 4i..4i+3 of a de-interleaved ++ * order in which the even index bits give the row and the odd bits the ++ * column (the kext's gather tables at 0xfffffff005595040/0x5595050). ++ */ ++static u32 hevc_scl_word(const u8 *raster, unsigned int n, unsigned int i) ++{ ++ unsigned int k, l, row, col, side = n == 16 ? 8 : 4; ++ u32 v = 0; ++ ++ for (k = 0; k < 4; k++) { ++ l = 4 * i + k; ++ row = (l & 1) | (l >> 1 & 2) | (l >> 2 & 4); ++ col = (l >> 1 & 1) | (l >> 2 & 2) | (l >> 3 & 4); ++ if (side == 4) { ++ row &= 3; ++ col &= 3; ++ } ++ v |= (u32)raster[row * side + col] << (8 * k); ++ } ++ return v; ++} ++ ++static void hevc_scl_list(struct apple_vxd_cmdbuf *cb, u32 reg, const u8 *raster, ++ unsigned int n) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < n; i++) ++ apple_vxd_cmd_reg1(cb, reg + 4 * i, hevc_scl_word(raster, n, i), 0); ++} ++ ++static void hevc_emit_scaling(struct apple_vxd_cmdbuf *cb, ++ const struct v4l2_ctrl_hevc_scaling_matrix *m) ++{ ++ unsigned int i; ++ ++ hevc_scl_list(cb, HEVC_ENT_SCALING + 0x000, m->scaling_list_32x32[0], 16); ++ hevc_scl_list(cb, HEVC_ENT_SCALING + 0x040, m->scaling_list_32x32[1], 16); ++ for (i = 0; i < 6; i++) ++ hevc_scl_list(cb, HEVC_ENT_SCALING + 0x080 + 0x40 * i, ++ m->scaling_list_16x16[i], 16); ++ for (i = 0; i < 6; i++) ++ hevc_scl_list(cb, HEVC_ENT_SCALING + 0x200 + 0x40 * i, ++ m->scaling_list_8x8[i], 16); ++ for (i = 0; i < 6; i++) ++ hevc_scl_list(cb, HEVC_ENT_SCALING + 0x380 + 0x10 * i, ++ m->scaling_list_4x4[i], 4); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SCALING + 0x3e0, m->scaling_list_dc_coef_32x32[0], 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SCALING + 0x3e4, m->scaling_list_dc_coef_32x32[1], 0); ++ for (i = 0; i < 6; i++) ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SCALING + 0x3e8 + 4 * i, ++ m->scaling_list_dc_coef_16x16[i], 0); ++} ++ ++/* 0x701c: Apple's line-buffer split for 4:2:0 (13d684 set at create). */ ++static u32 hevc_cache_policy(const struct hevc_pic *p) ++{ ++ u32 t = (p->f->caps->cache_base * 0x15556) >> 17; ++ ++ t = (t * 15) >> 5; ++ if (p->cache_div > 1) ++ t /= p->cache_div; ++ return 1 << 24 | ALIGN(t, 8) << 12 | 12; ++} ++ ++static void hevc_emit_picture(struct apple_vxd_cmdbuf *cb, struct hevc_pic *p) ++{ ++ const struct apple_vxd_hevc_frame *f = p->f; ++ const struct v4l2_ctrl_hevc_sps *sps = f->sps; ++ const struct v4l2_ctrl_hevc_pps *pps = f->pps; ++ const struct v4l2_ctrl_hevc_decode_params *dp = f->dp; ++ u32 min_type = p->min_type_q < 0 ? 0 : p->min_type_q; ++ u32 lt = 0, delta[4] = { }, size8, w5400; ++ unsigned int k; ++ ++ size8 = ((ALIGN(p->h, 8) - 1) << 16) | (ALIGN(p->w, 8) - 1); ++ w5400 = hevc_sps0(sps) | SPS_F(sps, SCALING_LIST_ENABLED) << 31; ++ ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_ABOVE, f->above & ~63U, HEVC_F_ADDR); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SIZE8, size8, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SPS0, w5400, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SPS0 + 4, hevc_sps1(sps) | ++ SPS_F(sps, SPS_TEMPORAL_MVP_ENABLED) << 16, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SPS0 + 8, hevc_tiles(pps) | ++ PPS_F(pps, TRANSFORM_SKIP_ENABLED) << 2, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_SPS0 + 12, ++ (pps->log2_parallel_merge_level_minus2 & 7) << 16 | ++ ((u32)pps->pps_cr_qp_offset & 31) << 8 | ++ ((u32)pps->pps_cb_qp_offset & 31), 0); ++ for (k = 0; k < APPLE_VXD_HEVC_GENC_BUFS; k++) { ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_GENC_SIZE + 4 * k, f->genc_size & ~0xfffU, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_GENC_BASE + 4 * k, f->genc[k] & ~0xfffU, ++ HEVC_F_ADDR); ++ } ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_FRAGMENT, f->frag & ~63U, HEVC_F_ADDR); ++ for (k = 0; k < 12; k++) ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_TILE_COLS + 4 * k, p->tile_cols[k], 0); ++ for (k = 0; k < 12; k++) ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_TILE_ROWS + 4 * k, p->tile_rows[k], 0); ++ ++ for (k = 0; k < dp->num_active_dpb_entries; k++) { ++ const struct v4l2_hevc_dpb_entry *e = &dp->dpb[k]; ++ s32 d = clamp_t(s32, dp->pic_order_cnt_val - e->pic_order_cnt_val, -128, 127); ++ ++ if (e->flags & V4L2_HEVC_DPB_ENTRY_LONG_TERM_REFERENCE) ++ lt |= BIT(k); ++ delta[k / 4] |= (u32)(u8)d << (8 * (k % 4)); ++ } ++ f->record->refs = lt | (2 - min_type) << 16; ++ memcpy(f->record->poc_delta, delta, sizeof(delta)); ++ f->record->mv = f->mv; ++ f->record->valid = 1; ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_REFS, f->record->refs, 0); ++ for (k = 0; k < 4; k++) ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_POC_DELTA + 4 * k, delta[k], 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_CUR_MV, f->mv, HEVC_F_ADDR); ++ if (p->col_slot < 0) { ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_COL_REFS, 0, 0); ++ } else { ++ const struct apple_vxd_hevc_record *r = f->refs[p->col_slot].rec; ++ ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_COL_REFS, r->refs, 0); ++ for (k = 0; k < 4; k++) ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_COL_POC_DELTA + 4 * k, ++ r->poc_delta[k], 0); ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_COL_MV, r->mv, HEVC_F_ADDR); ++ } ++ apple_vxd_cmd_reg1(cb, HEVC_ENT_CONTROL, ++ sps->chroma_format_idc << 28 | 0x10c0000, 0); ++ if (SPS_F(sps, SCALING_LIST_ENABLED)) ++ hevc_emit_scaling(cb, f->sm); ++ ++ apple_vxd_cmd_reg1(cb, HEVC_REN_PIC_CONFIG, hevc_pic_config(p), 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_CHROMA_QP, ((u32)pps->pps_cb_qp_offset & 31) | ++ ((u32)pps->pps_cr_qp_offset & 31) << 8, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_SIZE, (p->h - 1) << 16 | (p->w - 1), 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_SIZE8, size8, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_AUXLINE, f->auxline, HEVC_F_ADDR); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_CACHE, hevc_cache_policy(p), 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_STRIDE, f->dst_stride & ~63U, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_FORMAT, (sps->bit_depth_chroma_minus8 & 7) << 12 | ++ (sps->bit_depth_luma_minus8 & 7) << 8 | sps->chroma_format_idc, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_LAYOUT, HEVC_LAYOUT_LINEAR, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_CSTRIDE, f->dst_cstride | f->dst_cstride << 16, 0); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_DST, f->dst_luma, HEVC_F_ADDR); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_DST + 4, f->dst_chroma, HEVC_F_ADDR); ++ if (min_type < HEVC_I) { ++ for (k = 0; k < dp->num_active_dpb_entries; k++) { ++ apple_vxd_cmd_reg1(cb, HEVC_REN_REFS + 8 * k, f->refs[k].luma, ++ HEVC_F_ADDR); ++ apple_vxd_cmd_reg1(cb, HEVC_REN_REFS + 8 * k + 4, f->refs[k].chroma, ++ HEVC_F_ADDR); ++ } ++ } ++ apple_vxd_cmd_reg1(cb, HEVC_REN_INTRA, f->intra, HEVC_F_ADDR); ++ ++ /* The last slice descriptor learns the strides and the layout. */ ++ cb->vaddr[p->last_desc + 3] = f->dst_cstride | f->dst_cstride << 16; ++ cb->vaddr[p->last_desc + 6] = 0; ++ cb->vaddr[p->last_desc + 7] = 0; ++ cb->vaddr[p->last_desc + 8] = HEVC_LAYOUT_LINEAR; ++} ++ ++/* ++ * One picture: the slice chunks with their kind-1 packets, then the kind-4 ++ * chunk and packet. Returns 0, or -EINVAL / -ENOSPC with @why set. ++ */ ++int apple_vxd_hevc_build(struct apple_vxd_cmdbuf *cb, ++ const struct apple_vxd_hevc_frame *f, ++ const char **why) ++{ ++ const struct v4l2_ctrl_hevc_sps *sps = f->sps; ++ struct hevc_pic p = { .f = f, .col_slot = -1, .min_type_q = -1 }; ++ u32 ctb_log2, mcb, off, bytes, *pkt; ++ unsigned int i, k; ++ int ret; ++ ++ *why = NULL; ++ ret = apple_vxd_hevc_validate(sps, f->pps, why); ++ if (ret) ++ return ret; ++ if (!f->nslices || f->nslices > APPLE_VXD_HEVC_MAX_SLICES) { ++ *why = "no slices, or more than a job holds"; ++ return -EINVAL; ++ } ++ if (f->dp->num_active_dpb_entries > APPLE_VXD_HEVC_MAX_REFS) { ++ *why = "more than 15 reference pictures"; ++ return -EINVAL; ++ } ++ for (k = 0; k < f->dp->num_active_dpb_entries; k++) { ++ if (!f->refs[k].luma) { ++ *why = "a DPB entry has no CAPTURE buffer"; ++ return -EINVAL; ++ } ++ } ++ if (SPS_F(sps, SCALING_LIST_ENABLED) && (!f->sm || !f->sm->scaling_list_4x4[0][0])) { ++ *why = "scaling lists enabled but the SCALING_MATRIX control is not set"; ++ return -EINVAL; ++ } ++ ++ mcb = sps->log2_min_luma_coding_block_size_minus3 + 3; ++ ctb_log2 = mcb + sps->log2_diff_max_min_luma_coding_block_size; ++ hevc_coded_size(sps, &p.w, &p.h); ++ p.ctb_mb = 1 << (ctb_log2 - 4); ++ p.w_ctb = DIV_ROUND_UP(p.w, 1 << ctb_log2); ++ p.h_ctb = DIV_ROUND_UP(p.h, 1 << ctb_log2); ++ p.qp_bd = 6 * sps->bit_depth_luma_minus8; ++ hevc_tile_bounds(f->pps, ctb_log2 - 4, p.w_ctb, p.h_ctb, p.tile_cols, p.tile_rows); ++ ++ for (i = 0; i < f->nslices; i++) { ++ if (!i && SLICE_F(&f->slices[0], DEPENDENT_SLICE_SEGMENT)) { ++ *why = "picture starts with a dependent slice segment"; ++ return -EINVAL; ++ } ++ ret = hevc_emit_slice(cb, &p, i, why); ++ if (ret) { ++ if (!*why) ++ *why = "command buffer full"; ++ return ret; ++ } ++ ret = apple_vxd_cmdbuf_chunk_end(cb, &off, &bytes); ++ if (ret) { ++ *why = "command buffer full"; ++ return ret; ++ } ++ pkt = apple_vxd_cmdbuf_add_packet(cb, VXD_PKT_MAGIC0, VXD_PKT_H264_PAYLOAD); ++ if (!pkt) { ++ *why = "packet list full"; ++ return -ENOSPC; ++ } ++ pkt[VXD_PKT_FLAGS / 4] = (i ? 0 : 1) | ++ (!i && f->first_picture_of_session ? 0x10 : 0); ++ pkt[VXD_PKT_W4 / 4] = hevc_pic_config(&p); ++ pkt[VXD_PKT_CMD_BYTES / 4] = bytes; ++ pkt[VXD_PKT_W2 / 4] = cb->va + off; ++ pkt[VXD_PKT_W5 / 4] = f->stream_state; ++ pkt[VXD_PKT_KIND / 4] = VXD_PKT_KIND_PICTURE; ++ } ++ ++ hevc_emit_picture(cb, &p); ++ ret = apple_vxd_cmdbuf_pair_chunk_end(cb, &off, &bytes); ++ if (ret) { ++ *why = "command buffer full"; ++ return ret; ++ } ++ pkt = apple_vxd_cmdbuf_add_packet(cb, VXD_PKT_MAGIC0, VXD_PKT_H264_PAYLOAD); ++ if (!pkt) { ++ *why = "packet list full"; ++ return -ENOSPC; ++ } ++ pkt[VXD_PKT_W4 / 4] = hevc_pic_config(&p); ++ pkt[VXD_PKT_CMD_BYTES / 4] = bytes; ++ pkt[VXD_PKT_W2 / 4] = cb->va + off; ++ pkt[VXD_PKT_W5 / 4] = f->stream_state; ++ pkt[VXD_PKT_KIND / 4] = VXD_PKT_KIND_RENDER_TARGET; ++ return 0; ++} +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.h b/drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.h +new file mode 100644 +index 000000000..8be0b1bf4 +--- /dev/null ++++ b/drivers/media/platform/apple/vxd/apple-vxd-hevc-cmd.h +@@ -0,0 +1,109 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Apple VXD - HEVC (H.265) command-stream builder ++ * ++ * Pure code: no MMIO, no DMA API, no locking, so it also builds on the host ++ * (scripts/vdec/host/vdec-hevc-host.c) and its words can be compared with ++ * the reference generator. ++ */ ++#ifndef __APPLE_VXD_HEVC_CMD_H__ ++#define __APPLE_VXD_HEVC_CMD_H__ ++ ++#include ++#include ++#include ++ ++#include "apple-vxd-cmd.h" ++ ++#define APPLE_VXD_HEVC_MAX_SLICES 127 /* + the kind-4 packet <= 128 */ ++#define APPLE_VXD_HEVC_MAX_REFS 15 /* hardware reference slots */ ++#define APPLE_VXD_HEVC_SIDE_SIZE SZ_256K /* slice structure buffer */ ++#define APPLE_VXD_HEVC_SPDEST_SIZE SZ_1M /* slice parameter buffer */ ++#define APPLE_VXD_HEVC_INTRA_SIZE SZ_64K ++#define APPLE_VXD_HEVC_GENC_BUFS 4 ++ ++/* Per-SoC HEVC parameters (match data). */ ++struct apple_vxd_hevc_caps { ++ u8 buf_mult; /* 1, or 2 from vxd-version 9 on */ ++ bool tail_7474; /* two 0x7474 writes after every slice structure */ ++ u16 cache_base; /* 0x701c line split base */ ++}; ++ ++/* ++ * What a decoded picture keeps for later use as the co-located picture ++ * (the kext's 0x1c-byte picture record): its 0x5438 and 0x5440..0x544c ++ * words and its temporal MV buffer. It lives in the CAPTURE buffer's ++ * side buffer, so it follows the picture. ++ */ ++struct apple_vxd_hevc_record { ++ u32 refs; /* 0x5438 */ ++ u32 poc_delta[4]; /* 0x5440..0x544c */ ++ u32 mv; /* 0x5434 */ ++ u32 valid; ++}; ++ ++/* Per-CAPTURE side buffer layout (AllocatePictureMemory sizes). */ ++struct apple_vxd_hevc_aux_layout { ++ u32 mv, frag, above, record, size; ++}; ++ ++struct apple_vxd_hevc_ref { ++ u32 luma; ++ u32 chroma; ++ const struct apple_vxd_hevc_record *rec; ++}; ++ ++/* One located slice NAL in the OUTPUT buffer. */ ++struct apple_vxd_hevc_nal { ++ u32 offset; /* first NAL header byte */ ++ u32 size; /* through the last byte before the next start code */ ++ u32 header_bits; /* RBSP bits from the NAL header to slice_data() */ ++}; ++ ++struct apple_vxd_hevc_frame { ++ const struct v4l2_ctrl_hevc_sps *sps; ++ const struct v4l2_ctrl_hevc_pps *pps; ++ const struct v4l2_ctrl_hevc_decode_params *dp; ++ const struct v4l2_ctrl_hevc_scaling_matrix *sm; /* NULL: none set */ ++ const struct v4l2_ctrl_hevc_slice_params *slices; ++ const struct apple_vxd_hevc_nal *nals; ++ unsigned int nslices; ++ const struct apple_vxd_hevc_caps *caps; ++ ++ /* Device addresses. */ ++ u32 src; /* OUTPUT buffer */ ++ u32 dst_luma, dst_chroma, dst_stride, dst_cstride; ++ u32 mv, frag, above; /* this picture's side buffer */ ++ struct apple_vxd_hevc_ref refs[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; ++ u32 genc[APPLE_VXD_HEVC_GENC_BUFS]; ++ u32 genc_size; ++ u32 side; /* slice structures, CPU-written */ ++ u8 *side_cpu; ++ size_t side_size; ++ u32 spdest; /* their transfer destination */ ++ u32 intra, auxline, stream_state; ++ ++ /* Session state (C[13c469], C[136254]); the caller updates both. */ ++ bool first_slice_of_session; ++ bool first_picture_of_session; ++ ++ /* Output: this picture's record (in its side buffer). */ ++ struct apple_vxd_hevc_record *record; ++}; ++ ++u32 apple_vxd_hevc_genc_size(unsigned int width, unsigned int height); ++void apple_vxd_hevc_aux_layout(const struct v4l2_ctrl_hevc_sps *sps, ++ struct apple_vxd_hevc_aux_layout *l); ++u32 apple_vxd_hevc_aux_size(unsigned int width, unsigned int height); ++int apple_vxd_hevc_validate(const struct v4l2_ctrl_hevc_sps *sps, ++ const struct v4l2_ctrl_hevc_pps *pps, ++ const char **why); ++int apple_vxd_hevc_find_nals(const u8 *src, size_t len, ++ const struct v4l2_ctrl_hevc_slice_params *slices, ++ unsigned int nslices, struct apple_vxd_hevc_nal *nals, ++ const char **why); ++int apple_vxd_hevc_build(struct apple_vxd_cmdbuf *cb, ++ const struct apple_vxd_hevc_frame *f, ++ const char **why); ++ ++#endif /* __APPLE_VXD_HEVC_CMD_H__ */ diff --git a/kernel/patches/drivers/media/0010-media-apple-vxd-Add-the-stateless-HEVC-decoder.patch b/kernel/patches/drivers/media/0010-media-apple-vxd-Add-the-stateless-HEVC-decoder.patch new file mode 100644 index 00000000..dd13a658 --- /dev/null +++ b/kernel/patches/drivers/media/0010-media-apple-vxd-Add-the-stateless-HEVC-decoder.patch @@ -0,0 +1,559 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 21:25:29 +0100 +Subject: [PATCH] media: apple-vxd: Add the stateless HEVC decoder + +Expose V4L2_PIX_FMT_HEVC_SLICE next to H.264: frame-based decoding with +Annex-B start codes (what FFmpeg's v4l2request hwaccel, GStreamer's +v4l2slh265dec and libva-v4l2-request send), the SPS, PPS, SLICE_PARAMS, +SCALING_MATRIX and DECODE_PARAMS controls, NV12 output. + +Per picture the unit locates the slice NALs in the OUTPUT buffer, +resolves the DPB to CAPTURE buffers by timestamp and runs the builder. +The slice header's length comes from data_byte_offset, which FFmpeg and +libva count from the buffer and GStreamer from the slice's start code; +both are accepted. ENTRY_POINT_OFFSETS is not offered: the hardware +does not take them (the kext never reads them from its slice header). + +Buffers follow CHEVCDecoder: per context the four Genc buffers sized +from the picture, the intra, aux-line and stream-state buffers and the +slice structure buffer; per CAPTURE buffer the temporal MVs, fragment +pointer table, above parameters and the record the co-located picture's +words are read back from. The kext's 1 MiB slice parameter buffer per +picture is one per context here, as it is only read while its picture +decodes. + +The unit registers only when the SoC's match data gives the HEVC +parameters, and not on core revision 0x70600 (S8000 A0/A1), which needs +AppleD5500's BRN50577 dummy-picture workaround. + +Assisted-by: LLM +--- + drivers/media/platform/apple/vxd/Kconfig | 3 +- + drivers/media/platform/apple/vxd/Makefile | 2 +- + .../media/platform/apple/vxd/apple-vxd-core.c | 2 + + .../media/platform/apple/vxd/apple-vxd-hevc.c | 428 ++++++++++++++++++ + drivers/media/platform/apple/vxd/apple-vxd.h | 10 +- + 5 files changed, 441 insertions(+), 4 deletions(-) + create mode 100644 drivers/media/platform/apple/vxd/apple-vxd-hevc.c + +diff --git a/drivers/media/platform/apple/vxd/Kconfig b/drivers/media/platform/apple/vxd/Kconfig +index c08ae03ed..ef785fc88 100644 +--- a/drivers/media/platform/apple/vxd/Kconfig ++++ b/drivers/media/platform/apple/vxd/Kconfig +@@ -11,7 +11,8 @@ config VIDEO_APPLE_VXD + help + Support for the Apple "VXD" video decoder of the S8000/S8003 (A9) + SoCs: an Imagination PowerVR MSVDX core driven by the DEVA (MTX) +- firmware. It is exposed as a V4L2 stateless H.264 decoder. ++ firmware. It is exposed as a V4L2 stateless H.264 decoder, and as ++ an HEVC (H.265 Main) decoder on the SoCs whose decoder has it. + + The driver needs the DEVA firmware as apple/vxd-deva.bin. + +diff --git a/drivers/media/platform/apple/vxd/Makefile b/drivers/media/platform/apple/vxd/Makefile +index 0877a60ae..7e4fa0a78 100644 +--- a/drivers/media/platform/apple/vxd/Makefile ++++ b/drivers/media/platform/apple/vxd/Makefile +@@ -2,6 +2,6 @@ + apple-vxd-y := apple-vxd-core.o apple-vxd-gart.o apple-vxd-fw.o \ + apple-vxd-mbox.o apple-vxd-cmd.o apple-vxd-cmdbuf.o \ + apple-vxd-v4l2.o apple-vxd-h264.o apple-vxd-h264-parse.o \ +- apple-vxd-h264-cmd.o apple-vxd-hevc-cmd.o ++ apple-vxd-h264-cmd.o apple-vxd-hevc.o apple-vxd-hevc-cmd.o + + obj-$(CONFIG_VIDEO_APPLE_VXD) += apple-vxd.o +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index dff75927d..5fbca8fe8 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -422,6 +422,8 @@ static int apple_vxd_probe(struct platform_device *pdev) + } + + ret = apple_vxd_h264_register(vxd); ++ if (!ret) ++ ret = apple_vxd_hevc_register(vxd); + if (!ret) + ret = apple_vxd_v4l2_init(vxd); + if (ret) +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-hevc.c b/drivers/media/platform/apple/vxd/apple-vxd-hevc.c +new file mode 100644 +index 000000000..d3f628df1 +--- /dev/null ++++ b/drivers/media/platform/apple/vxd/apple-vxd-hevc.c +@@ -0,0 +1,428 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Apple VXD - HEVC (H.265) backend (V4L2 stateless, frame based) ++ * ++ * Userspace parses the parameter sets, the slice headers and the DPB ++ * (V4L2 stateless contract); the OUTPUT buffer carries the picture's slice ++ * NALs behind Annex-B start codes, and SLICE_PARAMS one entry per slice. ++ * Per picture this file ++ * ++ * 1. validates SPS/PPS against what the command recipe covers, ++ * 2. locates the slice NALs (apple_vxd_hevc_find_nals()), ++ * 3. resolves the DPB to CAPTURE buffers by timestamp, with their side ++ * buffers (temporal MVs and the co-located picture record), ++ * 4. builds the command stream (apple-vxd-hevc-cmd.c) and submits one ++ * 0x81 message per slice and the 0x88 message that ends the job. ++ * ++ * The context owns the buffers CHEVCDecoder::VACommandBufferInit allocates ++ * per decoder (0xfffffff0062b18f8): four Genc buffers sized from the ++ * picture, the intra and aux-line buffers, the stream state, and the slice ++ * structure buffer. Each CAPTURE buffer carries what AllocatePictureMemory ++ * allocates per picture: temporal MVs, the fragment pointer table and the ++ * above-parameter buffer. The kext also gives every picture its own 1 MiB ++ * slice parameter buffer; it is only read while that picture decodes, so ++ * one per context serves here, decodes being serialized. ++ */ ++#include ++#include ++ ++#include "apple-vxd.h" ++#include "apple-vxd-hevc-cmd.h" ++ ++/* AppleD5500 BRN50577: S8000 A0/A1 cores need a dummy picture first. */ ++#define APPLE_VXD_CORE_REV_BRN50577 0x70600 ++ ++struct apple_vxd_hevc_ctx { ++ struct apple_vxd_dev_buf genc[APPLE_VXD_HEVC_GENC_BUFS]; ++ u32 genc_size; ++ struct apple_vxd_dev_buf side; ++ struct apple_vxd_dev_buf spdest; ++ struct apple_vxd_dev_buf intra; ++ struct apple_vxd_dev_buf auxline; ++ struct apple_vxd_dev_buf stream_state; ++ bool first_slice; ++ bool first_picture; ++ struct apple_vxd_hevc_nal nals[APPLE_VXD_HEVC_MAX_SLICES]; ++}; ++ ++static const void *apple_vxd_hevc_ctrl(struct apple_vxd_ctx *ctx, u32 id, ++ unsigned int *elems) ++{ ++ struct v4l2_ctrl *ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, id); ++ ++ if (ctrl && elems) ++ *elems = ctrl->elems; ++ return ctrl ? ctrl->p_cur.p : NULL; ++} ++ ++static int apple_vxd_hevc_try_ctrl(struct apple_vxd_ctx *ctx, ++ struct v4l2_ctrl *ctrl) ++{ ++ const struct v4l2_ctrl_hevc_sps *sps = ctrl->p_new.p_hevc_sps; ++ ++ if (ctrl->id != V4L2_CID_STATELESS_HEVC_SPS) ++ return 0; ++ /* The default (all zero, before userspace sets an SPS) must pass. */ ++ if (!sps->pic_width_in_luma_samples) ++ return 0; ++ if (sps->chroma_format_idc != 1 || sps->bit_depth_luma_minus8 || ++ sps->bit_depth_chroma_minus8) { ++ dev_dbg(ctx->vxd->dev, "hevc sps: only 8-bit 4:2:0 is supported\n"); ++ return -EINVAL; ++ } ++ if (sps->pic_width_in_luma_samples > APPLE_VXD_MAX_PIC_W || ++ sps->pic_height_in_luma_samples > APPLE_VXD_MAX_PIC_H) { ++ dev_dbg(ctx->vxd->dev, "hevc sps: %ux%u too large\n", ++ sps->pic_width_in_luma_samples, sps->pic_height_in_luma_samples); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++static void apple_vxd_hevc_free(struct apple_vxd_dev *vxd, ++ struct apple_vxd_hevc_ctx *h) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < APPLE_VXD_HEVC_GENC_BUFS; i++) ++ apple_vxd_dev_buf_free(vxd, &h->genc[i]); ++ apple_vxd_dev_buf_free(vxd, &h->side); ++ apple_vxd_dev_buf_free(vxd, &h->spdest); ++ apple_vxd_dev_buf_free(vxd, &h->intra); ++ apple_vxd_dev_buf_free(vxd, &h->auxline); ++ apple_vxd_dev_buf_free(vxd, &h->stream_state); ++} ++ ++static int apple_vxd_hevc_start(struct apple_vxd_ctx *ctx) ++{ ++ const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; ++ struct apple_vxd_dev *vxd = ctx->vxd; ++ const struct apple_vxd_hevc_caps *caps = &vxd->variant->hevc; ++ struct apple_vxd_hevc_ctx *h; ++ unsigned int i; ++ int ret; ++ ++ h = kvzalloc_obj(*h); ++ if (!h) ++ return -ENOMEM; ++ ++ /* VACommandBufferInit sizes; M is the per-SoC buffer multiplier. */ ++ h->genc_size = apple_vxd_hevc_genc_size(pix->width, pix->height); ++ for (i = 0; i < APPLE_VXD_HEVC_GENC_BUFS; i++) { ++ ret = apple_vxd_dev_buf_alloc(vxd, h->genc_size, &h->genc[i]); ++ if (ret) ++ goto err; ++ } ++ ret = apple_vxd_dev_buf_alloc(vxd, APPLE_VXD_HEVC_SIDE_SIZE, &h->side); ++ if (!ret) ++ ret = apple_vxd_dev_buf_alloc(vxd, APPLE_VXD_HEVC_SPDEST_SIZE, &h->spdest); ++ if (!ret) ++ ret = apple_vxd_dev_buf_alloc(vxd, APPLE_VXD_HEVC_INTRA_SIZE, &h->intra); ++ if (!ret) ++ ret = apple_vxd_dev_buf_alloc(vxd, caps->buf_mult << 21, &h->auxline); ++ if (!ret) ++ ret = apple_vxd_dev_buf_alloc(vxd, caps->buf_mult << 12, &h->stream_state); ++ if (ret) ++ goto err; ++ ++ h->first_slice = true; ++ h->first_picture = true; ++ ctx->priv = h; ++ dev_dbg(vxd->dev, ++ "hevc: context buffers: genc 4 x %u bytes, aux line %u, stream state %u\n", ++ h->genc_size, caps->buf_mult << 21, caps->buf_mult << 12); ++ return 0; ++ ++err: ++ apple_vxd_hevc_free(vxd, h); ++ kvfree(h); ++ return ret; ++} ++ ++static void apple_vxd_hevc_stop(struct apple_vxd_ctx *ctx) ++{ ++ struct apple_vxd_hevc_ctx *h = ctx->priv; ++ ++ if (!h) ++ return; ++ apple_vxd_hevc_free(ctx->vxd, h); ++ kvfree(h); ++ ctx->priv = NULL; ++} ++ ++static enum apple_vxd_image_fmt ++apple_vxd_hevc_get_image_fmt(struct apple_vxd_ctx *ctx, struct v4l2_ctrl *ctrl) ++{ ++ if (ctrl->id != V4L2_CID_STATELESS_HEVC_SPS) ++ return APPLE_VXD_IMG_FMT_ANY; ++ ++ return APPLE_VXD_IMG_FMT_420_8BIT; ++} ++ ++static size_t apple_vxd_hevc_capture_aux_size(struct apple_vxd_ctx *ctx) ++{ ++ const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; ++ ++ return apple_vxd_hevc_aux_size(pix->width, pix->height); ++} ++ ++/* A CAPTURE buffer's side-buffer device and CPU addresses for @l. */ ++static bool apple_vxd_hevc_aux(struct vb2_v4l2_buffer *vbuf, ++ const struct apple_vxd_hevc_aux_layout *l, ++ u32 *va, struct apple_vxd_hevc_record **rec) ++{ ++ u8 *cpu = apple_vxd_buf_aux_cpu(vbuf); ++ ++ if (!cpu || apple_vxd_buf_aux_size(vbuf) < l->size) ++ return false; ++ *va = apple_vxd_buf_aux_va(vbuf); ++ *rec = (struct apple_vxd_hevc_record *)(cpu + l->record); ++ return true; ++} ++ ++static int apple_vxd_hevc_run(struct apple_vxd_ctx *ctx) ++{ ++ const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; ++ struct apple_vxd_hevc_ctx *h = ctx->priv; ++ struct apple_vxd_dev *vxd = ctx->vxd; ++ struct apple_vxd_hevc_frame f = { }; ++ struct apple_vxd_hevc_aux_layout l; ++ struct apple_vxd_cmdbuf *cb = NULL; ++ struct apple_vxd_hevc_record *rec; ++ struct apple_vxd_run run; ++ const char *why = NULL; ++ struct vb2_queue *cap_q; ++ unsigned int i, nslices = 0; ++ u32 stride, aux; ++ const u8 *src; ++ int ret; ++ ++ apple_vxd_run_preamble(ctx, &run); ++ ++ if (!vxd->hw_ready || vxd->fw_panicked) { ++ dev_dbg(vxd->dev, "decode job with no usable firmware\n"); ++ ret = -EAGAIN; ++ goto out; ++ } ++ if (!h) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ f.sps = apple_vxd_hevc_ctrl(ctx, V4L2_CID_STATELESS_HEVC_SPS, NULL); ++ f.pps = apple_vxd_hevc_ctrl(ctx, V4L2_CID_STATELESS_HEVC_PPS, NULL); ++ f.dp = apple_vxd_hevc_ctrl(ctx, V4L2_CID_STATELESS_HEVC_DECODE_PARAMS, NULL); ++ f.sm = apple_vxd_hevc_ctrl(ctx, V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, NULL); ++ f.slices = apple_vxd_hevc_ctrl(ctx, V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, &nslices); ++ if (!f.sps || !f.pps || !f.dp || !f.slices) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ret = apple_vxd_hevc_validate(f.sps, f.pps, &why); ++ if (ret) ++ goto out_why; ++ if (f.sps->pic_width_in_luma_samples > pix->width || ++ f.sps->pic_height_in_luma_samples > pix->height) { ++ why = "picture larger than the CAPTURE format"; ++ ret = -EINVAL; ++ goto out_why; ++ } ++ ++ src = vb2_plane_vaddr(&run.src->vb2_buf, 0); ++ if (!src) { ++ why = "OUTPUT buffer has no kernel mapping"; ++ ret = -EINVAL; ++ goto out_why; ++ } ++ ret = apple_vxd_hevc_find_nals(src, vb2_get_plane_payload(&run.src->vb2_buf, 0), ++ f.slices, min(nslices, APPLE_VXD_HEVC_MAX_SLICES), ++ h->nals, &why); ++ if (ret) ++ goto out_why; ++ ++ apple_vxd_hevc_aux_layout(f.sps, &l); ++ stride = pix->plane_fmt[0].bytesperline; ++ f.nals = h->nals; ++ f.nslices = nslices; ++ f.caps = &vxd->variant->hevc; ++ f.src = apple_vxd_buf_va(run.src); ++ f.dst_luma = apple_vxd_buf_va(run.dst); ++ f.dst_chroma = f.dst_luma + stride * pix->height; ++ f.dst_stride = stride; ++ f.dst_cstride = stride; ++ if (!apple_vxd_hevc_aux(run.dst, &l, &aux, &rec)) { ++ why = "CAPTURE side buffer missing or too small"; ++ ret = -EINVAL; ++ goto out_why; ++ } ++ f.mv = aux + l.mv; ++ f.frag = aux + l.frag; ++ f.above = aux + l.above; ++ f.record = rec; ++ rec->valid = 0; ++ ++ cap_q = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); ++ for (i = 0; i < f.dp->num_active_dpb_entries && i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { ++ struct vb2_buffer *ref = vb2_find_buffer(cap_q, f.dp->dpb[i].timestamp); ++ struct apple_vxd_hevc_record *rrec; ++ u32 raux; ++ ++ if (!ref || !apple_vxd_hevc_aux(to_vb2_v4l2_buffer(ref), &l, &raux, &rrec)) { ++ dev_dbg(vxd->dev, "hevc: dpb[%u] ts %llu has no CAPTURE buffer\n", ++ i, f.dp->dpb[i].timestamp); ++ continue; ++ } ++ f.refs[i].luma = apple_vxd_buf_va(to_vb2_v4l2_buffer(ref)); ++ f.refs[i].chroma = f.refs[i].luma + stride * pix->height; ++ f.refs[i].rec = rrec; ++ } ++ for (i = 0; i < APPLE_VXD_HEVC_GENC_BUFS; i++) ++ f.genc[i] = h->genc[i].va; ++ f.genc_size = h->genc_size; ++ f.side = h->side.va; ++ f.side_cpu = h->side.mem.vaddr; ++ f.side_size = APPLE_VXD_HEVC_SIDE_SIZE; ++ f.spdest = h->spdest.va; ++ f.intra = h->intra.va; ++ f.auxline = h->auxline.va; ++ f.stream_state = h->stream_state.va; ++ f.first_slice_of_session = h->first_slice; ++ f.first_picture_of_session = h->first_picture; ++ ++ cb = apple_vxd_cmdbuf_get(vxd); ++ if (!cb) { ++ why = "no free command buffer"; ++ ret = -EBUSY; ++ goto out_why; ++ } ++ ret = apple_vxd_hevc_build(cb, &f, &why); ++ if (ret) ++ goto out_why; ++ ++ dev_dbg(vxd->dev, ++ "hevc decode: src %u va %#x, %u slices (first NAL %#x+%u, header %u bits), dst %u Y %#x UV %#x stride %u, %zu command bytes in buf %u (va %#x), %u packets\n", ++ run.src->vb2_buf.index, f.src, nslices, h->nals[0].offset, ++ h->nals[0].size, h->nals[0].header_bits, run.dst->vb2_buf.index, ++ f.dst_luma, f.dst_chroma, stride, cb->used, cb->index, cb->va, ++ cb->npackets); ++ ++ /* Controls were only read while building; the job may finish any time now. */ ++ apple_vxd_run_postamble(ctx, &run); ++ ret = apple_vxd_submit(ctx, cb); ++ if (ret) { ++ dev_warn_ratelimited(vxd->dev, "hevc decode: submit failed: %d\n", ret); ++ if (cb->state == APPLE_VXD_CMDBUF_HOST) ++ apple_vxd_cmdbuf_put(vxd, cb); ++ return ret; ++ } ++ h->first_slice = false; ++ h->first_picture = false; ++ return 0; ++ ++out_why: ++ dev_warn_ratelimited(vxd->dev, "hevc decode: picture refused (%d): %s\n", ++ ret, why ? why : "?"); ++out: ++ if (cb) ++ apple_vxd_cmdbuf_put(vxd, cb); ++ apple_vxd_run_postamble(ctx, &run); ++ return ret; ++} ++ ++static const struct apple_vxd_ctrl_desc apple_vxd_hevc_ctrl_descs[] = { ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_SPS, ++ .cfg.ops = &apple_vxd_ctrl_ops, ++ }, ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_PPS, ++ }, ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, ++ .cfg.flags = V4L2_CTRL_FLAG_DYNAMIC_ARRAY, ++ .cfg.dims = { APPLE_VXD_HEVC_MAX_SLICES }, ++ }, ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, ++ }, ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS, ++ }, ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_MODE, ++ .cfg.min = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED, ++ .cfg.max = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED, ++ .cfg.def = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED, ++ }, ++ { ++ .cfg.id = V4L2_CID_STATELESS_HEVC_START_CODE, ++ .cfg.min = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B, ++ .cfg.max = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B, ++ .cfg.def = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B, ++ }, ++ { ++ .cfg.id = V4L2_CID_MPEG_VIDEO_HEVC_PROFILE, ++ .cfg.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, ++ .cfg.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE, ++ .cfg.def = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, ++ }, ++ { ++ .cfg.id = V4L2_CID_MPEG_VIDEO_HEVC_LEVEL, ++ .cfg.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1, ++ .cfg.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1, ++ .cfg.def = V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1, ++ }, ++}; ++ ++static const struct apple_vxd_ctrls apple_vxd_hevc_ctrls = { ++ .ctrls = apple_vxd_hevc_ctrl_descs, ++ .num_ctrls = ARRAY_SIZE(apple_vxd_hevc_ctrl_descs), ++}; ++ ++static const struct apple_vxd_decoded_fmt_desc apple_vxd_hevc_decoded_fmts[] = { ++ { ++ .fourcc = V4L2_PIX_FMT_NV12, ++ .image_fmt = APPLE_VXD_IMG_FMT_420_8BIT, ++ }, ++}; ++ ++static const struct apple_vxd_coded_fmt_desc apple_vxd_hevc_fmt = { ++ .fourcc = V4L2_PIX_FMT_HEVC_SLICE, ++ .frmsize = { ++ .min_width = APPLE_VXD_MIN_PIC_W, ++ .max_width = APPLE_VXD_MAX_PIC_W, ++ .step_width = 16, ++ .min_height = APPLE_VXD_MIN_PIC_H, ++ .max_height = APPLE_VXD_MAX_PIC_H, ++ .step_height = 16, ++ }, ++ .ctrls = &apple_vxd_hevc_ctrls, ++ .ops = &(const struct apple_vxd_coded_fmt_ops){ ++ .start = apple_vxd_hevc_start, ++ .stop = apple_vxd_hevc_stop, ++ .run = apple_vxd_hevc_run, ++ .try_ctrl = apple_vxd_hevc_try_ctrl, ++ .get_image_fmt = apple_vxd_hevc_get_image_fmt, ++ .capture_aux_size = apple_vxd_hevc_capture_aux_size, ++ }, ++ .num_decoded_fmts = ARRAY_SIZE(apple_vxd_hevc_decoded_fmts), ++ .decoded_fmts = apple_vxd_hevc_decoded_fmts, ++}; ++ ++/* ++ * Adds HEVC to the device's coded formats when the SoC's match data has ++ * it: AppleD5500 advertises HEVCSupported for vxd-version 7, 9 and 10. ++ */ ++int apple_vxd_hevc_register(struct apple_vxd_dev *vxd) ++{ ++ const struct apple_vxd_variant *v = vxd->variant; ++ ++ if (!v->hevc_supported || !v->hevc.buf_mult) ++ return 0; ++ if (vxd->core_rev == APPLE_VXD_CORE_REV_BRN50577) { ++ dev_info(vxd->dev, "hevc: core revision %#x needs BRN50577, not implemented\n", ++ vxd->core_rev); ++ return 0; ++ } ++ return apple_vxd_register_coded_fmt(vxd, &apple_vxd_hevc_fmt); ++} +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index f7eab496c..1d3b74382 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -11,7 +11,7 @@ + * IRQ drain, fences (apple-vxd-mbox.c) + * - command buffers and their encoders (apple-vxd-cmdbuf.c, -cmd.c) + * - the V4L2 stateless node (apple-vxd-v4l2.c) and its codec units: +- * H.264 (apple-vxd-h264*.c) ++ * H.264 (apple-vxd-h264*.c) and HEVC (apple-vxd-hevc*.c) + * + * [E] marks facts taken from the kext. + */ +@@ -40,6 +40,7 @@ + #include + + #include "apple-vxd-cmd.h" ++#include "apple-vxd-hevc-cmd.h" + #include "apple-vxd-regs.h" + + #define APPLE_VXD_DRV_NAME "apple-vxd" +@@ -62,7 +63,7 @@ enum apple_vxd_image_fmt { + APPLE_VXD_IMG_FMT_420_8BIT, + }; + +-/* One per codec unit. */ ++/* One per codec unit (H.264, HEVC). */ + #define APPLE_VXD_MAX_CODED_FMTS 2 + + /* SoC variant data (compatible table). */ +@@ -76,6 +77,8 @@ struct apple_vxd_variant { + * last client closes, and boot it again on the next open. + */ + bool sleep_stop; ++ /* HEVC buffer multiplier and command quirks; zero leaves HEVC off. */ ++ struct apple_vxd_hevc_caps hevc; + }; + + /* +@@ -511,4 +514,7 @@ extern const struct v4l2_ctrl_ops apple_vxd_ctrl_ops; + /* h264.c */ + int apple_vxd_h264_register(struct apple_vxd_dev *vxd); + ++/* hevc.c */ ++int apple_vxd_hevc_register(struct apple_vxd_dev *vxd); ++ + #endif /* __APPLE_VXD_H__ */ diff --git a/kernel/patches/drivers/media/0011-media-apple-vxd-Enable-HEVC-on-the-S8000-and-T8010-d.patch b/kernel/patches/drivers/media/0011-media-apple-vxd-Enable-HEVC-on-the-S8000-and-T8010-d.patch new file mode 100644 index 00000000..399b0925 --- /dev/null +++ b/kernel/patches/drivers/media/0011-media-apple-vxd-Enable-HEVC-on-the-S8000-and-T8010-d.patch @@ -0,0 +1,61 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 21:25:37 +0100 +Subject: [PATCH] media: apple-vxd: Enable HEVC on the S8000 and T8010 decoders + +AppleD5500 advertises HEVCSupported for vxd-version 7, 9 and 10 and keys +its HEVC choices on that version (VACreateDecodeDeviceHEVC, +0xfffffff006293f2c, called with the ADT vxd-version): the aux-line and +stream-state buffers are scaled by 2 from version 9 on, versions 7 and 9 +write 0x7474 twice after every slice structure, and the 0x701c line +split starts from 0x1800 instead of 0xc00 from version 10 on. Give the +S8000 (vxd-version 7, A9) and T8010 (10, A10) variants those values, +which turns the HEVC unit on for both. + +Neither has decoded HEVC on Linux yet. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 16 ++++++++++++++++ + 1 file changed, 16 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index 5fbca8fe8..b6bf2d300 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -305,12 +305,23 @@ static const struct dev_pm_ops apple_vxd_pm_ops = { + RUNTIME_PM_OPS(apple_vxd_runtime_suspend, apple_vxd_runtime_resume, NULL) + }; + ++/* ++ * HEVC: AppleD5500 keys its per-SoC HEVC choices on the ADT vxd-version ++ * (VACreateDecodeDeviceHEVC 0xfffffff006293f2c): the buffer multiplier ++ * (2 for versions 9 and 10), the 0x7474 pair after each slice structure ++ * (versions 7 and 9) and the 0x701c line split base (0x1800 from 10 on). ++ */ + static const struct apple_vxd_variant s8000_vxd_variant = { + .vxd_version = 7, + .num_gart_pts = 512, /* "setting 2GB mapping" [E] */ + /* HEVCSupported = ((rev - 7) < 4) && ((0xD >> (rev - 7)) & 1) [E] */ + .hevc_supported = true, + .sleep_stop = true, ++ .hevc = { ++ .buf_mult = 1, ++ .tail_7474 = true, ++ .cache_base = 0xc00, ++ }, + }; + + /* The T8010 core has two pixel pipes; it reports them in CORE_ID. */ +@@ -319,6 +330,11 @@ static const struct apple_vxd_variant t8010_vxd_variant = { + .num_gart_pts = 512, + .hevc_supported = true, + .has_integration = true, ++ .hevc = { ++ .buf_mult = 2, ++ .tail_7474 = false, ++ .cache_base = 0x1800, ++ }, + }; + + static const struct of_device_id apple_vxd_of_match[] = { diff --git a/kernel/patches/drivers/media/0012-media-apple-vxd-require-a-proven-firmware-stop-befor.patch b/kernel/patches/drivers/media/0012-media-apple-vxd-require-a-proven-firmware-stop-befor.patch new file mode 100644 index 00000000..3670d596 --- /dev/null +++ b/kernel/patches/drivers/media/0012-media-apple-vxd-require-a-proven-firmware-stop-befor.patch @@ -0,0 +1,120 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:51 +0100 +Subject: [PATCH] media: apple-vxd: require a proven firmware stop before kexec + +release has no shutdown callback and no B1 ProbeTerminate path. Add the +checked S8000 stop at shutdown: failed Terminate keeps firmware and +command buffers plus power, and kexec takes the cold-reset escape before +those pages become incoming RAM. Successful stop may release buffers and +force suspend. Other variants keep their shutdown behavior. + +Assisted-by: LLM +--- + .../media/platform/apple/vxd/apple-vxd-core.c | 26 ++++++++++++++ + .../media/platform/apple/vxd/apple-vxd-fw.c | 35 +++++++++++++++++++ + drivers/media/platform/apple/vxd/apple-vxd.h | 1 + + 3 files changed, 62 insertions(+) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-core.c b/drivers/media/platform/apple/vxd/apple-vxd-core.c +index b6bf2d300..c89e7e987 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-core.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-core.c +@@ -1,3 +1,6 @@ ++#include ++#include ++ + // SPDX-License-Identifier: GPL-2.0 + /* + * Apple VXD (Imagination MSVDX) video decoder driver - core +@@ -476,9 +479,32 @@ static void apple_vxd_remove(struct platform_device *pdev) + pm_runtime_disable(&pdev->dev); + } + ++static void apple_vxd_shutdown(struct platform_device *pdev) ++{ ++ struct apple_vxd_dev *vxd = platform_get_drvdata(pdev); ++ int ret; ++ ++ if (!vxd->variant->sleep_stop) ++ return; ++ ++ cancel_delayed_work_sync(&vxd->watchdog_work); ++ flush_work(&vxd->job_done_work); ++ ret = apple_vxd_fw_shutdown_stop(vxd); ++ if (ret) { ++ /* Outgoing allocations cannot be reserved against the next kernel. */ ++ if (kexec_in_progress) ++ apple_cold_reset("VXD Terminate failed; DMA buffers cannot cross kexec"); ++ return; /* Retain every buffer and the runtime-PM reference. */ ++ } ++ ret = pm_runtime_force_suspend(&pdev->dev); ++ if (ret) ++ dev_warn(&pdev->dev, "power off at shutdown failed: %d\n", ret); ++} ++ + static struct platform_driver apple_vxd_driver = { + .probe = apple_vxd_probe, + .remove = apple_vxd_remove, ++ .shutdown = apple_vxd_shutdown, + .driver = { + .name = APPLE_VXD_DRV_NAME, + .pm = &apple_vxd_pm_ops, +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-fw.c b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +index ede737efd..bccd04259 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-fw.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-fw.c +@@ -512,6 +512,41 @@ int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd) + return 0; + } + ++int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *vxd) ++{ ++ int ret; ++ ++ mutex_lock(&vxd->power_lock); ++ vxd->sleep_gate = true; ++ if (!vxd->fw_loaded) { ++ mutex_unlock(&vxd->power_lock); ++ return 0; ++ } ++ ++ apple_vxd_mbox_stop(vxd); ++ vxd->hw_ready = false; ++ vxd_write(vxd, VXD_REG_FW_READY_MAGIC, 0); ++ vxd_write(vxd, VXD_REG_VECT_CFG, VXD_VECT_CFG_TERMINATE); ++ ret = vxd_known_state_v1(vxd, "Terminate"); ++ if (ret) { ++ dev_err(vxd->dev, "firmware stop at shutdown failed: %d; retaining buffers and power\n", ret); ++ mutex_unlock(&vxd->power_lock); ++ return ret; ++ } ++ ++ apple_vxd_cmdbufs_fini(vxd); ++ apple_vxd_gart_unmap(vxd, &vxd->fw_map); ++ dma_free_coherent(vxd->dev, vxd->fw_size, vxd->fw_vaddr, vxd->fw_dma); ++ vxd->fw_vaddr = NULL; ++ vxd->fw_loaded = false; ++ mutex_unlock(&vxd->power_lock); ++ ++ /* The reference boot took; the PM core holds its own until complete. */ ++ pm_runtime_put_noidle(vxd->dev); ++ dev_info(vxd->dev, "firmware stopped at shutdown\n"); ++ return 0; ++} ++ + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd) + { + bool was_loaded; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index 7922ebb73..b601ea6b7 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -471,6 +471,7 @@ int apple_vxd_boot_firmware(struct apple_vxd_dev *vxd); + int apple_vxd_fw_open(struct apple_vxd_dev *vxd); + void apple_vxd_fw_close(struct apple_vxd_dev *vxd); + int apple_vxd_fw_sleep_stop(struct apple_vxd_dev *vxd); ++int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *vxd); + void apple_vxd_fw_cleanup(struct apple_vxd_dev *vxd); + + /* mbox.c */ diff --git a/kernel/patches/drivers/media/0013-media-apple-vxd-Resolve-temporal-direct-references-by-timestamp.patch b/kernel/patches/drivers/media/0013-media-apple-vxd-Resolve-temporal-direct-references-by-timestamp.patch new file mode 100644 index 00000000..5cab19eb --- /dev/null +++ b/kernel/patches/drivers/media/0013-media-apple-vxd-Resolve-temporal-direct-references-by-timestamp.patch @@ -0,0 +1,150 @@ +From 3ddce65f7c19a80e93ff87d56efb11761357ebe5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 14:26:26 +0100 +Subject: [PATCH] media: apple-vxd: Resolve temporal direct references by + timestamp + +The inverse L0 table is indexed by the DPB slots recorded in the +co-located picture's stored motion. Building it with the current +request's slots assumes that userspace preserves dpb[] ordering, which +the stateless H.264 API does not require. FFmpeg compacts its DPB and +moves the past reference when inserting the future P picture, causing +temporal direct B pictures to use the wrong reference. + +Save each picture's slot-to-CAPTURE timestamp map alongside its stored +motion buffer. Resolve the saved slots against the current L0 timestamps +when emitting the inverse table. Leave missing references unmapped and +replace the snapshot whenever the CAPTURE buffer is decoded again. + +Assisted-by: LLM +--- + .../platform/apple/vxd/apple-vxd-h264-cmd.c | 25 +++++++++++++++---- + .../platform/apple/vxd/apple-vxd-h264-cmd.h | 3 +++ + .../media/platform/apple/vxd/apple-vxd-h264.c | 18 +++++++++++++ + drivers/media/platform/apple/vxd/apple-vxd.h | 3 +++ + 4 files changed, 44 insertions(+), 5 deletions(-) + +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c +index c092a3f49..d559b7840 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c +@@ -268,7 +268,7 @@ static void vxd_h264_emit_b(struct apple_vxd_cmdbuf *cb, + { + const struct v4l2_ctrl_h264_decode_params *dp = f->dp; + u8 inverse[V4L2_H264_REF_LIST_LEN]; +- unsigned int i, n; ++ unsigned int i, j, n; + u8 col = l1[0]; + + w[0] = vxd_h264_poc(dp->top_field_order_cnt); +@@ -299,11 +299,26 @@ static void vxd_h264_emit_b(struct apple_vxd_cmdbuf *cb, + w[1] = APPLE_VXD_H264_UNUSED_VA; + } + +- /* Inverse L0: first position of each slot (bottom fields at +16). */ ++ /* ++ * Stored motion names slots from the co-located picture's DPB, not ++ * this request's DPB. Userspace may reorder dpb[] between pictures. ++ * Translate those saved slots to the current L0 by CAPTURE timestamp. ++ * References absent from L0 stay NO_REF; descending order keeps the ++ * first position when a list contains a reference more than once. ++ */ + memset(inverse, APPLE_VXD_H264_NO_REF, sizeof(inverse)); +- for (i = V4L2_H264_REF_LIST_LEN; i--; ) +- if (l0[i] != APPLE_VXD_H264_NO_REF) +- inverse[(l0[i] >> 3 & 0x10) | (l0[i] & 0x1f)] = i; ++ if (col < V4L2_H264_NUM_DPB_ENTRIES && f->dpb_ref_ts[col]) { ++ for (i = V4L2_H264_REF_LIST_LEN; i--; ) { ++ u8 slot = l0[i]; ++ ++ if (slot >= V4L2_H264_NUM_DPB_ENTRIES) ++ continue; ++ for (j = 0; j < V4L2_H264_NUM_DPB_ENTRIES; j++) ++ if ((f->dpb_ref_valid[col] & BIT(j)) && ++ f->dpb_ref_ts[col][j] == dp->dpb[slot].reference_ts) ++ inverse[j] = i; ++ } ++ } + vxd_h264_pack_list(inverse, V4L2_H264_REF_LIST_LEN, &w[2]); + n = vxd_h264_pack_list(l1, l1_minus1 + 1, &w[10]); + apple_vxd_cmd_block(cb, VXD_H264_BE_COL_PIC0, 0, w, 10 + n); +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h +index 192678d47..66e836946 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h +@@ -54,6 +54,9 @@ struct apple_vxd_h264_frame { + u32 dpb_chroma[V4L2_H264_NUM_DPB_ENTRIES]; + /* Each reference's stored-MB buffer: the B co-located picture's. */ + u32 dpb_colocated[V4L2_H264_NUM_DPB_ENTRIES]; ++ /* Saved DPB slot identities belonging to each stored-MB buffer. */ ++ const u64 *dpb_ref_ts[V4L2_H264_NUM_DPB_ENTRIES]; ++ u16 dpb_ref_valid[V4L2_H264_NUM_DPB_ENTRIES]; + struct apple_vxd_h264_bufs bufs; + bool first_picture; /* first picture of the context */ + }; +diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264.c b/drivers/media/platform/apple/vxd/apple-vxd-h264.c +index ca289e11d..c6c9df417 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd-h264.c ++++ b/drivers/media/platform/apple/vxd/apple-vxd-h264.c +@@ -319,6 +319,7 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx) + struct apple_vxd_run run; + const char *why = NULL; + struct vb2_queue *cap_q; ++ struct apple_vxd_buffer *dst; + unsigned int i, nslices; + const u8 *src; + size_t src_len; +@@ -406,6 +407,7 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx) + for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) { + const struct v4l2_h264_dpb_entry *e = &dp->dpb[i]; + struct vb2_buffer *ref; ++ struct apple_vxd_buffer *buf; + + if (!(e->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID)) + continue; +@@ -418,6 +420,10 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx) + f.dpb_luma[i] = apple_vxd_buf_va(to_vb2_v4l2_buffer(ref)); + f.dpb_chroma[i] = f.dpb_luma[i] + stride * pix->height; + f.dpb_colocated[i] = apple_vxd_buf_aux_va(to_vb2_v4l2_buffer(ref)); ++ buf = container_of(to_vb2_v4l2_buffer(ref), struct apple_vxd_buffer, ++ base.vb); ++ f.dpb_ref_ts[i] = buf->h264_ref_ts; ++ f.dpb_ref_valid[i] = buf->h264_ref_valid; + } + why = apple_vxd_h264_check_refs(h, nslices, &f); + if (why) { +@@ -435,6 +441,18 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx) + if (ret) + goto out_why; + ++ /* Keep the slot numbering used by this picture with its stored motion. */ ++ dst = container_of(run.dst, struct apple_vxd_buffer, base.vb); ++ dst->h264_ref_valid = 0; ++ for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) { ++ const struct v4l2_h264_dpb_entry *e = &dp->dpb[i]; ++ ++ dst->h264_ref_ts[i] = e->reference_ts; ++ if ((e->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) && ++ (e->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) ++ dst->h264_ref_valid |= BIT(i); ++ } ++ + dev_dbg(vxd->dev, + "decode: src %u va %#x (%zu bytes, %u slices, first %c NAL %#x+%u, header %u bits), dst %u Y %#x UV %#x stride %u, %zu command bytes in buf %u (va %#x), %u packets\n", + run.src->vb2_buf.index, f.src_va, src_len, nslices, +diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h +index 1d3b74382..7922ebb73 100644 +--- a/drivers/media/platform/apple/vxd/apple-vxd.h ++++ b/drivers/media/platform/apple/vxd/apple-vxd.h +@@ -250,6 +250,9 @@ struct apple_vxd_buffer { + * register 0x62ec), allocated with the buffer. + */ + struct apple_vxd_dev_buf aux; ++ /* DPB identity when this CAPTURE's H.264 stored motion was written. */ ++ u64 h264_ref_ts[V4L2_H264_NUM_DPB_ENTRIES]; ++ u16 h264_ref_valid; + }; + + /* +-- +2.49.0 diff --git a/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..64a11814 --- /dev/null +++ b/kernel/patches/drivers/of/0001-of-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,35 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] of: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/of/of_reserved_mem.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c +index 8c9d6395d..76cf3e5a4 100644 +--- a/drivers/of/of_reserved_mem.c ++++ b/drivers/of/of_reserved_mem.c +@@ -275,6 +275,16 @@ void __init fdt_scan_reserved_mem_late(void) + return; + } + ++ /* Early architecture fixups may have added static reservations. */ ++ total_reserved_mem_cnt = reserved_mem_count; ++ fdt_for_each_subnode(child, fdt, node) { ++ int len; ++ ++ if (of_fdt_device_is_available(fdt, child) && ++ fdt_getprop(fdt, child, "reg", &len) && len) ++ total_reserved_mem_cnt++; ++ } ++ + /* Attempt dynamic allocation of a new reserved_mem array */ + if (alloc_reserved_mem_array()) + return; diff --git a/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..c4b7b0ee --- /dev/null +++ b/kernel/patches/drivers/pci/0033-dt-bindings-pci-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,28 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: pci: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Describe the board-selected retained handoff. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +index f6497a4e1..f9b72ce26 100644 +--- a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml ++++ b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +@@ -140,6 +140,10 @@ allOf: + - apple,nvme-hmb + + properties: ++ apple,ans-kexec-handoff: ++ type: boolean ++ description: Retain storage port power across a certified stopped-DMA kexec handoff. ++ + compatible: + oneOf: + - enum: diff --git a/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..e030657f --- /dev/null +++ b/kernel/patches/drivers/pci/0034-PCI-apple-h9p-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,668 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] PCI: apple-h9p: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +The storage port is handed over powered, with its reference clock running, +PERST# asserted and the LTSSM stopped: the state the boot loader leaves it +in, except that the link is down. The incoming kernel brings it up as on a +cold boot, with the full port power cycle, NVMe mode and the port enable +sequence before the link is trained. iOS trains this link only after that +sequence and never re-trains a powered port by toggling PERST# alone; +doing so after a kexec leaves the root port in Detect.Quiet with the LTSSM +enable not latched. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 459 +++++++++++++++++++++--- + include/linux/apple-ans.h | 23 ++ + 2 files changed, 432 insertions(+), 50 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 15f7a0cbe..f0095394a 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -15,6 +15,8 @@ + + #include + #include ++#include ++#include + #include + #include + #include +@@ -38,12 +40,15 @@ + #include + #include + #include ++#include + #include + #include + #include ++#include + + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -248,7 +253,7 @@ + #define S8003_PORT_LINKCDMSTS_LTSSM GENMASK(14, 9) + #define S8003_LTSSM_L0 0x11 + +-/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (j71t) */ ++/* ADT perst-to-refclk-delay-us and pci-bridge t-refclk-to-perst (S8003) */ + #define S8003_PERST_TO_REFCLK_US 60 + #define S8003_REFCLK_TO_PERST_US 100 + #define S8003_PERST_HOLD_MS 100 +@@ -269,7 +274,7 @@ + + /* + * NVMMU invalidate FIFO status, in the port 0 block (iBoot sets the live ADT +- * nvme-mmu0 invalidate-fifo-workaround = 1 on j71t). iOS (AppleNVMeMMU, ++ * nvme-mmu0 invalidate-fifo-workaround = 1 on S8003). iOS (AppleNVMeMMU, + * 0xfffffff005eebc9c) waits up to 20 ms for it to drain before every + * invalidate and never reads the invalidate register back. + */ +@@ -429,7 +434,7 @@ static const struct apple_h9p_tunable t7000_port_tunables[] = { + }; + + /* +- * S8003 tunables, which iBoot puts in the live ADT; copied from the j71t ++ * S8003 tunables, which iBoot puts in the live ADT; copied from the S8003 + * live ADT (iOS 16.7.10 iBoot), where both bridges carry the same values. + * apcie-phy-tunables and apcie-common-tunables apply after the common init, + * in that order. +@@ -582,6 +587,7 @@ struct apple_h9p_tunables { + struct apple_h9p_nvmmu { + struct apple_h9p_pcie *pcie; + void __iomem *base; ++ bool poisoned; + /* Programmed: the ANS glue may map through it */ + bool ready; + /* Serializes the invalidate FIFO wait and the invalidate write */ +@@ -711,6 +717,7 @@ struct apple_h9p_pcie { + u32 late_ports; + unsigned int late_tries; + struct delayed_work late_work; ++ bool ans_handoff_stopped; + + struct apple_h9p_nvmmu nvmmu; + +@@ -1554,6 +1561,64 @@ static irqreturn_t apple_h9p_nvmmu_irq(int irq, void *data) + return IRQ_HANDLED; + } + ++static int apple_h9p_nvmmu_program(struct apple_h9p_pcie *pcie) ++{ ++ struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; ++ u32 val; ++ int ret; ++ ++ if (pcie->hw->nvmmu_inv_fifo) { ++ /* ++ * iOS's S8003 order: the invalidate FIFO drained, the old SART ++ * aperture off before its address changes, the zeroed scratch ++ * page and TCBs visible, then scratch page, TCB base, init, SART. ++ */ ++ ret = readl_poll_timeout(pcie->base_port[0] + ++ S8003_PORT_NVMMU_INV_FIFO, val, ++ val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ dma_wmb(); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ } else { ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), ++ nvmmu->base + H9P_NVMMU_TCB_BASE_HI); ++ h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); ++ h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), ++ nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); ++ } ++ h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, ++ nvmmu->base + H9P_NVMMU_TCB_CTRL); ++ ++ ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, ++ pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ if (ret) ++ return ret; ++ ++ h9p_writel_flush(nvmmu->va_base - 0x80000000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_BASE); ++ h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, ++ H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, ++ nvmmu->base + H9P_NVMMU_SART_VA_END); ++ h9p_writel_flush(nvmmu->pa_base >> 20, ++ nvmmu->base + H9P_NVMMU_SART_PA_BASE); ++ h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); ++ ++ return 0; ++} ++ + static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + { + struct apple_h9p_nvmmu *nvmmu = &pcie->nvmmu; +@@ -1561,7 +1626,7 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + struct device_node *mem_np; + struct resource res; + resource_size_t size; +- u32 iova, val; ++ u32 iova; + int irq; + int ret; + +@@ -1630,55 +1695,10 @@ static int apple_h9p_setup_nvmmu(struct apple_h9p_pcie *pcie) + } + + spin_lock_init(&nvmmu->inv_lock); +- if (pcie->hw->nvmmu_inv_fifo) { +- /* +- * iOS's S8003 order: the invalidate FIFO drained, the old SART +- * aperture off before its address changes, the zeroed scratch +- * page and TCBs visible, then scratch page, TCB base, init, SART. +- */ +- ret = readl_poll_timeout(pcie->base_port[0] + +- S8003_PORT_NVMMU_INV_FIFO, val, +- val & S8003_PORT_NVMMU_INV_FIFO_EMPTY, +- 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); +- if (ret) +- goto out_put_node; +- h9p_writel_flush(0, nvmmu->base + H9P_NVMMU_SART_CTRL); +- dma_wmb(); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- } else { +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_dma), +- nvmmu->base + H9P_NVMMU_TCB_BASE_HI); +- h9p_writel_flush(lower_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_LO); +- h9p_writel_flush(upper_32_bits(nvmmu->tcb_table_dma), +- nvmmu->base + H9P_NVMMU_TCB_TABLE_HI); +- } +- h9p_writel_flush(H9P_NVMMU_TCB_CTRL_INIT, +- nvmmu->base + H9P_NVMMU_TCB_CTRL); +- +- ret = apple_h9p_wait(nvmmu->base + H9P_NVMMU_TCB_CTRL, +- pcie->hw->nvmmu_init_busy, 0, 0, 250000); ++ ret = apple_h9p_nvmmu_program(pcie); + if (ret) + goto out_put_node; + +- h9p_writel_flush(nvmmu->va_base - 0x80000000U, +- nvmmu->base + H9P_NVMMU_SART_VA_BASE); +- h9p_writel_flush(round_up(nvmmu->va_base + nvmmu->size, +- H9P_NVMMU_SART_ALIGNMENT) - 0x80100000U, +- nvmmu->base + H9P_NVMMU_SART_VA_END); +- h9p_writel_flush(nvmmu->pa_base >> 20, +- nvmmu->base + H9P_NVMMU_SART_PA_BASE); +- h9p_writel_flush(1, nvmmu->base + H9P_NVMMU_SART_CTRL); +- + irq = platform_get_irq_byname_optional(pcie->pdev, "nvmmu"); + if (irq > 0) { + ret = devm_request_irq(dev, irq, apple_h9p_nvmmu_irq, 0, +@@ -1726,6 +1746,7 @@ static int apple_h9p_nvmmu_invalidate(struct apple_h9p_pcie *pcie, + spin_unlock_irqrestore(&nvmmu->inv_lock, flags); + + if (ret) { ++ WRITE_ONCE(nvmmu->poisoned, true); + dev_err_ratelimited(pcie->dev, + "NVMMU invalidate FIFO not empty (%#x), tag %u\n", + val, tag); +@@ -1781,6 +1802,8 @@ int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + nvmmu = &pcie->nvmmu; + if (!nvmmu->ready) + return -EOPNOTSUPP; ++ if (READ_ONCE(nvmmu->poisoned)) ++ return -EIO; + + tcb = (u32 *)nvmmu->tcb + tag * H9P_NVMMU_TCB_DWORDS; + sgl = (u32 *)nvmmu->tcb_sgl + tag * H9P_NVMMU_SGL_WORDS; +@@ -2705,6 +2728,8 @@ static int apple_s8003_power_cycle_port(struct apple_h9p_pcie *pcie, + return ret; + } + ++ if (!port && pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); + apple_s8003_port_off(pcie, port); + usleep_range(S8003_POWER_OFF_SETTLE_US, S8003_POWER_OFF_SETTLE_US + 1000); + ret = apple_h9p_wait(rc + S8003_RC_PORT_CTL(port), +@@ -2887,6 +2912,18 @@ static int apple_s8003_start_link(struct apple_h9p_pcie *pcie, + return ret; + } + ++static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *rc = pcie->base_rc; ++ ++ return (readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_ENABLE) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_ON) && ++ (readl(rc + S8003_RC_PORT_REFCLK(0)) & S8003_RC_PORT_REFCLK_ON) && ++ !(readl(rc + S8003_RC_PORT_CTL(0)) & S8003_RC_PORT_CTL_RESET) && ++ (readl(rc + S8003_RC_PORT_PWR(0)) & S8003_RC_PORT_PWR_REFCLK_NOGATE) && ++ (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -3059,6 +3096,11 @@ static void apple_s8003_late_ports_work(struct work_struct *work) + unsigned int port; + int ret; + ++ /* Lab 0019: no secondary bus publication without storage bus 1. */ ++ if (of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") && ++ !(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return; ++ + for_each_set_bit(port, &pending, H9P_NUM_PORTS) { + ret = apple_s8003_setup_endpoint_port(pcie, port); + if (ret == -EPROBE_DEFER) { +@@ -3418,6 +3460,314 @@ static void apple_h9p_pcie_pwrctrl_cleanup(void *data) + pci_pwrctrl_destroy_devices(pcie->dev); + } + ++/* SPDX-License-Identifier: GPL-2.0-only */ ++static struct apple_h9p_pcie *apple_ans_endpoint_host(struct device *dev) ++{ ++ struct device *host = dev; ++ struct apple_h9p_pcie *pcie; ++ ++ while (host && host->bus == dev->bus) ++ host = host->parent; ++ if (!host || !host->parent) ++ return NULL; ++ pcie = apple_h9p_pcie_lookup(host->parent); ++ if (!pcie || !of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie") || ++ apple_h9p_pcie_device_port(pcie, dev) != 0) ++ return NULL; ++ return pcie; ++} ++ ++ ++/* release6: caller froze and synchronized I/O, before SHN. */ ++/* Lab 0018: caller synchronized queues and stopped CC/RDY before this. */ ++int apple_ans_nvmmu_reset(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ struct apple_h9p_nvmmu *nvmmu; ++ int ret; ++ ++ if (!pcie || !(pcie->enabled_ports & BIT(0))) ++ return -ENODEV; ++ nvmmu = &pcie->nvmmu; ++ if (!nvmmu->ready) ++ return -ENODEV; ++ memset(nvmmu->tcb, 0, nvmmu->tcb_size); ++ memset(nvmmu->tcb_table, 0, nvmmu->tcb_table_size); ++ memset(nvmmu->tcb_sgl, 0, nvmmu->tcb_sgl_size); ++ ret = apple_h9p_nvmmu_program(pcie); ++ WRITE_ONCE(nvmmu->poisoned, !!ret); ++ if (ret) ++ return ret; ++ bitmap_zero(nvmmu->mapped, APPLE_ANS_NVMMU_MAX_REQS); ++ dev_info(dev, "NVMMU: stopped-controller cache/SART rebuilt\n"); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_reset); ++ ++bool apple_ans_is_s8003(struct device *dev) ++{ ++ return apple_ans_endpoint_host(dev) != NULL; ++} ++EXPORT_SYMBOL_GPL(apple_ans_is_s8003); ++ ++int apple_ans_link_up(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -ENODEV; ++ if (!(READ_ONCE(pcie->enabled_ports) & BIT(0))) ++ return -ENOLINK; ++ return !!(readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & ++ S8003_PORT_LINKSTS_UP); ++} ++EXPORT_SYMBOL_GPL(apple_ans_link_up); ++ ++/* Host-only evidence, no endpoint config/BAR reads or invalidate register. */ ++void apple_ans_host_dump(struct device *dev, const char *reason) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return; ++ dev_info(dev, "ANS host %s: ports %#x link %#x LTSSM %#x FIFO %#x poisoned %u mapped %*pb\n", ++ reason, pcie->enabled_ports, ++ readl(pcie->base_port[0] + H9P_PORT_LINKSTS), ++ readl(pcie->base_port[0] + H9P_PORT_LTSSMCTL), ++ readl(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO), ++ READ_ONCE(pcie->nvmmu.poisoned), APPLE_ANS_NVMMU_MAX_REQS, ++ pcie->nvmmu.mapped); ++} ++EXPORT_SYMBOL_GPL(apple_ans_host_dump); ++ ++int apple_ans_nvmmu_drain(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ u32 fifo; ++ int ret; ++ ++ if (!pcie || !pcie->nvmmu.ready) ++ return -ENODEV; ++ if (!bitmap_empty(pcie->nvmmu.mapped, APPLE_ANS_NVMMU_MAX_REQS)) ++ return -EBUSY; ++ ret = readl_poll_timeout(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 10, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ dev_info(dev, "Apple ANS: outgoing NVMMU FIFO empty check (20ms) result %d\n", ret); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_nvmmu_drain); ++ ++int apple_ans_handoff_stopped(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ unsigned int tag; ++ u32 fifo; ++ int ret; ++ ++ if (!pcie) ++ return -ENODEV; ++ /* All queues are drained and BME is clear before retiring translations. */ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } ++ /* The last invalidate also has to leave the FIFO before certification. */ ++ ret = readl_poll_timeout_atomic(pcie->base_port[0] + S8003_PORT_NVMMU_INV_FIFO, ++ fifo, fifo & S8003_PORT_NVMMU_INV_FIFO_EMPTY, ++ 1, S8003_NVMMU_INV_FIFO_TIMEOUT_US); ++ if (ret) ++ return ret; ++ ret = apple_ans_dart_stop(dev); ++ if (!ret) ++ WRITE_ONCE(pcie->ans_handoff_stopped, true); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_stopped); ++ ++int apple_ans_handoff_begin(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_ans_endpoint_host(dev); ++ ++ if (!pcie) ++ return -EBUSY; ++ WRITE_ONCE(pcie->ans_handoff_stopped, false); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); ++ ++static void apple_h9p_s8003_shutdown(struct platform_device *pdev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ unsigned int port; ++ ++ if (!kexec_in_progress || !pcie || ++ !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) ++ return; ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!READ_ONCE(pcie->ans_handoff_stopped) || ++ (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && ++ !apple_s8003_port_powered(pcie))) ++ panic("Apple ANS: refusing uncertified powered handoff"); ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; ++ ++ if (!(pcie->enabled_ports & BIT(port))) ++ continue; ++ /* release6: block root routing, retain refclk, then isolate. */ ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[port], 1)) ++ panic("Apple PCIe: PERST failed during handoff"); ++ if (apple_s8003_port_stop(pcie, port)) ++ panic("Apple PCIe: LTSSM stop failed during handoff"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (!port && pcie->nvmmu.ready) { ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ } ++ if (!port && !of_property_read_bool(pcie->dev->of_node, ++ "apple,ans-kexec-handoff")) ++ apple_s8003_port_off(pcie, port); ++ if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && ++ (!apple_s8003_port_powered(pcie) || ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) ++ panic("Apple ANS: outgoing powered port certificate failed"); ++ pcie->enabled_ports &= ~BIT(port); ++ } ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++} ++ ++static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) ++{ ++ void __iomem *cfg = pcie->base_config; ++ ++ cancel_delayed_work_sync(&pcie->late_work); ++ if (!(pcie->enabled_ports & BIT(0))) ++ return; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ if (gpiod_direction_output(pcie->perst[0], 1) || ++ apple_s8003_port_stop(pcie, 0)) ++ panic("ANS capture isolation failed"); ++ udelay(S8003_PERST_TO_REFCLK_US); ++ if (pcie->nvmmu.base) ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->nvmmu.ready = false; ++ pcie->enabled_ports &= ~BIT(0); ++} ++ ++struct apple_h9p_crash_state { ++ struct apple_h9p_pcie *pcie; ++ int result; ++ bool busy; ++ bool found; ++}; ++ ++/* No NVMe driver, queues, SHN, or old DMA buffers are needed in phoenix. */ ++static int apple_h9p_crash_stop_ans(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_crash_state *state = data; ++ struct apple_h9p_pcie *pcie = state->pcie; ++ void __iomem *bar; ++ u32 csts = 0; ++ int i, ret; ++ ++ if (pdev->vendor != PCI_VENDOR_ID_APPLE || ++ ((pdev->class >> 8) != PCI_CLASS_STORAGE_EXPRESS && ++ (pdev->class >> 8) != PCI_CLASS_STORAGE_OTHER)) ++ return 0; ++ state->found = true; ++ if (pdev->driver) { ++ state->busy = true; ++ return 1; ++ } ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ state->result = -ENOLINK; ++ return 0; ++ } ++ ret = pci_enable_device_mem(pdev); ++ if (ret) { ++ state->result = ret; ++ return 0; ++ } ++ bar = pci_iomap_range(pdev, 0, 0, SZ_8K); ++ if (!bar) { ++ ret = -ENOMEM; ++ goto disable; ++ } ++ /* An unknown queue must never be rung or sent a shutdown notification. */ ++ dev_info(pcie->dev, "phoenix ANS: CC %#x CSTS %#x, clearing EN without SHN\n", ++ readl(bar + NVME_REG_CC), readl(bar + NVME_REG_CSTS)); ++ writel(0, bar + NVME_REG_CC); ++ ret = -ETIMEDOUT; ++ for (i = 0; i < 2000; i++) { ++ if (apple_ans_link_up(&pdev->dev) <= 0) { ++ ret = -ENOLINK; ++ break; ++ } ++ csts = readl(bar + NVME_REG_CSTS); ++ if (csts == U32_MAX) { ++ ret = -EIO; ++ break; ++ } ++ if (!(csts & NVME_CSTS_RDY)) { ++ ret = 0; ++ break; ++ } ++ usleep_range(1000, 1100); ++ } ++ dev_info(pcie->dev, "phoenix ANS: disable result %d CSTS %#x; blocking bus master\n", ++ ret, csts); ++ pci_iounmap(pdev, bar); ++disable: ++ pci_clear_master(pdev); ++ pci_disable_device(pdev); ++ state->result = ret; ++ return 0; ++} ++ ++static ssize_t crash_quiesce_store(struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ struct apple_h9p_crash_state state = { .pcie = pcie }; ++ ++ if (!pcie || !is_kdump_kernel() || ++ !of_device_is_compatible(dev->of_node, "apple,s8003-pcie") || ++ !sysfs_streq(buf, "1")) ++ return -EPERM; ++ /* The endpoint gate must stay closed; don't park under a live driver. */ ++ if (pcie->late_ports || work_busy(&pcie->late_work.work)) ++ return -EBUSY; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_crash_stop_ans, &state); ++ if (state.busy) ++ return -EBUSY; ++ /* Isolation is still required after a bounded RDY timeout. */ ++ /* Wi-Fi stays linked until its own shutdown callback has run. */ ++ apple_h9p_capture_park(pcie); ++ dev_info(dev, "phoenix: ANS/PCIe quiesced (controller found %d result %d)\n", ++ state.found, state.result); ++ return count; ++} ++static DEVICE_ATTR_WO(crash_quiesce); ++ ++static struct attribute *apple_h9p_crash_attrs[] = { ++ &dev_attr_crash_quiesce.attr, ++ NULL, ++}; ++ ++static const struct attribute_group apple_h9p_crash_group = { ++ .attrs = apple_h9p_crash_attrs, ++}; ++ ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3426,6 +3776,7 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + u32 doorbell; + int ret; + ++ + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; +@@ -3578,6 +3929,13 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (ret) + return ret; + ++ if (is_kdump_kernel() && of_device_is_compatible(pcie->dev->of_node, "apple,s8003-pcie")) { ++ ret = devm_device_add_group(dev, &apple_h9p_crash_group); ++ if (ret) ++ return ret; ++ } ++ ++ + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +@@ -3660,6 +4018,7 @@ MODULE_DEVICE_TABLE(of, apple_h9p_pcie_of_match); + + static struct platform_driver apple_h9p_pcie_driver = { + .probe = apple_h9p_pcie_probe, ++ .shutdown = apple_h9p_s8003_shutdown, + .driver = { + .name = "pcie-apple-h9p", + .of_match_table = apple_h9p_pcie_of_match, +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index a59861010..10be3ddef 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -17,11 +17,34 @@ struct device; + * Map @npages pages for FlatDMA request @tag, or retire the tag if @npages is + * zero. @to_memory: the controller writes the pages (an NVMe read). + */ ++#if IS_REACHABLE(CONFIG_APPLE_DART) ++int apple_ans_dart_stop(struct device *dev); ++#else ++static inline int apple_ans_dart_stop(struct device *dev) ++{ ++ return -EOPNOTSUPP; ++} ++#endif ++ + #if IS_REACHABLE(CONFIG_PCIE_APPLE_H9P) ++int apple_ans_handoff_begin(struct device *dev); ++int apple_ans_handoff_stopped(struct device *dev); ++int apple_ans_nvmmu_drain(struct device *dev); ++int apple_ans_nvmmu_reset(struct device *dev); ++int apple_ans_link_up(struct device *dev); ++bool apple_ans_is_s8003(struct device *dev); ++void apple_ans_host_dump(struct device *dev, const char *reason); + int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova); + #else ++static inline int apple_ans_handoff_begin(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_handoff_stopped(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_drain(struct device *dev) { return -EBUSY; } ++static inline int apple_ans_nvmmu_reset(struct device *dev) { return -EOPNOTSUPP; } ++static inline int apple_ans_link_up(struct device *dev) { return 1; } ++static inline bool apple_ans_is_s8003(struct device *dev) { return false; } ++static inline void apple_ans_host_dump(struct device *dev, const char *reason) {} + static inline int apple_ans_nvmmu_map(struct device *dev, unsigned int tag, + const u64 *pages, unsigned int npages, + bool to_memory, dma_addr_t *iova) diff --git a/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch b/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch new file mode 100644 index 00000000..62d5bda6 --- /dev/null +++ b/kernel/patches/drivers/pci/0035-nvme-pci-keep-the-certified-ANS-kexec-handoff.patch @@ -0,0 +1,337 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] nvme-pci: keep the certified ANS kexec handoff + +Resetting the ANS host or its translations destroys the state required +by the next kernel. Retain the stopped-DMA handoff only where the device +tree requests it. + +Assisted-by: LLM +--- + drivers/nvme/host/pci-apple-ans.c | 33 +++++++- + drivers/nvme/host/pci-internal.h | 2 + + drivers/nvme/host/pci.c | 128 ++++++++++++++++++++++++++++-- + 3 files changed, 155 insertions(+), 8 deletions(-) + +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index 5ba239e47..fdd9fa091 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -84,6 +84,7 @@ struct apple_ans_nvme { + u32 cc; + unsigned long cc_jiffies; + bool cc_written; ++ bool host_fault; + dma_addr_t hmb_dma; + u32 hmb_size; + struct apple_ans_nvme_req req[APPLE_ANS_NVMMU_MAX_REQS]; +@@ -218,13 +219,20 @@ static int nvme_pci_apple_ans_reset_dma(struct nvme_dev *dev) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long flags; +- unsigned int tag; + int ret; + +- for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { +- ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ans->host == &apple_ans_host_s8003) { ++ ret = apple_ans_nvmmu_reset(dev->dev); + if (ret) + return ret; ++ } else { ++ unsigned int tag; ++ ++ for (tag = 0; tag < APPLE_ANS_NVMMU_MAX_REQS; tag++) { ++ ret = apple_ans_nvmmu_map(dev->dev, tag, NULL, 0, false, NULL); ++ if (ret) ++ return ret; ++ } + } + + spin_lock_irqsave(&ans->req_lock, flags); +@@ -326,10 +334,24 @@ static int nvme_pci_apple_ans_prepare_enable(struct nvme_dev *dev) + return 0; + } + ++static int nvme_pci_apple_ans_check_link(struct nvme_dev *dev) ++{ ++ struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); ++ ++ if (ans && ans->host == &apple_ans_host_s8003 && ++ (ans->host_fault || apple_ans_link_up(dev->dev) <= 0)) { ++ ans->host_fault = true; ++ return -EIO; ++ } ++ return 0; ++} ++ + static int nvme_pci_apple_ans_write_cc(struct nvme_dev *dev, u32 cc) + { + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + writel(cc, dev->bar + NVME_REG_CC); + if (!ans) + return 0; +@@ -349,6 +371,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + struct apple_ans_nvme *ans = nvme_pci_apple_ans(dev); + unsigned long deadline; + ++ if (nvme_pci_apple_ans_check_link(dev)) ++ return -EIO; + *csts = readl(dev->bar + NVME_REG_CSTS); + if (!ans || !ans->cc_written || !ans->host->rdy_timeout_ms || + (ans->cc & NVME_CC_SHN_MASK)) +@@ -360,6 +384,8 @@ static int nvme_pci_apple_ans_read_csts(struct nvme_dev *dev, u32 *csts) + time_after(jiffies, deadline)) { + dev_err(dev->dev, "ANS RDY timeout (CC %#x, CSTS %#x)\n", + ans->cc, *csts); ++ ans->host_fault = true; ++ apple_ans_host_dump(dev->dev, "RDY deadline"); + return -ETIMEDOUT; + } + +@@ -419,6 +445,7 @@ static int nvme_pci_apple_ans_free_req(struct nvme_dev *dev, + dev_err_ratelimited(dev->dev, + "failed to release ANS DMA slot %u: %d\n", + tag, ret); ++ nvme_reset_ctrl(&dev->ctrl); + return ret; + } + +diff --git a/drivers/nvme/host/pci-internal.h b/drivers/nvme/host/pci-internal.h +index a62777abd..d3fe17867 100644 +--- a/drivers/nvme/host/pci-internal.h ++++ b/drivers/nvme/host/pci-internal.h +@@ -74,6 +74,8 @@ struct nvme_dev { + struct nvme_ctrl ctrl; + u32 last_ps; + bool hmb; ++ unsigned long apple_ans_flags; ++ atomic_t apple_ans_irq_count; + struct sg_table *hmb_sgt; + mempool_t *dmavec_mempool; + const struct nvme_pci_dma_ops *dma_ops; +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 9ffe2fb74..40b2448d1 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -5,6 +5,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -17,6 +18,10 @@ + #include + #include + #include ++#include ++#include ++#include ++#include + #include + #include + #include +@@ -1559,6 +1564,9 @@ static irqreturn_t nvme_irq(int irq, void *data) + DEFINE_IO_COMP_BATCH(iob); + bool found; + ++ if (!nvmeq->qid && dev->dma_ops == &nvme_pci_apple_ans_ops) ++ atomic_inc(&dev->apple_ans_irq_count); ++ + if (nvme_apple_ans_shared_vector(dev)) + found = nvme_apple_ans_irq_vector(dev, nvmeq->cq_vector, &iob); + else +@@ -1836,6 +1844,40 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) + "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off pcie_port_pm=off\" and report a bug\n"); + } + ++/* No endpoint MMIO/config or doorbell reads: safe first evidence for ANS. */ ++static void nvme_apple_ans_dump(struct nvme_dev *dev, const char *reason) ++{ ++ unsigned int i; ++ ++ dev_info(dev->dev, "Apple ANS: snapshot %s state %d online %u allocated %u vectors %u recovery_used %u\n", ++ reason, nvme_ctrl_state(&dev->ctrl), dev->online_queues, ++ dev->ctrl.queue_count, dev->num_vecs, ++ test_bit(0, &dev->apple_ans_flags)); ++ apple_ans_host_dump(dev->dev, reason); ++ for (i = 0; i < dev->ctrl.queue_count; i++) { ++ struct nvme_queue *q = &dev->queues[i]; ++ u16 head = READ_ONCE(q->cq_head); ++ struct nvme_completion *cqe; ++ int irq = -1; ++ ++ if (!q->cqes || head >= q->q_depth) ++ continue; ++ cqe = &q->cqes[head]; ++ if (test_bit(NVMEQ_ENABLED, &q->flags) && ++ !test_bit(NVMEQ_POLLED, &q->flags)) ++ irq = pci_irq_vector(to_pci_dev(dev->dev), q->cq_vector); ++ dev_info(dev->dev, "Apple ANS: q%u flags %#lx depth %u SQ tail %u/%u CQ head %u phase %u next status %#x cid %#x sqhead %u vector %u irq %d ANS admin IRQ actions %u SQ %pad CQ %pad\n", ++ i, READ_ONCE(q->flags), q->q_depth, ++ READ_ONCE(q->sq_tail), READ_ONCE(q->last_sq_tail), ++ head, READ_ONCE(q->cq_phase), ++ le16_to_cpu(READ_ONCE(cqe->status)), ++ le16_to_cpu(READ_ONCE(cqe->command_id)), ++ le16_to_cpu(READ_ONCE(cqe->sq_head)), q->cq_vector, ++ irq, atomic_read(&dev->apple_ans_irq_count), ++ &q->sq_dma_addr, &q->cq_dma_addr); ++ } ++} ++ + static enum blk_eh_timer_return nvme_timeout(struct request *req) + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); +@@ -1844,8 +1886,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + struct request *abort_req; + struct nvme_command cmd = { }; + struct pci_dev *pdev = to_pci_dev(dev->dev); +- u32 csts = readl(dev->bar + NVME_REG_CSTS); ++ u32 csts = ~0U; + u8 opcode; ++ bool ans = dev->dma_ops == &nvme_pci_apple_ans_ops && ++ apple_ans_is_s8003(dev->dev); ++ ++ if (ans) { ++ nvme_apple_ans_dump(dev, "timeout before CSTS"); ++ if (dev->dma_ops->read_csts(dev, &csts)) ++ goto disable; ++ } else { ++ csts = readl(dev->bar + NVME_REG_CSTS); ++ } + + /* + * Shutdown the device immediately if we see it is disconnected. This +@@ -1921,6 +1973,11 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) + * returned to the driver, or if this is the admin queue. + */ + opcode = nvme_req(req)->cmd->common.opcode; ++ if (ans) { ++ if (test_and_set_bit(0, &dev->apple_ans_flags)) ++ nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); ++ goto disable; ++ } + if (!nvmeq->qid || (iod->flags & IOD_ABORTED)) { + dev_warn(dev->ctrl.device, + "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, reset controller\n", +@@ -2804,6 +2861,7 @@ static umode_t nvme_pci_attrs_are_visible(struct kobject *kobj, + return a->mode; + } + ++ + static struct attribute *nvme_pci_attrs[] = { + &dev_attr_cmb.attr, + &dev_attr_cmbloc.attr, +@@ -3283,11 +3341,46 @@ static bool nvme_pci_ctrl_is_dead(struct nvme_dev *dev) + return (csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY); + } + +-static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++/* Certify stopped DMA before the kexec PCIe handoff. */ ++static int nvme_apple_ans_handoff_disable(struct nvme_dev *dev, int shutdown_ret) ++{ ++ struct pci_dev *pdev = to_pci_dev(dev->dev); ++ u32 csts = ~0U, cc, expected; ++ int ret, err; ++ ++ ret = dev->dma_ops->read_csts(dev, &csts); ++ if (shutdown_ret || ret || (csts & NVME_CSTS_CFS) || ++ (csts & NVME_CSTS_SHST_MASK) != NVME_CSTS_SHST_CMPLT) { ++ pci_clear_master(pdev); ++ return -EBUSY; ++ } ++ dev->ctrl.ctrl_config &= ~(NVME_CC_ENABLE | NVME_CC_SHN_MASK); ++ expected = dev->ctrl.ctrl_config; ++ ret = dev->dma_ops->write_cc(dev, expected); ++ if (!ret) { ++ ret = read_poll_timeout(dev->dma_ops->read_csts, err, ++ err || !(csts & NVME_CSTS_RDY), ++ 1000, 2000000, false, dev, &csts); ++ if (!ret) ++ ret = err; ++ } ++ if (ret) ++ goto block_dma; ++ cc = readl(dev->bar + NVME_REG_CC); ++ if ((csts & NVME_CSTS_CFS) || ++ (cc & (NVME_CC_ENABLE | NVME_CC_SHN_MASK))) ++ ret = -EBUSY; ++block_dma: ++ pci_clear_master(pdev); ++ return ret; ++} ++ ++static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool paired) + { + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); + dead = nvme_pci_ctrl_is_dead(dev); +@@ -3298,15 +3391,26 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown) +- nvme_wait_freeze_timeout(&dev->ctrl); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) ++ panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); + } + + nvme_quiesce_io_queues(&dev->ctrl); + ++ if (paired && (dead || !dev->ctrl.queue_count)) ++ panic("Apple ANS: dead outgoing controller; refusing kexec"); + if (!dead && dev->ctrl.queue_count > 0) { ++ if (paired) { ++ nvme_sync_io_queues(&dev->ctrl); ++ ret = apple_ans_nvmmu_drain(dev->dev); ++ if (ret) ++ panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ } + nvme_delete_io_queues(dev); +- nvme_disable_ctrl(&dev->ctrl, shutdown); ++ dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); ++ ret = nvme_disable_ctrl(&dev->ctrl, shutdown); ++ if (paired) ++ ret = nvme_apple_ans_handoff_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } + nvme_suspend_io_queues(dev); +@@ -3329,7 +3433,15 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) + if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) + nvme_unquiesce_admin_queue(&dev->ctrl); + } ++ if (paired && !ret) ++ ret = apple_ans_handoff_stopped(dev->dev); + mutex_unlock(&dev->shutdown_lock); ++ return ret; ++} ++ ++static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) ++{ ++ nvme_dev_disable_paired(dev, shutdown, false); + } + + static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) +@@ -3920,6 +4032,12 @@ static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); + ++ if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) ++ panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + nvme_disable_prepare_reset(dev, true); + } + diff --git a/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch b/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch new file mode 100644 index 00000000..f82f0350 --- /dev/null +++ b/kernel/patches/drivers/pci/0036-PCI-apple-cold-reset-after-an-uncertified-kexec-hand.patch @@ -0,0 +1,85 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:07 +0100 +Subject: [PATCH] PCI: apple: cold-reset after an uncertified kexec handoff + +The warm guard sets panic_timeout to zero. Neither an uncertified ANS +stop nor failed PERST, LTSSM stop or powered-port certificate can safely +continue into the incoming kernel. Flush the reason and request a cold +watchdog reset instead of hanging in panic. Return without further port +writes if restart unexpectedly returns. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 9 +++++---- + include/linux/soc/apple/cold-reset.h | 19 +++++++++++++++++++ + 2 files changed, 24 insertions(+), 4 deletions(-) + create mode 100644 include/linux/soc/apple/cold-reset.h + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index f0095394a..28305b0ee 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -49,6 +49,7 @@ + #include + #include + #include ++#include + + #include "pci-host-common.h" + +@@ -3609,7 +3610,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!READ_ONCE(pcie->ans_handoff_stopped) || + (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + !apple_s8003_port_powered(pcie))) +- panic("Apple ANS: refusing uncertified powered handoff"); ++ return apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); + for (port = 0; port < H9P_NUM_PORTS; port++) { + void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; + +@@ -3621,9 +3622,9 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, + S8003_RC_PORT_PWR_REFCLK_NOGATE); + if (gpiod_direction_output(pcie->perst[port], 1)) +- panic("Apple PCIe: PERST failed during handoff"); ++ return apple_cold_reset("Apple PCIe: PERST failed during handoff"); + if (apple_s8003_port_stop(pcie, port)) +- panic("Apple PCIe: LTSSM stop failed during handoff"); ++ return apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + udelay(S8003_PERST_TO_REFCLK_US); + if (!port && pcie->nvmmu.ready) { + h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); +@@ -3635,7 +3636,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || + gpiod_get_value_cansleep(pcie->perst[0]) != 1)) +- panic("Apple ANS: outgoing powered port certificate failed"); ++ return apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); + pcie->enabled_ports &= ~BIT(port); + } + dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); +diff --git a/include/linux/soc/apple/cold-reset.h b/include/linux/soc/apple/cold-reset.h +new file mode 100644 +index 000000000..34467be50 +--- /dev/null ++++ b/include/linux/soc/apple/cold-reset.h +@@ -0,0 +1,19 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_COLD_RESET_H ++#define _LINUX_SOC_APPLE_COLD_RESET_H ++ ++#include ++#include ++#include ++ ++/* Shutdown has no safe return to a warm kernel after an unproven DMA stop. */ ++static inline void apple_cold_reset(const char *reason) ++{ ++ pr_emerg("Apple unsafe handover: %s; forcing cold watchdog reset\n", reason); ++ pr_flush(1000, true); ++ kmsg_dump_desc(KMSG_DUMP_PANIC, reason); ++ WRITE_ONCE(reboot_mode, REBOOT_COLD); ++ emergency_restart(); ++} ++ ++#endif diff --git a/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch b/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch new file mode 100644 index 00000000..5075181b --- /dev/null +++ b/kernel/patches/drivers/pci/0037-nvme-pci-cold-reset-after-unsafe-ANS-kexec-shutdown.patch @@ -0,0 +1,76 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:08:51 +0100 +Subject: [PATCH] nvme-pci: cold-reset after unsafe ANS kexec shutdown + +An unsafe ANS shutdown cannot panic under a zero panic timeout. Use the +shared logged and flushed cold-reset path for each failed certificate. +If reset unexpectedly returns, leave without further device operations. + +Assisted-by: LLM +--- + drivers/nvme/host/pci.c | 29 +++++++++++++++++++++-------- + 1 file changed, 21 insertions(+), 8 deletions(-) + +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 40b2448d1..2f356a8ce 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -3391,20 +3392,30 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + * Give the controller a chance to complete all entered requests + * if doing a safe shutdown. + */ +- if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) +- panic("Apple ANS: outgoing I/O did not drain; refusing kexec"); ++ if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) { ++ apple_cold_reset("Apple ANS: outgoing I/O did not drain"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + + nvme_quiesce_io_queues(&dev->ctrl); + +- if (paired && (dead || !dev->ctrl.queue_count)) +- panic("Apple ANS: dead outgoing controller; refusing kexec"); ++ if (paired && (dead || !dev->ctrl.queue_count)) { ++ apple_cold_reset("Apple ANS: dead outgoing controller"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + if (!dead && dev->ctrl.queue_count > 0) { + if (paired) { + nvme_sync_io_queues(&dev->ctrl); + ret = apple_ans_nvmmu_drain(dev->dev); +- if (ret) +- panic("Apple ANS: outgoing drain failed %d; refusing kexec", ret); ++ if (ret) { ++ dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); ++ apple_cold_reset("Apple ANS: outgoing drain failed"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + nvme_delete_io_queues(dev); + dev_info(dev->dev, "Apple ANS: shutdown starting NVMe SHN paired=%u\n", paired); +@@ -4034,8 +4045,10 @@ static void nvme_shutdown(struct pci_dev *pdev) + + if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && + dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { +- if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) +- panic("Apple ANS: refusing unsafe powered kexec handoff"); ++ if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) { ++ apple_cold_reset("Apple ANS: refusing unsafe powered kexec handoff"); ++ return; ++ } + return; + } + nvme_disable_prepare_reset(dev, true); diff --git a/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch b/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch new file mode 100644 index 00000000..5b49b55d --- /dev/null +++ b/kernel/patches/drivers/pci/0038-PCI-apple-use-explicit-returns-after-cold-reset.patch @@ -0,0 +1,62 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:08:51 +0100 +Subject: [PATCH] PCI: apple: use explicit returns after cold reset + +Use standard C void returns while retaining the early exit if the reset +helper unexpectedly returns. Successful handoff ordering is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 24 ++++++++++++++++-------- + 1 file changed, 16 insertions(+), 8 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 28305b0ee..5890d632a 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3609,8 +3609,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + cancel_delayed_work_sync(&pcie->late_work); + if (!READ_ONCE(pcie->ans_handoff_stopped) || + (of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && +- !apple_s8003_port_powered(pcie))) +- return apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); ++ !apple_s8003_port_powered(pcie))) { ++ apple_cold_reset("Apple ANS: refusing uncertified powered handoff"); ++ return; ++ } + for (port = 0; port < H9P_NUM_PORTS; port++) { + void __iomem *cfg = pcie->base_config + port * H9P_CFG_PORT_STRIDE; + +@@ -3621,10 +3623,14 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); + h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(port), 0, + S8003_RC_PORT_PWR_REFCLK_NOGATE); +- if (gpiod_direction_output(pcie->perst[port], 1)) +- return apple_cold_reset("Apple PCIe: PERST failed during handoff"); +- if (apple_s8003_port_stop(pcie, port)) +- return apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); ++ if (gpiod_direction_output(pcie->perst[port], 1)) { ++ apple_cold_reset("Apple PCIe: PERST failed during handoff"); ++ return; ++ } ++ if (apple_s8003_port_stop(pcie, port)) { ++ apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); ++ return; ++ } + udelay(S8003_PERST_TO_REFCLK_US); + if (!port && pcie->nvmmu.ready) { + h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); +@@ -3635,8 +3641,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_s8003_port_off(pcie, port); + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || +- gpiod_get_value_cansleep(pcie->perst[0]) != 1)) +- return apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); ++ gpiod_get_value_cansleep(pcie->perst[0]) != 1)) { ++ apple_cold_reset("Apple ANS: outgoing powered port certificate failed"); ++ return; ++ } + pcie->enabled_ports &= ~BIT(port); + } + dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); diff --git a/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch b/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch new file mode 100644 index 00000000..462bcde9 --- /dev/null +++ b/kernel/patches/drivers/pci/0039-apple-ans-declare-retained-DART-resume.patch @@ -0,0 +1,27 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:00:00 +0100 +Subject: [PATCH] apple-ans: declare retained DART resume after handoff API + +Declare the retained DART resume operation after the ANS interface has +been introduced, so the paired sleep consumer can call it. + +Assisted-by: LLM +--- + include/linux/apple-ans.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/include/linux/apple-ans.h b/include/linux/apple-ans.h +index 10be3ddef..a312a683c 100644 +--- a/include/linux/apple-ans.h ++++ b/include/linux/apple-ans.h +@@ -19,7 +19,9 @@ struct device; + */ + #if IS_REACHABLE(CONFIG_APPLE_DART) + int apple_ans_dart_stop(struct device *dev); ++int apple_ans_dart_start(struct device *dev); + #else ++static inline int apple_ans_dart_start(struct device *dev) { return -EOPNOTSUPP; } + static inline int apple_ans_dart_stop(struct device *dev) + { + return -EOPNOTSUPP; diff --git a/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch b/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch new file mode 100644 index 00000000..210631ae --- /dev/null +++ b/kernel/patches/drivers/pci/0040-nvme-pci-certify-ANS-sleep-stop-and-rebuild-through-.patch @@ -0,0 +1,163 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] nvme-pci: certify ANS sleep stop and rebuild through + warm-storage APIs + +On S8003 the NVMe controller (ANS) sits behind a PCIe port that keeps +power and reference clock over system sleep, while its DART loses its +translations. The generic nvme_suspend() path neither drains the paired +controller nor rebuilds the DART, so storage did not come back after +suspend to idle. + +For the paired ANS controller, nvme_suspend() now begins the handoff, +moves the controller to RESETTING and shuts it down through +nvme_dev_disable_paired(). preinit restores the DART with +apple_ans_dart_start(). In nvme_dev_disable_paired() a failure while +preparing for sleep unfreezes the queues and returns -EBUSY, so the +suspend is aborted instead of taking the cold-reset path used for kexec. +Resume returns -ENODEV when the PCIe link was lost over sleep. + +Assisted-by: LLM +--- + drivers/nvme/host/pci-apple-ans.c | 7 ++++ + drivers/nvme/host/pci.c | 54 +++++++++++++++++++++++++------ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/drivers/nvme/host/pci-apple-ans.c b/drivers/nvme/host/pci-apple-ans.c +index fdd9fa091..bdd9f991f 100644 +--- a/drivers/nvme/host/pci-apple-ans.c ++++ b/drivers/nvme/host/pci-apple-ans.c +@@ -6,6 +6,7 @@ + * Copyright (C) 2026 Paul Praschl + */ + ++#include + #include + #include + #include +@@ -276,6 +277,12 @@ static int nvme_pci_apple_ans_preinit(struct nvme_dev *dev) + if (!ans) + return 0; + ++ if (apple_ans_is_s8003(dev->dev)) { ++ ret = apple_ans_dart_start(dev->dev); ++ if (ret) ++ return ret; ++ } ++ + core_mask = readl(dev->bar + APPLE_ANS_REG_CORE_MASK); + log_size = readl(dev->bar + APPLE_ANS_REG_LOG_SIZE); + hmb_size = readl(dev->bar + APPLE_ANS_REG_HMB_SIZE_REQ); +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 2f356a8ce..6868811d5 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -3381,6 +3381,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; ++ bool for_sleep = paired && !kexec_in_progress; + int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); +@@ -3393,28 +3394,40 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + * if doing a safe shutdown. + */ + if (!dead && shutdown && nvme_wait_freeze_timeout(&dev->ctrl) <= 0 && paired) { +- apple_cold_reset("Apple ANS: outgoing I/O did not drain"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (for_sleep) { ++ nvme_unfreeze(&dev->ctrl); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EBUSY; ++ } else { ++ apple_cold_reset("Apple ANS: outgoing I/O did not drain"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } + } + } + + nvme_quiesce_io_queues(&dev->ctrl); + + if (paired && (dead || !dev->ctrl.queue_count)) { +- apple_cold_reset("Apple ANS: dead outgoing controller"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (!for_sleep) { ++ apple_cold_reset("Apple ANS: dead outgoing controller"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } ++ goto sleep_failed; + } + if (!dead && dev->ctrl.queue_count > 0) { + if (paired) { + nvme_sync_io_queues(&dev->ctrl); + ret = apple_ans_nvmmu_drain(dev->dev); + if (ret) { +- dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); +- apple_cold_reset("Apple ANS: outgoing drain failed"); +- mutex_unlock(&dev->shutdown_lock); +- return -EIO; ++ if (!for_sleep) { ++ dev_err(dev->dev, "Apple ANS: outgoing drain failed %d\n", ret); ++ apple_cold_reset("Apple ANS: outgoing drain failed"); ++ mutex_unlock(&dev->shutdown_lock); ++ return -EIO; ++ } ++ goto sleep_failed; + } + } + nvme_delete_io_queues(dev); +@@ -3424,6 +3437,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + ret = nvme_apple_ans_handoff_disable(dev, ret); + nvme_poll_irqdisable(&dev->queues[0]); + } ++sleep_failed: + nvme_suspend_io_queues(dev); + nvme_suspend_queue(dev, 0); + pci_free_irq_vectors(pdev); +@@ -4101,6 +4115,9 @@ static int nvme_resume(struct device *dev) + struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); + struct nvme_ctrl *ctrl = &ndev->ctrl; + ++ if (pci_dev_is_disconnected(to_pci_dev(dev))) ++ return -ENODEV; ++ + if (ndev->last_ps == U32_MAX || + nvme_set_power_state(ctrl, ndev->last_ps) != 0) + goto reset; +@@ -4120,6 +4137,20 @@ static int nvme_suspend(struct device *dev) + int ret = -EBUSY; + + ndev->last_ps = U32_MAX; ++ if (ndev->dma_ops == &nvme_pci_apple_ans_ops && apple_ans_is_s8003(dev)) { ++ ret = apple_ans_handoff_begin(dev); ++ if (ret) ++ return ret; ++ if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || ++ !nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) ++ return -EBUSY; ++ ret = nvme_dev_disable_paired(ndev, true, true); ++ if (ret) { ++ nvme_try_sched_reset(ctrl); ++ return -EBUSY; ++ } ++ return 0; ++ } + + /* + * The platform does not remove power for a kernel managed suspend so +@@ -4201,6 +4232,9 @@ static int nvme_simple_resume(struct device *dev) + struct pci_dev *pdev = to_pci_dev(dev); + struct nvme_dev *ndev = pci_get_drvdata(pdev); + ++ if (pci_dev_is_disconnected(to_pci_dev(dev))) ++ return -ENODEV; ++ + return nvme_try_sched_reset(&ndev->ctrl); + } + diff --git a/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch b/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch new file mode 100644 index 00000000..5d551f21 --- /dev/null +++ b/kernel/patches/drivers/pci/0041-dt-bindings-pci-describe-paired-sleep-and-optional-h.patch @@ -0,0 +1,39 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] dt-bindings: pci: describe paired sleep and optional + handoff link flush + +Document two boolean properties of the S8003 H9P controller. +apple,paired-system-sleep marks an ANS controller whose port keeps power +across system sleep and is stopped and restarted together with it. +apple,ans-kexec-link-flush makes a warm handoff consume a stale link-up +status under PERST before the link is stopped. + +Assisted-by: LLM +--- + .../devicetree/bindings/pci/apple,t8010-pcie.yaml | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +index f9b72ce26..cebf60d8a 100644 +--- a/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml ++++ b/Documentation/devicetree/bindings/pci/apple,t8010-pcie.yaml +@@ -231,6 +231,17 @@ properties: + without exposing a root bridge. This is used when an enabled port shares + PHY state with a firmware-enabled sibling port. + ++ apple,ans-kexec-link-flush: ++ type: boolean ++ description: ++ Consume stale ANS link status under PERST during warm handoff. ++ ++ apple,paired-system-sleep: ++ type: boolean ++ description: ++ The ANS controller is paired with retained-power port reset over system ++ sleep. ++ + interrupt-controller: true + + '#interrupt-cells': diff --git a/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch b/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch new file mode 100644 index 00000000..8ad218f5 --- /dev/null +++ b/kernel/patches/drivers/pci/0042-PCI-apple-h9p-pair-s2idle-with-retained-ANS-power-an.patch @@ -0,0 +1,402 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] PCI: apple-h9p: pair s2idle with retained ANS power and + early admission + +System sleep on J71t has to keep the ANS port powered: removing its +power or reference clock loses the NVMe controller until a cold boot. + +Add sleep support to the S8003 path of the H9P driver, enabled by +apple,paired-system-sleep. prepare pins the endpoints and holds off +config accesses. suspend asserts PERST and stops the link but keeps +power and refclk, and a failure before the first destructive write rolls +back with power kept. resume brings the ports back through the powered +reset and marks endpoints that did not return as disconnected. The +powered reset (PERST, LTSSM stop and a fresh link training, without a +power cycle) is added here with its only caller; the common block is not +reinitialized across s2idle. + +A suspend precheck refuses with -EBUSY, before sync, anything but +suspend to idle, and also refuses while a filesystem is mounted from an +NBD device, port bring-up work is pending, the port is not powered, or a +bound GPU, VXD or SEP driver holds no reserved sleep admission. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 278 +++++++++++++++++++++++- + 1 file changed, 277 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 5890d632a..7a0a832e3 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -51,6 +51,11 @@ + #include + #include + ++#include ++#include ++#include ++#include ++#include "../pci.h" + #include "pci-host-common.h" + + #define H9P_NUM_PORTS 4 +@@ -719,6 +724,13 @@ struct apple_h9p_pcie { + unsigned int late_tries; + struct delayed_work late_work; + bool ans_handoff_stopped; ++ bool ans_sleep_powered; ++ u32 resume_ports; ++ struct pci_dev *sleep_pinned[32]; ++ unsigned int sleep_npinned; ++ bool sleep_pin_overflow; ++ struct notifier_block sleep_precheck; ++ + + struct apple_h9p_nvmmu nvmmu; + +@@ -1094,6 +1106,9 @@ static int apple_h9p_pcie_attach_genpd(struct apple_h9p_pcie *pcie) + return PTR_ERR(pcie->pd_dev[i]); + } + ++ if (of_property_read_bool(dev->of_node, "apple,paired-system-sleep")) ++ dev_pm_syscore_device(pcie->pd_dev[i], true); ++ + pcie->pd_link[i] = device_link_add(dev, pcie->pd_dev[i], + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | +@@ -1239,6 +1254,20 @@ static void apple_h9p_limit_config_field(u32 *val, int where, int size, + *val = (*val & ~mask) | (value & mask); + } + ++static bool apple_h9p_sleep_config_blocked(struct pci_bus *bus) ++{ ++ struct pci_config_window *cfg = bus->sysdata; ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(cfg->parent); ++ struct pci_bus *root = bus; ++ ++ if (!pcie || bus->number == cfg->busr.start) ++ return false; ++ while (root->parent && root->parent->parent) ++ root = root->parent; ++ return root->self && !(READ_ONCE(pcie->enabled_ports) & ++ BIT(PCI_SLOT(root->self->devfn))); ++} ++ + static int apple_h9p_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) + { +@@ -1258,6 +1287,10 @@ static int apple_h9p_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + pcie = NULL; + } + ++ if (apple_h9p_sleep_config_blocked(bus)) { ++ *val = ~0U; ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ } + ret = pci_generic_config_read(bus, devfn, where, size, val); + if (ret != PCIBIOS_SUCCESSFUL || !pcie) + return ret; +@@ -1295,6 +1328,9 @@ static int apple_h9p_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + return PCIBIOS_DEVICE_NOT_FOUND; + } + ++ if (apple_h9p_sleep_config_blocked(bus)) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ + if (where <= PCI_INTERRUPT_LINE && where + size > PCI_INTERRUPT_LINE) + val |= 0xffu << ((PCI_INTERRUPT_LINE - where) << 3); + +@@ -2925,6 +2961,33 @@ static bool apple_s8003_port_powered(struct apple_h9p_pcie *pcie) + (readl(rc + S8003_RC_PORT_READY(0)) & S8003_RC_PORT_READY_DONE); + } + ++static int apple_s8003_powered_reset(struct apple_h9p_pcie *pcie) ++{ ++ int ret; ++ ++ if (!apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) ++ ret = apple_s8003_port_stop(pcie, 0); ++ if (!ret && READ_ONCE(pcie->ans_sleep_powered) && ++ (readl(pcie->base_port[0] + S8003_PORT_LINKSTS) & S8003_PORT_LINKSTS_UP)) { ++ u32 val; ++ ++ h9p_rmw(pcie->base_port[0] + H9P_PORT_LTSSMCTL, 0, ++ H9P_PORT_LTSSM_ENABLE); ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ ret = apple_s8003_port_stop(pcie, 0); ++ } ++ if (!ret) ++ ret = apple_s8003_start_link(pcie, 0); ++ /* A failed powered reset stays under PERST; no full-cycle fallback. */ ++ if (ret) ++ gpiod_direction_output(pcie->perst[0], 1); ++ return ret; ++} ++ + /* Power the port up as validated for its kind, and train its link once */ + static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + unsigned int port) +@@ -2940,6 +3003,8 @@ static int apple_s8003_setup_port_once(struct apple_h9p_pcie *pcie, + * validated on it, the other ports the restart sequence that is the + * only one validated on the Wi-Fi port. + */ ++ if (!port && READ_ONCE(pcie->ans_sleep_powered)) ++ return apple_s8003_powered_reset(pcie); + if (pcie->hw->nvme_mode_ports & BIT(port)) + ret = apple_s8003_power_cycle_port(pcie, port); + else +@@ -3598,6 +3663,208 @@ int apple_ans_handoff_begin(struct device *dev) + } + EXPORT_SYMBOL_GPL(apple_ans_handoff_begin); + ++ ++static bool apple_h9p_sleep_enabled(struct apple_h9p_pcie *pcie) ++{ ++ return of_property_read_bool(pcie->dev->of_node, "apple,paired-system-sleep"); ++} ++ ++static int apple_h9p_sleep_pin(struct pci_dev *pdev, void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ if (pdev->dev_flags & PCI_DEV_FLAGS_NO_D3) ++ return 0; ++ if (pcie->sleep_npinned == ARRAY_SIZE(pcie->sleep_pinned)) { ++ pcie->sleep_pin_overflow = true; ++ return 1; ++ } ++ pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3; ++ pcie->sleep_pinned[pcie->sleep_npinned++] = pci_dev_get(pdev); ++ return 0; ++} ++ ++static void apple_h9p_sleep_complete(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ ++ while (pcie && pcie->sleep_npinned) { ++ struct pci_dev *pdev = pcie->sleep_pinned[--pcie->sleep_npinned]; ++ ++ pdev->dev_flags &= ~PCI_DEV_FLAGS_NO_D3; ++ pci_dev_put(pdev); ++ } ++ if (pcie) ++ pcie->ans_sleep_powered = false; ++} ++ ++static int apple_h9p_sleep_prepare(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ pcie->ans_handoff_stopped = false; ++ pcie->ans_sleep_powered = false; ++ pcie->sleep_pin_overflow = false; ++ pci_walk_bus(pcie->bridge->bus, apple_h9p_sleep_pin, pcie); ++ ret = pcie->sleep_pin_overflow ? -EBUSY : 0; ++ if (ret) ++ apple_h9p_sleep_complete(dev); ++ return ret; ++} ++ ++static void apple_h9p_sleep_cut_off(struct apple_h9p_pcie *pcie, unsigned int port) ++{ ++ struct pci_dev *rp; ++ ++ gpiod_direction_output(pcie->perst[port], 1); ++ apple_s8003_port_stop(pcie, port); ++ /* Never remove ANS power/refclk, even after failed resume. */ ++ pcie->enabled_ports &= ~BIT(port); ++ rp = pci_get_slot(pcie->bridge->bus, PCI_DEVFN(port, 0)); ++ if (!rp) ++ return; ++ if (rp->subordinate) ++ pci_walk_bus(rp->subordinate, pci_dev_set_disconnected, NULL); ++ pci_dev_set_disconnected(rp, NULL); ++ pci_dev_put(rp); ++} ++ ++static int apple_h9p_sleep_resume(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ unsigned int port; ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ for (port = 0; port < H9P_NUM_PORTS; port++) { ++ struct pci_dev *rp; ++ ++ if (!(pcie->resume_ports & BIT(port))) ++ continue; ++ if (port && !gpiod_get_value_cansleep(pcie->perst[port]) && ++ apple_h9p_link_up(pcie, port)) ++ continue; ++ ret = port ? apple_s8003_setup_port_tries(pcie, port) : ++ apple_s8003_powered_reset(pcie); ++ if (ret) { ++ apple_h9p_sleep_cut_off(pcie, port); ++ dev_err(dev, "port %u lost after sleep: %d; endpoints disconnected\n", port, ret); ++ continue; ++ } ++ pcie->enabled_ports |= BIT(port); ++ msleep(PCIE_RESET_CONFIG_WAIT_MS); ++ rp = pci_get_slot(pcie->bridge->bus, PCI_DEVFN(port, 0)); ++ if (rp) { ++ pci_restore_state(rp); ++ pci_dev_put(rp); ++ } ++ } ++ if (pcie->enabled_ports & BIT(0)) { ++ ret = apple_h9p_nvmmu_program(pcie); ++ if (ret) ++ apple_h9p_sleep_cut_off(pcie, 0); ++ } ++ pcie->resume_ports = 0; ++ return 0; ++} ++ ++static int apple_h9p_sleep_suspend(struct device *dev) ++{ ++ struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(dev); ++ void __iomem *cfg; ++ u32 val; ++ int ret; ++ ++ if (!pcie || !apple_h9p_sleep_enabled(pcie)) ++ return 0; ++ cfg = pcie->base_config; ++ if (!READ_ONCE(pcie->ans_handoff_stopped) || !apple_s8003_port_powered(pcie)) ++ return -EBUSY; ++ pcie->resume_ports = pcie->enabled_ports; ++ /* Latch before the first destructive write; rollback must retain power. */ ++ pcie->ans_sleep_powered = true; ++ writew(0, cfg + PCI_COMMAND); ++ h9p_rmw(cfg + PCI_PRIMARY_BUS, GENMASK(23, 8), 0); ++ h9p_rmw(pcie->base_rc + S8003_RC_PORT_PWR(0), 0, ++ S8003_RC_PORT_PWR_REFCLK_NOGATE); ++ ret = gpiod_direction_output(pcie->perst[0], 1); ++ if (!ret) { ++ /* r1e: consume the falling link before stopping LTSSM. */ ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ ret = apple_s8003_port_stop(pcie, 0); ++ } ++ if (ret) { ++ apple_h9p_sleep_resume(dev); ++ return ret; ++ } ++ udelay(S8003_PERST_TO_REFCLK_US); ++ h9p_writel_flush(0, pcie->nvmmu.base + H9P_NVMMU_SART_CTRL); ++ pcie->enabled_ports &= ~BIT(0); ++ return 0; ++} ++ ++static void apple_h9p_sleep_nbd(struct super_block *sb, void *data) ++{ ++ if (sb->s_bdev && !strncmp(sb->s_bdev->bd_disk->disk_name, "nbd", 3)) ++ *(bool *)data = true; ++} ++ ++/* Until GS provides a reserved admission capability, bound accelerators fail closed. */ ++static int apple_h9p_sleep_stack(struct device *dev, void *data) ++{ ++ const char *name; ++ ++ if (!dev->driver) ++ return 0; ++ name = dev->driver->name; ++ if (strstr(name, "g5p") || strstr(name, "powervr") || ++ strstr(name, "vxd") || strstr(name, "sep")) { ++ if (pm_suspend_admission_reserved(dev)) ++ return 0; ++ dev_warn(dev, "system sleep refused: no reserved safe sleep capability\n"); ++ return -EBUSY; ++ } ++ return 0; ++} ++ ++static int apple_h9p_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_h9p_pcie *pcie = container_of(nb, struct apple_h9p_pcie, sleep_precheck); ++ bool nbd = false; ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ if (!apple_h9p_sleep_enabled(pcie)) ++ return NOTIFY_DONE; ++ if (action != PM_SUSPEND_TO_IDLE) ++ return notifier_from_errno(-EBUSY); ++ iterate_supers(apple_h9p_sleep_nbd, &nbd); ++ if (nbd || pcie->late_ports || work_busy(&pcie->late_work.work) || ++ !pcie->pd_dev || !apple_s8003_port_powered(pcie) || ++ bus_for_each_dev(&platform_bus_type, NULL, NULL, apple_h9p_sleep_stack)) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_OK; ++} ++ ++static void apple_h9p_sleep_unregister(void *data) ++{ ++ struct apple_h9p_pcie *pcie = data; ++ ++ unregister_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++} ++ ++static const struct dev_pm_ops apple_h9p_sleep_ops = { ++ .prepare = apple_h9p_sleep_prepare, ++ .complete = apple_h9p_sleep_complete, ++ .suspend_noirq = apple_h9p_sleep_suspend, ++ .resume_noirq = apple_h9p_sleep_resume, ++}; ++ + static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + { + struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); +@@ -3627,6 +3894,7 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_cold_reset("Apple PCIe: PERST failed during handoff"); + return; + } ++ + if (apple_s8003_port_stop(pcie, port)) { + apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + return; +@@ -3777,6 +4045,9 @@ static const struct attribute_group apple_h9p_crash_group = { + }; + + ++static int apple_h9p_sleep_precheck(struct notifier_block *, unsigned long, void *); ++static void apple_h9p_sleep_unregister(void *); ++ + static int apple_h9p_pcie_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -3948,7 +4219,11 @@ static int apple_h9p_pcie_probe(struct platform_device *pdev) + if (pcie->late_ports) + schedule_delayed_work(&pcie->late_work, + msecs_to_jiffies(S8003_LATE_PORT_RETRY_MS)); +- return 0; ++ pcie->sleep_precheck.notifier_call = apple_h9p_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&pcie->sleep_precheck); ++ if (ret) ++ return ret; ++ return devm_add_action_or_reset(dev, apple_h9p_sleep_unregister, pcie); + } + + static const struct apple_h9p_hw apple_h9p_hw_t7000 = { +@@ -4032,6 +4307,7 @@ static struct platform_driver apple_h9p_pcie_driver = { + .name = "pcie-apple-h9p", + .of_match_table = apple_h9p_pcie_of_match, + .suppress_bind_attrs = true, ++ .pm = pm_sleep_ptr(&apple_h9p_sleep_ops), + }, + }; + module_platform_driver(apple_h9p_pcie_driver); diff --git a/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch b/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch new file mode 100644 index 00000000..bd2908f3 --- /dev/null +++ b/kernel/patches/drivers/pci/0043-PCI-apple-h9p-make-the-handoff-link-flush-a-separate.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] PCI: apple-h9p: make the handoff link flush a separate + opt-in + +Waiting for the link-up status to fall under PERST before LTSSM is +stopped was tied to the sleep path. A warm handoff can see the same +stale status, but not every board needs the wait. + +Do the flush in the outgoing kexec shutdown path only, and only when +apple,ans-kexec-link-flush is set, so it stays off unless a board asks +for it. The incoming kernel restarts the storage port with the cold-boot +power cycle, which needs no flush. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 7a0a832e3..46c24f2b5 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3894,7 +3894,12 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + apple_cold_reset("Apple PCIe: PERST failed during handoff"); + return; + } ++ if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-link-flush")) { ++ u32 val; + ++ readl_poll_timeout(pcie->base_port[0] + S8003_PORT_LINKSTS, ++ val, !(val & S8003_PORT_LINKSTS_UP), 100, 100000); ++ } + if (apple_s8003_port_stop(pcie, port)) { + apple_cold_reset("Apple PCIe: LTSSM stop failed during handoff"); + return; diff --git a/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch b/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch new file mode 100644 index 00000000..9e84e7ac --- /dev/null +++ b/kernel/patches/drivers/pci/0044-nvme-pci-stop-Apple-ANS-before-a-restart-as-before-a.patch @@ -0,0 +1,61 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:15:38 +0100 +Subject: [PATCH] nvme-pci: stop Apple ANS before a restart as before a kexec + +A restart that re-enters the boot loader (PSCI SYSTEM_RESET to m1n1) +leaves the S8003 ANS controller to a loader that brings its port up again +and reads from it. On a plain shutdown nvme_disable_prepare_reset() sets +SHN but keeps CC.EN, and power-cycling the port in that state hangs the +SoC. + +Take the paired, certified stop that a kexec handoff already uses on every +restart as well: drain, SHN, CC.EN and RDY clear, bus mastering off and the +NVMMU/DART retired. Such a failure is not a sleep refusal, so it no longer +takes the sleep branch; it ends in a cold watchdog reset, as for kexec. +Power-off and halt keep the existing shutdown. + +Assisted-by: LLM +--- + drivers/nvme/host/pci.c | 15 ++++++++++++--- + 1 file changed, 12 insertions(+), 3 deletions(-) + +diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c +index 6868811d5..9fe60de0e 100644 +--- a/drivers/nvme/host/pci.c ++++ b/drivers/nvme/host/pci.c +@@ -3381,7 +3381,7 @@ static int nvme_dev_disable_paired(struct nvme_dev *dev, bool shutdown, bool pai + enum nvme_ctrl_state state = nvme_ctrl_state(&dev->ctrl); + struct pci_dev *pdev = to_pci_dev(dev->dev); + bool dead; +- bool for_sleep = paired && !kexec_in_progress; ++ bool for_sleep = paired && !kexec_in_progress && system_state != SYSTEM_RESTART; + int ret = paired ? -EBUSY : 0; + + mutex_lock(&dev->shutdown_lock); +@@ -4053,14 +4053,23 @@ static void nvme_reset_done(struct pci_dev *pdev) + flush_work(&dev->ctrl.reset_work); + } + ++/* ++ * A restart re-enters the boot loader, which starts the storage port again ++ * and reads from the controller. It needs the controller stopped as iBoot ++ * leaves it (SHN complete, CC.EN and RDY clear, DMA certified stopped): the ++ * stop a kexec handoff takes. Anything less ends in a cold reset. ++ */ + static void nvme_shutdown(struct pci_dev *pdev) + { + struct nvme_dev *dev = pci_get_drvdata(pdev); ++ bool restart = !kexec_in_progress && system_state == SYSTEM_RESTART; + +- if (kexec_in_progress && apple_ans_is_s8003(dev->dev) && ++ if ((kexec_in_progress || restart) && apple_ans_is_s8003(dev->dev) && + dev->dma_ops == &nvme_pci_apple_ans_ops && !apple_ans_handoff_begin(dev->dev)) { + if (!nvme_wait_reset(&dev->ctrl) || nvme_dev_disable_paired(dev, true, true)) { +- apple_cold_reset("Apple ANS: refusing unsafe powered kexec handoff"); ++ apple_cold_reset(restart ? ++ "Apple ANS: unsafe controller stop before restart" : ++ "Apple ANS: refusing unsafe powered kexec handoff"); + return; + } + return; diff --git a/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch b/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch new file mode 100644 index 00000000..179866ce --- /dev/null +++ b/kernel/patches/drivers/pci/0045-PCI-apple-h9p-hand-off-S8003-ports-on-restart-and-po.patch @@ -0,0 +1,70 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 11:15:39 +0100 +Subject: [PATCH] PCI: apple-h9p: hand off S8003 ports on restart and power off + Wi-Fi + +On a restart that re-enters the boot loader the S8003 host's shutdown did +nothing, so the Wi-Fi function kept its link and bus mastering. Its +firmware can keep writing into memory that the boot loader and the next +kernel reuse, and the boot loader does not touch ports other than storage. + +Run the kexec handoff on restart as well: block root routing, assert PERST, +stop the LTSSM and disable the NVMMU SART, with the same certificate and +cold-reset fallback. The storage port then ends powered with PERST held, +as for kexec. Ports other than storage are additionally powered off with +the iOS port power-down, the state the boot loader leaves them in and from +which probe restarts them. The kexec handoff itself is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 5d6e717ee..90491965d 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3891,12 +3891,20 @@ static const struct dev_pm_ops apple_h9p_sleep_ops = { + .resume_noirq = apple_h9p_sleep_resume, + }; + ++/* ++ * A kexec and a restart take the same handoff. After a restart the boot ++ * loader starts the storage port again from this state, as it does from ++ * iBoot's, but it does not stop an endpoint left on any other port: Wi-Fi ++ * would keep DMAing into memory the next kernel reuses. Those ports are ++ * therefore powered off after the isolation, as the boot loader leaves them. ++ */ + static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + { + struct apple_h9p_pcie *pcie = apple_h9p_pcie_lookup(&pdev->dev); ++ bool restart = !kexec_in_progress && system_state == SYSTEM_RESTART; + unsigned int port; + +- if (!kexec_in_progress || !pcie || ++ if ((!kexec_in_progress && !restart) || !pcie || + !of_device_is_compatible(pdev->dev.of_node, "apple,s8003-pcie")) + return; + cancel_delayed_work_sync(&pcie->late_work); +@@ -3938,6 +3946,8 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + if (!port && !of_property_read_bool(pcie->dev->of_node, + "apple,ans-kexec-handoff")) + apple_s8003_port_off(pcie, port); ++ if (port && restart) ++ apple_s8003_port_off(pcie, port); + if (!port && of_property_read_bool(pcie->dev->of_node, "apple,ans-kexec-handoff") && + (!apple_s8003_port_powered(pcie) || + gpiod_get_value_cansleep(pcie->perst[0]) != 1)) { +@@ -3946,7 +3956,10 @@ static void apple_h9p_s8003_shutdown(struct platform_device *pdev) + } + pcie->enabled_ports &= ~BIT(port); + } +- dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); ++ if (restart) ++ dev_info(pcie->dev, "restart: PCIe PERST asserted, LTSSM stopped, ports 1+ powered off; NVMMU SART disabled\n"); ++ else ++ dev_info(pcie->dev, "kexec: PCIe PERST asserted, LTSSM stopped, ports powered; NVMMU SART disabled\n"); + } + + static void apple_h9p_capture_park(struct apple_h9p_pcie *pcie) diff --git a/kernel/patches/drivers/pci/0046-PCI-apple-h9p-name-the-s2idle-precheck-refusal.patch b/kernel/patches/drivers/pci/0046-PCI-apple-h9p-name-the-s2idle-precheck-refusal.patch new file mode 100644 index 00000000..c4431845 --- /dev/null +++ b/kernel/patches/drivers/pci/0046-PCI-apple-h9p-name-the-s2idle-precheck-refusal.patch @@ -0,0 +1,64 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:26:19 +0100 +Subject: [PATCH] PCI: apple-h9p: name the s2idle precheck refusal + +The S8003 sleep precheck refused for six different reasons with the same +silent -EBUSY. Name the first one that applies (nbd-root, late-ports, +late-work, no-power-domain, storage-port-not-powered, +unreserved-accelerator or not-s2idle) in one line, only during a suspend +attempt, so the readiness query stays quiet. + +The decision itself is unchanged. + +Assisted-by: LLM +--- + drivers/pci/controller/pcie-apple-h9p.c | 28 ++++++++++++++++++------- + 1 file changed, 21 insertions(+), 7 deletions(-) + +diff --git a/drivers/pci/controller/pcie-apple-h9p.c b/drivers/pci/controller/pcie-apple-h9p.c +index 79b69fdbe..f81e1b166 100644 +--- a/drivers/pci/controller/pcie-apple-h9p.c ++++ b/drivers/pci/controller/pcie-apple-h9p.c +@@ -3846,6 +3846,7 @@ static int apple_h9p_sleep_stack(struct device *dev, void *data) + static int apple_h9p_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) + { + struct apple_h9p_pcie *pcie = container_of(nb, struct apple_h9p_pcie, sleep_precheck); ++ const char *why = NULL; + bool nbd = false; + + if (action == PM_SUSPEND_PRECHECK_COMPLETE) +@@ -3853,13 +3854,26 @@ static int apple_h9p_sleep_precheck(struct notifier_block *nb, unsigned long act + if (!apple_h9p_sleep_enabled(pcie)) + return NOTIFY_DONE; + if (action != PM_SUSPEND_TO_IDLE) +- return notifier_from_errno(-EBUSY); +- iterate_supers(apple_h9p_sleep_nbd, &nbd); +- if (nbd || pcie->late_ports || work_busy(&pcie->late_work.work) || +- !pcie->pd_dev || !apple_s8003_port_powered(pcie) || +- bus_for_each_dev(&platform_bus_type, NULL, NULL, apple_h9p_sleep_stack)) +- return notifier_from_errno(-EBUSY); +- return NOTIFY_OK; ++ why = "not-s2idle"; ++ else if (iterate_supers(apple_h9p_sleep_nbd, &nbd), nbd) ++ why = "nbd-root"; ++ else if (pcie->late_ports) ++ why = "late-ports"; ++ else if (work_busy(&pcie->late_work.work)) ++ why = "late-work"; ++ else if (!pcie->pd_dev) ++ why = "no-power-domain"; ++ else if (!apple_s8003_port_powered(pcie)) ++ why = "storage-port-not-powered"; ++ else if (bus_for_each_dev(&platform_bus_type, NULL, NULL, apple_h9p_sleep_stack)) ++ why = "unreserved-accelerator"; ++ if (!why) ++ return NOTIFY_OK; ++ /* Only for an attempt: the readiness query polls this. */ ++ if (data == PM_SUSPEND_PRECHECK_ATTEMPT) ++ dev_info(pcie->dev, "system sleep refused: %s late_ports=%#x\n", why, ++ pcie->late_ports); ++ return notifier_from_errno(-EBUSY); + } + + static void apple_h9p_sleep_unregister(void *data) diff --git a/kernel/patches/drivers/perf/0001-dt-bindings-perf-Add-Apple-A9-port-bandwidth-monitor.patch b/kernel/patches/drivers/perf/0001-dt-bindings-perf-Add-Apple-A9-port-bandwidth-monitor.patch new file mode 100644 index 00000000..6a0def38 --- /dev/null +++ b/kernel/patches/drivers/perf/0001-dt-bindings-perf-Add-Apple-A9-port-bandwidth-monitor.patch @@ -0,0 +1,46 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 10:22:55 +0000 +Subject: [PATCH] dt-bindings: perf: Add Apple A9 port bandwidth monitor + +Assisted-by: LLM +--- + .../bindings/perf/apple,s8000-amc-pmu.yaml | 29 +++++++++++++++++++ + 1 file changed, 29 insertions(+) + create mode 100644 Documentation/devicetree/bindings/perf/apple,s8000-amc-pmu.yaml + +diff --git a/Documentation/devicetree/bindings/perf/apple,s8000-amc-pmu.yaml b/Documentation/devicetree/bindings/perf/apple,s8000-amc-pmu.yaml +new file mode 100644 +index 000000000..f8c3591c1 +--- /dev/null ++++ b/Documentation/devicetree/bindings/perf/apple,s8000-amc-pmu.yaml +@@ -0,0 +1,29 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/perf/apple,s8000-amc-pmu.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: Apple A9 memory controller bandwidth counters ++ ++maintainers: ++ - Berkay Kösebay ++ ++description: | ++ The memory cache controller has fixed counters for CPU and GPU traffic. ++ The register range covers only the port bandwidth monitor, not the full ++ memory controller. Events count bytes across the whole SoC. The counters ++ are polled and do not require an interrupt. ++ ++properties: ++ compatible: ++ const: apple,s8000-amc-pmu ++ ++ reg: ++ maxItems: 1 ++ ++required: ++ - compatible ++ - reg ++ ++additionalProperties: false diff --git a/kernel/patches/drivers/perf/0002-perf-apple-Add-A9-memory-port-byte-counters.patch b/kernel/patches/drivers/perf/0002-perf-apple-Add-A9-memory-port-byte-counters.patch new file mode 100644 index 00000000..d793892f --- /dev/null +++ b/kernel/patches/drivers/perf/0002-perf-apple-Add-A9-memory-port-byte-counters.patch @@ -0,0 +1,437 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 10:22:55 +0000 +Subject: [PATCH] perf: apple: Add A9 memory port byte counters + +Assisted-by: LLM +--- + drivers/perf/Kconfig | 9 + + drivers/perf/Makefile | 1 + + drivers/perf/apple_amc_pmu.c | 391 +++++++++++++++++++++++++++++++++++ + 3 files changed, 401 insertions(+) + create mode 100644 drivers/perf/apple_amc_pmu.c + +diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig +index 245e7bb76..5f324b8bb 100644 +--- a/drivers/perf/Kconfig ++++ b/drivers/perf/Kconfig +@@ -325,4 +325,13 @@ config NVIDIA_TEGRA410_C2C_PMU + Enable perf support for counters in NVIDIA C2C interface of NVIDIA + Tegra410 SoC. + ++config APPLE_AMC_PMU ++ tristate "Apple A9 memory controller bandwidth PMU" ++ depends on ARM64 && (ARCH_APPLE || COMPILE_TEST) ++ depends on PERF_EVENTS ++ help ++ Count system-wide CPU and GPU memory traffic on Apple A9 SoCs. ++ The PMU exposes four fixed byte events through perf and does not ++ support task filtering or sampling interrupts. ++ + endmenu +diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile +index eb8a022da..3652b87ed 100644 +--- a/drivers/perf/Makefile ++++ b/drivers/perf/Makefile +@@ -37,3 +37,4 @@ obj-$(CONFIG_MESON_DDR_PMU) += amlogic/ + obj-$(CONFIG_CXL_PMU) += cxl_pmu.o + obj-$(CONFIG_NVIDIA_TEGRA410_CMEM_LATENCY_PMU) += nvidia_t410_cmem_latency_pmu.o + obj-$(CONFIG_NVIDIA_TEGRA410_C2C_PMU) += nvidia_t410_c2c_pmu.o ++obj-$(CONFIG_APPLE_AMC_PMU) += apple_amc_pmu.o +diff --git a/drivers/perf/apple_amc_pmu.c b/drivers/perf/apple_amc_pmu.c +new file mode 100644 +index 000000000..82fc6b95c +--- /dev/null ++++ b/drivers/perf/apple_amc_pmu.c +@@ -0,0 +1,391 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Apple A9 memory-cache-controller port bandwidth PMU. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#define AMC_CONTROL 0x00 ++#define AMC_ENABLE 0x04 ++#define AMC_CONFIG(n, w) (0x08 + (n) * 0x1c + (w) * 4) ++#define AMC_COUNT(n) (0x20 + (n) * 0x1c) ++#define AMC_PORT_MASK GENMASK(4, 0) ++#define AMC_EVENTS 4 ++/* At 2 bytes/tick a 32-bit counter wraps after 8 GiB, not 4 GiB. */ ++#define AMC_POLL_NS (10 * NSEC_PER_MSEC) ++ ++struct apple_amc_data { ++ u32 port_filter[2]; ++ u32 request_filter[2]; ++ u32 byte_filter; ++ u32 line_filter; ++ u32 control; ++}; ++ ++struct apple_amc { ++ struct pmu pmu; ++ void __iomem *base; ++ const struct apple_amc_data *data; ++ raw_spinlock_t lock; ++ struct hrtimer timer; ++ struct perf_event *events[AMC_EVENTS]; ++ unsigned long running; ++ struct hlist_node node; ++ int cpu; ++ bool suspended; ++}; ++ ++static int amc_hp_state; ++ ++#define to_amc(p) container_of(p, struct apple_amc, pmu) ++#define AMC_EVENT(name, id) PMU_EVENT_ATTR_STRING(name, amc_##name, "event=" #id) ++AMC_EVENT(cpu_read, 0); ++AMC_EVENT(cpu_write, 1); ++AMC_EVENT(gfx_read, 2); ++AMC_EVENT(gfx_write, 3); ++PMU_FORMAT_ATTR(event, "config:0-1"); ++ ++static struct attribute *amc_event_attrs[] = { ++ &amc_cpu_read.attr.attr, &amc_cpu_write.attr.attr, ++ &amc_gfx_read.attr.attr, &amc_gfx_write.attr.attr, NULL, ++}; ++static struct attribute *amc_format_attrs[] = { &format_attr_event.attr, NULL }; ++static const struct attribute_group amc_events = { ++ .name = "events", .attrs = amc_event_attrs, ++}; ++static const struct attribute_group amc_format = { ++ .name = "format", .attrs = amc_format_attrs, ++}; ++ ++static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct apple_amc *amc = to_amc(dev_get_drvdata(dev)); ++ ++ return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(amc->cpu))); ++} ++static DEVICE_ATTR_RO(cpumask); ++static struct attribute *amc_cpu_attrs[] = { &dev_attr_cpumask.attr, NULL }; ++static const struct attribute_group amc_cpus = { .attrs = amc_cpu_attrs }; ++static const struct attribute_group *amc_groups[] = { ++ &amc_events, &amc_format, &amc_cpus, NULL, ++}; ++ ++/* Match Apple's read, disable, zero, restore sequence for each physical slot. */ ++static u32 amc_drain(struct apple_amc *amc, int slot) ++{ ++ u32 count = readl(amc->base + AMC_COUNT(slot)); ++ u32 enable = readl(amc->base + AMC_ENABLE); ++ ++ writel(enable & ~BIT(slot), amc->base + AMC_ENABLE); ++ writel(0, amc->base + AMC_COUNT(slot)); ++ writel(enable, amc->base + AMC_ENABLE); ++ return count; ++} ++ ++/* lock held; widen before scaling (Apple's intermediate is only 32 bits). */ ++static void amc_update(struct apple_amc *amc) ++{ ++ u64 bytes[AMC_EVENTS]; ++ int i; ++ ++ if (!amc->running || amc->suspended) ++ return; ++ bytes[0] = (u64)amc_drain(amc, 0) * 128; ++ bytes[0] += (u64)amc_drain(amc, 4) * 2; ++ for (i = 1; i < AMC_EVENTS; i++) ++ bytes[i] = (u64)amc_drain(amc, i) * 2; ++ for_each_set_bit(i, &amc->running, AMC_EVENTS) ++ local64_add(bytes[i], &amc->events[i]->count); ++} ++ ++static void amc_enable(struct apple_amc *amc) ++{ ++ u32 mask = readl(amc->base + AMC_ENABLE) & ~AMC_PORT_MASK; ++ ++ if (amc->running && !amc->suspended) ++ mask |= AMC_PORT_MASK; ++ writel(mask, amc->base + AMC_ENABLE); ++} ++ ++static void amc_program(struct apple_amc *amc) ++{ ++ const struct apple_amc_data *d = amc->data; ++ int n; ++ ++ writel(readl(amc->base + AMC_ENABLE) & ~AMC_PORT_MASK, ++ amc->base + AMC_ENABLE); ++ for (n = 0; n < 5; n++) { ++ writel(n == 0 ? d->line_filter : d->byte_filter, ++ amc->base + AMC_CONFIG(n, 0)); ++ writel(0, amc->base + AMC_CONFIG(n, 1)); ++ writel(d->request_filter[n == 1 || n == 3], ++ amc->base + AMC_CONFIG(n, 2)); ++ writel(d->port_filter[n == 2 || n == 3], ++ amc->base + AMC_CONFIG(n, 3)); ++ writel(0, amc->base + AMC_CONFIG(n, 4)); ++ writel(0, amc->base + AMC_CONFIG(n, 5)); ++ writel(0, amc->base + AMC_COUNT(n)); ++ } ++ writel(d->control, amc->base + AMC_CONTROL); ++ amc_enable(amc); ++} ++ ++static enum hrtimer_restart amc_poll(struct hrtimer *timer) ++{ ++ struct apple_amc *amc = container_of(timer, struct apple_amc, timer); ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&amc->lock, flags); ++ amc_update(amc); ++ if (!amc->running || amc->suspended) { ++ raw_spin_unlock_irqrestore(&amc->lock, flags); ++ return HRTIMER_NORESTART; ++ } ++ hrtimer_forward_now(timer, ns_to_ktime(AMC_POLL_NS)); ++ raw_spin_unlock_irqrestore(&amc->lock, flags); ++ return HRTIMER_RESTART; ++} ++ ++static int amc_event_init(struct perf_event *event) ++{ ++ struct apple_amc *amc = to_amc(event->pmu); ++ struct perf_event *sibling; ++ unsigned long used = 0; ++ ++ if (event->attr.type != event->pmu->type) ++ return -ENOENT; ++ if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) ++ return -EOPNOTSUPP; ++ if (event->cpu < 0 || event->attr.config >= AMC_EVENTS || ++ event->attr.config1 || event->attr.config2) ++ return -EINVAL; ++ if (event->group_leader != event && !is_software_event(event->group_leader)) { ++ if (event->group_leader->pmu != event->pmu) ++ return -EINVAL; ++ used |= BIT(event->group_leader->attr.config); ++ } ++ for_each_sibling_event(sibling, event->group_leader) { ++ if (sibling == event || is_software_event(sibling)) ++ continue; ++ if (sibling->pmu != event->pmu) ++ return -EINVAL; ++ used |= BIT(sibling->attr.config); ++ } ++ if (used & BIT(event->attr.config)) ++ return -EINVAL; ++ event->cpu = amc->cpu; ++ event->hw.idx = event->attr.config; ++ return 0; ++} ++ ++static void amc_start(struct perf_event *event, int flags) ++{ ++ struct apple_amc *amc = to_amc(event->pmu); ++ unsigned long irqflags; ++ ++ raw_spin_lock_irqsave(&amc->lock, irqflags); ++ amc_update(amc); ++ /* Clear stale samples before the first active event. */ ++ if (!amc->running && !amc->suspended) { ++ amc_program(amc); ++ hrtimer_start(&amc->timer, ns_to_ktime(AMC_POLL_NS), HRTIMER_MODE_REL); ++ } ++ set_bit(event->hw.idx, &amc->running); ++ event->hw.state = 0; ++ amc_enable(amc); ++ raw_spin_unlock_irqrestore(&amc->lock, irqflags); ++} ++ ++static void amc_stop(struct perf_event *event, int flags) ++{ ++ struct apple_amc *amc = to_amc(event->pmu); ++ unsigned long irqflags; ++ ++ raw_spin_lock_irqsave(&amc->lock, irqflags); ++ amc_update(amc); ++ clear_bit(event->hw.idx, &amc->running); ++ event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; ++ amc_enable(amc); ++ raw_spin_unlock_irqrestore(&amc->lock, irqflags); ++} ++ ++static int amc_add(struct perf_event *event, int flags) ++{ ++ struct apple_amc *amc = to_amc(event->pmu); ++ unsigned long irqflags; ++ int idx = event->hw.idx; ++ ++ raw_spin_lock_irqsave(&amc->lock, irqflags); ++ if (amc->events[idx]) { ++ raw_spin_unlock_irqrestore(&amc->lock, irqflags); ++ return -EAGAIN; ++ } ++ amc->events[idx] = event; ++ event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; ++ raw_spin_unlock_irqrestore(&amc->lock, irqflags); ++ if (flags & PERF_EF_START) ++ amc_start(event, flags); ++ return 0; ++} ++ ++static void amc_del(struct perf_event *event, int flags) ++{ ++ struct apple_amc *amc = to_amc(event->pmu); ++ unsigned long irqflags; ++ ++ amc_stop(event, PERF_EF_UPDATE); ++ raw_spin_lock_irqsave(&amc->lock, irqflags); ++ amc->events[event->hw.idx] = NULL; ++ raw_spin_unlock_irqrestore(&amc->lock, irqflags); ++} ++ ++static void amc_read(struct perf_event *event) ++{ ++ struct apple_amc *amc = to_amc(event->pmu); ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&amc->lock, flags); ++ amc_update(amc); ++ raw_spin_unlock_irqrestore(&amc->lock, flags); ++} ++ ++static int amc_offline(unsigned int cpu, struct hlist_node *node) ++{ ++ struct apple_amc *amc = hlist_entry(node, struct apple_amc, node); ++ unsigned int target; ++ ++ if (cpu != amc->cpu) ++ return 0; ++ target = cpumask_any_but(cpu_online_mask, cpu); ++ if (target >= nr_cpu_ids) ++ return 0; ++ perf_pmu_migrate_context(&amc->pmu, cpu, target); ++ amc->cpu = target; ++ return 0; ++} ++ ++static int amc_suspend(struct device *dev) ++{ ++ struct apple_amc *amc = dev_get_drvdata(dev); ++ unsigned long flags; ++ ++ hrtimer_cancel(&amc->timer); ++ raw_spin_lock_irqsave(&amc->lock, flags); ++ amc_update(amc); ++ amc->suspended = true; ++ amc_enable(amc); ++ raw_spin_unlock_irqrestore(&amc->lock, flags); ++ return 0; ++} ++ ++static int amc_resume(struct device *dev) ++{ ++ struct apple_amc *amc = dev_get_drvdata(dev); ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&amc->lock, flags); ++ amc->suspended = false; ++ amc_program(amc); ++ raw_spin_unlock_irqrestore(&amc->lock, flags); ++ if (amc->running) ++ hrtimer_start(&amc->timer, ns_to_ktime(AMC_POLL_NS), HRTIMER_MODE_REL); ++ return 0; ++} ++static DEFINE_SIMPLE_DEV_PM_OPS(amc_pm_ops, amc_suspend, amc_resume); ++ ++static int amc_probe(struct platform_device *pdev) ++{ ++ struct apple_amc *amc; ++ struct resource *res; ++ int ret; ++ ++ amc = devm_kzalloc(&pdev->dev, sizeof(*amc), GFP_KERNEL); ++ if (!amc) ++ return -ENOMEM; ++ amc->data = of_device_get_match_data(&pdev->dev); ++ amc->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); ++ if (IS_ERR(amc->base)) ++ return PTR_ERR(amc->base); ++ if (resource_size(res) < AMC_COUNT(4) + sizeof(u32)) ++ return -EINVAL; ++ amc->cpu = raw_smp_processor_id(); ++ raw_spin_lock_init(&amc->lock); ++ hrtimer_setup(&amc->timer, amc_poll, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ platform_set_drvdata(pdev, amc); ++ amc->pmu = (struct pmu) { ++ .module = THIS_MODULE, .parent = &pdev->dev, ++ .task_ctx_nr = perf_invalid_context, ++ .capabilities = PERF_PMU_CAP_NO_EXCLUDE, ++ .attr_groups = amc_groups, .event_init = amc_event_init, ++ .add = amc_add, .del = amc_del, .start = amc_start, ++ .stop = amc_stop, .read = amc_read, ++ }; ++ ret = cpuhp_state_add_instance_nocalls(amc_hp_state, &amc->node); ++ if (ret) ++ return ret; ++ ret = perf_pmu_register(&amc->pmu, "apple_amc", -1); ++ if (ret) { ++ cpuhp_state_remove_instance_nocalls(amc_hp_state, &amc->node); ++ return ret; ++ } ++ /* First event starts polling and programs the hardware. */ ++ return 0; ++} ++ ++static void amc_remove(struct platform_device *pdev) ++{ ++ struct apple_amc *amc = platform_get_drvdata(pdev); ++ ++ perf_pmu_unregister(&amc->pmu); ++ hrtimer_cancel(&amc->timer); ++ cpuhp_state_remove_instance_nocalls(amc_hp_state, &amc->node); ++ amc->running = 0; ++ amc_enable(amc); ++} ++ ++static const struct apple_amc_data s8000_data = { ++ .port_filter = { 0xf20, 0xf40 }, ++ .request_filter = { 0x20031, 0x20032 }, ++ .byte_filter = 0x7ff0060, .line_filter = 0x7ff0070, ++ .control = 0x11, ++}; ++static const struct of_device_id amc_of_match[] = { ++ { .compatible = "apple,s8000-amc-pmu", .data = &s8000_data }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, amc_of_match); ++static struct platform_driver amc_driver = { ++ .probe = amc_probe, .remove = amc_remove, ++ .driver = { ++ .name = "apple-amc-pmu", .of_match_table = amc_of_match, ++ .pm = pm_sleep_ptr(&amc_pm_ops), ++ }, ++}; ++ ++static int __init amc_init(void) ++{ ++ int ret; ++ ++ amc_hp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, ++ "perf/apple-amc:offline", NULL, amc_offline); ++ if (amc_hp_state < 0) ++ return amc_hp_state; ++ ret = platform_driver_register(&amc_driver); ++ if (ret) ++ cpuhp_remove_multi_state(amc_hp_state); ++ return ret; ++} ++module_init(amc_init); ++ ++static void __exit amc_exit(void) ++{ ++ platform_driver_unregister(&amc_driver); ++ cpuhp_remove_multi_state(amc_hp_state); ++} ++module_exit(amc_exit); ++MODULE_DESCRIPTION("Apple A9 memory controller bandwidth PMU"); ++MODULE_LICENSE("GPL"); diff --git a/kernel/patches/drivers/phy/0001-phy-apple-ausb-Switch-the-PHY-between-host-and-devic.patch b/kernel/patches/drivers/phy/0001-phy-apple-ausb-Switch-the-PHY-between-host-and-devic.patch new file mode 100644 index 00000000..1174f7fe --- /dev/null +++ b/kernel/patches/drivers/phy/0001-phy-apple-ausb-Switch-the-PHY-between-host-and-devic.patch @@ -0,0 +1,165 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Fri, 30 Oct 2026 16:50:08 +0100 +Subject: [PATCH] phy: apple: ausb: Switch the PHY between host and device mode + +The AUSB PHY has separate tunables for the device and the host role. The +driver always loads the device tunables, and its set_mode callback was +disabled, so a host controller sharing the PHY ran it with device +tunables. + +Implement set_mode as iOS does when it moves the port between the two +roles: load the tunables of the new role, clear the trim word at 0x1c, +then pulse the PHY reset. AUSB_SIG is left alone. Only an actual mode +change pulses the reset, so a second host-side user does not. A +powered-down PHY takes the mode at its next power-on, and power-off +returns it to device mode for the next user. + +The DWC2 never sets a PHY mode, and no host controller node in the Apple +device trees has a phys property, so nothing changes until a user calls +phy_set_mode(): the S8000 Lightning role switch added next. + +The camera adapter sequence was exercised on an iPad 5 with an earlier +kernel. This shared-driver integration still needs hardware validation. + +Assisted-by: LLM +--- + drivers/phy/apple/ausb.c | 66 +++++++++++++++++++++++++++++----------- + 1 file changed, 48 insertions(+), 18 deletions(-) + +diff --git a/drivers/phy/apple/ausb.c b/drivers/phy/apple/ausb.c +index 39ba39d83..f3223b28d 100644 +--- a/drivers/phy/apple/ausb.c ++++ b/drivers/phy/apple/ausb.c +@@ -30,6 +30,7 @@ + /* Tunable registers */ + #define AUSB_CFG0 0x08 + #define AUSB_CFG1 0x0c ++#define AUSB_TRIM 0x1c + + struct ausb_usbphy { + struct device *dev; +@@ -40,6 +41,9 @@ struct ausb_usbphy { + u32 cfg1_device; + u32 cfg0_host; + u32 cfg1_host; ++ /* Serialized by the PHY core's phy->mutex. */ ++ bool host; ++ bool powered; + }; + + static inline void mask32(void __iomem *reg, u32 mask, u32 set) +@@ -60,11 +64,23 @@ static inline void clear32(void __iomem *reg, u32 clear) + mask32(reg, clear, 0); + } + +-#if 0 +-static __maybe_unused int ausb_phy_usb2_set_mode(struct phy *phy, enum phy_mode mode, +- int submode) ++static void ausb_phy_write_tunables(struct ausb_usbphy *aphy) ++{ ++ writel(aphy->host ? aphy->cfg0_host : aphy->cfg0_device, ++ aphy->base + AUSB_CFG0); ++ writel(aphy->host ? aphy->cfg1_host : aphy->cfg1_device, ++ aphy->base + AUSB_CFG1); ++} ++ ++/* ++ * Role change on a running PHY, as iOS does it in both directions: mode ++ * tunables, trim word 0, then a reset pulse. AUSB_SIG is left alone. ++ */ ++static int ausb_phy_usb2_set_mode(struct phy *phy, enum phy_mode mode, ++ int submode) + { + struct ausb_usbphy *aphy = phy_get_drvdata(phy); ++ bool host; + + switch (mode) { + case PHY_MODE_USB_HOST: +@@ -72,35 +88,45 @@ static __maybe_unused int ausb_phy_usb2_set_mode(struct phy *phy, enum phy_mode + case PHY_MODE_USB_HOST_FS: + case PHY_MODE_USB_HOST_HS: + case PHY_MODE_USB_HOST_SS: +- writel(aphy->cfg0_host, aphy->base + AUSB_CFG0); +- writel(aphy->cfg1_host, aphy->base + AUSB_CFG1); +- set32(aphy->base + AUSB_SIG, AUSB_SIG_HOST); +- return 0; +- ++ host = true; ++ break; + case PHY_MODE_USB_DEVICE: + case PHY_MODE_USB_DEVICE_LS: + case PHY_MODE_USB_DEVICE_FS: + case PHY_MODE_USB_DEVICE_HS: + case PHY_MODE_USB_DEVICE_SS: +- writel(aphy->cfg0_device, aphy->base + AUSB_CFG0); +- writel(aphy->cfg1_device, aphy->base + AUSB_CFG1); +- clear32(aphy->base + AUSB_SIG, AUSB_SIG_HOST); +- return 0; +- ++ host = false; ++ break; + default: + dev_err(aphy->dev, "Unknown mode for usb2 phy: %d\n", mode); + return -EINVAL; + } ++ ++ /* A second host-side user must not pulse the reset again. */ ++ if (host == aphy->host) ++ return 0; ++ aphy->host = host; ++ /* A powered-down PHY picks the mode up at its next power-on. */ ++ if (!aphy->powered) ++ return 0; ++ ++ ausb_phy_write_tunables(aphy); ++ writel(0, aphy->base + AUSB_TRIM); ++ set32(aphy->base + AUSB_CTL, AUSB_CTL_RESET); ++ usleep_range(20, 40); ++ clear32(aphy->base + AUSB_CTL, AUSB_CTL_RESET); ++ usleep_range(1000, 1500); ++ dev_dbg(aphy->dev, "%s mode\n", host ? "host" : "device"); ++ ++ return 0; + } +-#endif + + static int ausb_phy_usb2_power_on(struct phy *phy) + { + struct ausb_usbphy *aphy = phy_get_drvdata(phy); + +- /* set a sane tunable */ +- writel(aphy->cfg0_device, aphy->base + AUSB_CFG0); +- writel(aphy->cfg1_device, aphy->base + AUSB_CFG1); ++ /* set a sane tunable: device mode unless a host user asked first */ ++ ausb_phy_write_tunables(aphy); + + set32(aphy->base + AUSB_CTL, AUSB_CTL_RESET); + udelay(20); +@@ -109,6 +135,7 @@ static int ausb_phy_usb2_power_on(struct phy *phy) + clear32(aphy->base + AUSB_CTL, AUSB_CTL_RESET); + udelay(1500); + set32(aphy->base + AUSB_SIG, AUSB_SIG_VBUSDET_FORCE_EN); ++ aphy->powered = true; + + return 0; + } +@@ -124,6 +151,9 @@ static int ausb_phy_usb2_power_off(struct phy *phy) + clear32(aphy->base + AUSB_CTL, AUSB_CTL_RESET); + udelay(1500); + clear32(aphy->base + AUSB_SIG, AUSB_SIG_VBUSDET_FORCE_EN); ++ aphy->powered = false; ++ /* The device controller never sets a mode: default it for the next user. */ ++ aphy->host = false; + + return 0; + } +@@ -131,7 +161,7 @@ static int ausb_phy_usb2_power_off(struct phy *phy) + + static const struct phy_ops ausb_phy_usb2_ops = { + .owner = THIS_MODULE, +- //.set_mode = ausb_phy_usb2_set_mode, ++ .set_mode = ausb_phy_usb2_set_mode, + .init = ausb_phy_usb2_power_on, + .exit = ausb_phy_usb2_power_off, + }; diff --git a/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch b/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch new file mode 100644 index 00000000..dc5b66a6 --- /dev/null +++ b/kernel/patches/drivers/pinctrl/0004-pinctrl-apple-gpio-retire-inherited-GPIO-interrupt-m.patch @@ -0,0 +1,130 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] pinctrl: apple-gpio: retire inherited GPIO interrupt modes + after kexec + +After a kexec, or after a panic that ran no shutdown callbacks, GPIO +pins can still be in an interrupt mode set up by the previous kernel. +Once the chained parent interrupts are enabled, such a pin raises an +interrupt that nobody maps or acknowledges, and it keeps firing. + +On apple,s8000-pinctrl, switch every GPIO interrupt mode off at probe, +before the parent interrupts are enabled, and keep outputs, mux, pulls, +data and grouping. Consumers arm their own pins again later. In the +handler, a pending pin without a mapping is masked and its group latch +cleared, with a rate-limited warning. + +Assisted-by: LLM +--- + drivers/pinctrl/pinctrl-apple-gpio.c | 62 +++++++++++++++++++++++++++- + 1 file changed, 60 insertions(+), 2 deletions(-) + +diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c +index b97023c97..766fcf5bc 100644 +--- a/drivers/pinctrl/pinctrl-apple-gpio.c ++++ b/drivers/pinctrl/pinctrl-apple-gpio.c +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -46,6 +47,7 @@ struct apple_gpio_pinctrl { + + void __iomem *base; + struct regmap *map; ++ bool retire_irqs; + + struct pinctrl_desc pinctrl_desc; + struct gpio_chip gpio_chip; +@@ -379,8 +381,33 @@ static void apple_gpio_irq_handler(struct irq_desc *desc) + chained_irq_enter(chip, desc); + for (pinh = 0; pinh < gc->ngpio; pinh += 32) { + pending = readl_relaxed(pctl->base + REG_IRQ(*grpp, pinh)); +- for_each_set_bit(pinl, &pending, 32) +- generic_handle_domain_irq(gc->irq.domain, pinh + pinl); ++ for_each_set_bit(pinl, &pending, 32) { ++ unsigned int pin = pinh + pinl; ++ int ret; ++ ++ ret = pin < gc->ngpio ? ++ generic_handle_domain_irq(gc->irq.domain, pin) : -EINVAL; ++ if (ret && pctl->retire_irqs) { ++ /* An inherited, unmapped source has nobody to ack it. ++ * Mask only this GPIO, then clear this group's latch. ++ */ ++ if (pin < gc->ngpio) { ++ u32 reg = apple_gpio_get_reg(pctl, pin); ++ u32 mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ u32 off = FIELD_PREP(REG_GPIOx_MODE, ++ REG_GPIOx_IN_IRQ_OFF); ++ ++ if (!(reg & REG_GPIOx_PERIPH) && ++ mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, off); ++ } ++ writel(BIT(pinl), pctl->base + REG_IRQ(*grpp, pinh)); ++ dev_warn_ratelimited(pctl->dev, ++ "masked orphan inherited GPIO IRQ group=%u pin=%u\n", ++ *grpp, pin); ++ } ++ } + } + chained_irq_exit(chip, desc); + } +@@ -403,6 +430,35 @@ static const struct irq_chip apple_gpio_irqchip = { + + /* Probe & register */ + ++static void apple_gpio_quiesce_irqs(struct apple_gpio_pinctrl *pctl) ++{ ++ struct gpio_chip *gc = &pctl->gpio_chip; ++ unsigned int pin, group, mode, masked = 0; ++ u32 reg; ++ ++ if (!pctl->retire_irqs) ++ return; ++ /* Before chained parent IRQs are enabled, retire old GPIO IRQ modes. ++ * Preserve outputs, mux, pulls, data and grouping. Consumers rearm their ++ * own pins later; panic entry need not have run any shutdown callbacks. ++ */ ++ for (pin = 0; pin < gc->ngpio; pin++) { ++ reg = apple_gpio_get_reg(pctl, pin); ++ mode = FIELD_GET(REG_GPIOx_MODE, reg); ++ if (!(reg & REG_GPIOx_PERIPH) && mode >= REG_GPIOx_IN_IRQ_HI && ++ mode <= REG_GPIOx_IN_IRQ_ANY) { ++ apple_gpio_set_reg(pctl, pin, REG_GPIOx_MODE, ++ FIELD_PREP(REG_GPIOx_MODE, REG_GPIOx_IN_IRQ_OFF)); ++ masked++; ++ } ++ } ++ for (group = 0; group < gc->irq.num_parents; group++) ++ for (pin = 0; pin < gc->ngpio; pin += 32) ++ writel(GENMASK(min(32U, gc->ngpio - pin) - 1, 0), ++ pctl->base + REG_IRQ(group, pin)); ++ dev_info(pctl->dev, "inherited GPIO IRQ baseline: masked %u inherited inputs\n", masked); ++} ++ + static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + { + struct gpio_irq_chip *girq = &pctl->gpio_chip.irq; +@@ -452,6 +508,7 @@ static int apple_gpio_register(struct apple_gpio_pinctrl *pctl) + girq->handler = handle_level_irq; + } + ++ apple_gpio_quiesce_irqs(pctl); + ret = devm_gpiochip_add_data(pctl->dev, &pctl->gpio_chip, pctl); + + out_free_irq_data: +@@ -485,6 +542,7 @@ static int apple_gpio_pinctrl_probe(struct platform_device *pdev) + if (!pctl) + return -ENOMEM; + pctl->dev = &pdev->dev; ++ pctl->retire_irqs = of_device_is_compatible(pdev->dev.of_node, "apple,s8000-pinctrl"); + pctl->soc = of_device_get_match_data(&pdev->dev); + pctl->gpio_chip.irq.num_parents = nirqs; + dev_set_drvdata(&pdev->dev, pctl); diff --git a/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch b/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch new file mode 100644 index 00000000..d9a8480b --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0001-pmdomain-apple-select-certified-AOP-reload-through-D.patch @@ -0,0 +1,127 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 84 ++++++++++++++++++++++++++ + 1 file changed, 84 insertions(+) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..0ae0eda2d 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -17,6 +18,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -417,6 +419,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + goto err_remove; + } + ++ platform_set_drvdata(pdev, ps); + return 0; + err_remove: + if (ps->has_genpd) { +@@ -446,3 +449,84 @@ MODULE_AUTHOR("Hector Martin "); + MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); + + module_platform_driver(apple_pmgr_ps_driver); ++ ++/* ++ * Read-only observer for bring-up. The caller's nodes follow the same ++ * child -> parent order as genpd power transitions. No domain is attached, ++ * resumed, pinned on, reset, or written by this function. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data) ++{ ++ struct platform_device *providers[5]; ++ struct apple_pmgr_ps *ps[5]; ++ unsigned long flags; ++ unsigned int i, j, found = 0; ++ int ret = 0; ++ bool accessible = true; ++ ++ if (!count || count > ARRAY_SIZE(providers) || !read) ++ return -EINVAL; ++ for (i = 0; i < count; i++) { ++ for (j = 0; j < i; j++) ++ if (nodes[i] == nodes[j]) { ++ ret = -EINVAL; ++ goto put; ++ } ++ providers[i] = of_find_device_by_node(nodes[i]); ++ if (!providers[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ device_lock(&providers[i]->dev); ++ found++; ++ if (!providers[i]->dev.driver || ++ strcmp(providers[i]->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ ps[i] = platform_get_drvdata(providers[i]); ++ if (!ps[i]) { ++ ret = -EPROBE_DEFER; ++ goto put; ++ } ++ } ++ ++ /* PMGR domains use GENPD_FLAG_IRQ_SAFE, i.e. genpd.slock. */ ++ local_irq_save(flags); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->genpd.slock, i); ++ for (i = 0; i < count; i++) ++ spin_lock_nested(&ps[i]->lock, i); ++ for (i = 0; i < count; i++) { ++ states[i].genpd_status = ps[i]->genpd.status; ++ ret = regmap_read(ps[i]->regmap, ps[i]->offset, &states[i].raw); ++ if (ret) ++ goto unlock; ++ if (states[i].genpd_status != GENPD_STATE_ON || ++ FIELD_GET(APPLE_PMGR_PS_ACTUAL, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ FIELD_GET(APPLE_PMGR_PS_TARGET, states[i].raw) != APPLE_PMGR_PS_ACTIVE || ++ states[i].raw & (APPLE_PMGR_RESET | APPLE_PMGR_DEV_DISABLE | ++ APPLE_PMGR_PARENT_OFF)) ++ accessible = false; ++ } ++ if (accessible) ++ read(data); ++ else ++ ret = -EHOSTDOWN; ++unlock: ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->lock); ++ for (i = count; i > 0; i--) ++ spin_unlock(&ps[i - 1]->genpd.slock); ++ local_irq_restore(flags); ++put: ++ while (found) { ++ found--; ++ device_unlock(&providers[found]->dev); ++ put_device(&providers[found]->dev); ++ } ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_observe); diff --git a/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..0ae20f3c --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0002-pmdomain-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,113 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pmdomain: apple: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 34 +++++++++++++++++++++----- + 1 file changed, 28 insertions(+), 6 deletions(-) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 65dc32d30..7995d3b71 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -58,6 +58,8 @@ struct apple_pmgr_ps { + u32 min_state; + bool has_genpd; + bool manual_pm; ++ bool warm; ++ bool keep_on; + }; + + #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) +@@ -83,17 +85,21 @@ static int apple_pmgr_ps_set(struct apple_pmgr_ps *ps, u32 pstate, bool auto_ena + + dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", ps->name, pstate, reg); + +- regmap_write(ps->regmap, ps->offset, reg); ++ ret = regmap_write(ps->regmap, ps->offset, reg); ++ if (ret) ++ return ret; + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, reg, + (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, + APPLE_PMGR_PS_SET_TIMEOUT); +- if (ret < 0) ++ if (ret < 0) { + dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", + ps->name, pstate, reg); ++ return ret; ++ } + +- if (auto_enable) { ++ if (auto_enable && !ps->keep_on) { + /* Not all devices implement this; this is a no-op where not implemented. */ + reg &= ~APPLE_PMGR_FLAGS; + reg |= APPLE_PMGR_AUTO_ENABLE; +@@ -136,6 +142,9 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) + unsigned long flags; + int ret; + ++ if (ps->keep_on) ++ return -EBUSY; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_PWRGATE, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -168,6 +177,10 @@ static void apple_pmgr_clk_disable(struct clk_hw *hw) + unsigned long flags; + int ret; + ++ /* The lab has no separate clk gate sharing this power-state word. */ ++ if (ps->keep_on) ++ return; ++ + spin_lock_irqsave(&ps->lock, flags); + ret = apple_pmgr_ps_set(ps, gate->off_state, false); + spin_unlock_irqrestore(&ps->lock, flags); +@@ -306,6 +319,10 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + return ret; + } + ++ ps->warm = of_device_is_compatible(node, "apple,s8000-pmgr-pwrstate") && ++ of_property_read_bool(of_chosen, "linux,booted-from-kexec"); ++ ps->keep_on = ps->warm; ++ ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); + clock_only = of_property_read_bool(node, "apple,clock-gate-only"); + if (!clock_only) { + ps->genpd.flags |= GENPD_FLAG_IRQ_SAFE; +@@ -315,12 +332,17 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); +- if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) ++ if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && ++ !(ps->warm && ps->manual_pm)) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, + FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + + active = apple_pmgr_ps_is_active(ps); +- ps->manual_pm = of_property_read_bool(node, "apple,manual-pm"); ++ if (ps->warm && active && !ps->manual_pm) { ++ ret = apple_pmgr_ps_set(ps, APPLE_PMGR_PS_ACTIVE, false); ++ if (ret) ++ return ret; ++ } + if (!clock_only && of_property_read_bool(node, "apple,always-on")) { + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + if (!active) { +@@ -331,7 +353,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) + } + + /* Turn on auto-PM if the domain is already on */ +- if (active) ++ if (active && !ps->warm) + regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, + ps->manual_pm ? 0 : APPLE_PMGR_AUTO_ENABLE); + diff --git a/kernel/patches/drivers/pmdomain/0003-pmdomain-apple-end-the-warm-boot-hold-at-the-provide.patch b/kernel/patches/drivers/pmdomain/0003-pmdomain-apple-end-the-warm-boot-hold-at-the-provide.patch new file mode 100644 index 00000000..27bb4253 --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0003-pmdomain-apple-end-the-warm-boot-hold-at-the-provide.patch @@ -0,0 +1,185 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:08:04 +0100 +Subject: [PATCH] pmdomain: apple: end the warm boot hold at the provider's + sync_state + +On a kexec boot every S8000 power state keeps its inherited state: +power-off returns -EBUSY, clock-gate disable does nothing and auto-PM +stays off. That hold never ended, so for the whole warm boot no domain +was ever gated, at runtime or in system sleep, while a cold boot gates +every idle domain. genpd ignores the -EBUSY, so a warm system sleep +reported success with every domain still powered. + +The hold is only needed while the previous kernel's hardware is handed +over, that is until the domain's consumers have probed. genpd already +keeps an on-at-boot domain on until its provider's sync_state on a cold +boot. End the warm hold at the same point: in sync_state, drop keep_on, +turn auto-PM on for a domain that is still powered (as probe does on a +cold boot) and let genpd power off what nothing uses. From there the +domain behaves as on a cold boot. + +Export apple_pmgr_ps_power_held() so that a consumer can refuse a +transition that would need its domain off while the hold is still in +place, e.g. when one of the domain's consumers never probes. + +Cold boots are unchanged. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 69 +++++++++++++++++++++++-- + include/linux/soc/apple/pmgr-pwrstate.h | 23 +++++++++ + 2 files changed, 88 insertions(+), 4 deletions(-) + create mode 100644 include/linux/soc/apple/pmgr-pwrstate.h + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 1a8034d8f..4f7086330 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + + #define APPLE_PMGR_RESET BIT(31) + #define APPLE_PMGR_AUTO_ENABLE BIT(28) +@@ -61,6 +62,10 @@ struct apple_pmgr_ps { + bool has_genpd; + bool manual_pm; + bool warm; ++ /* ++ * Warm (kexec) boot only: hold the inherited state until this ++ * provider's sync_state, i.e. until every consumer has probed. ++ */ + bool keep_on; + }; + +@@ -101,7 +106,7 @@ static int apple_pmgr_ps_set(struct apple_pmgr_ps *ps, u32 pstate, bool auto_ena + return ret; + } + +- if (auto_enable && !ps->keep_on) { ++ if (auto_enable && !READ_ONCE(ps->keep_on)) { + /* Not all devices implement this; this is a no-op where not implemented. */ + reg &= ~APPLE_PMGR_FLAGS; + reg |= APPLE_PMGR_AUTO_ENABLE; +@@ -144,7 +149,7 @@ static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) + unsigned long flags; + int ret; + +- if (ps->keep_on) ++ if (READ_ONCE(ps->keep_on)) + return -EBUSY; + + spin_lock_irqsave(&ps->lock, flags); +@@ -179,8 +184,8 @@ static void apple_pmgr_clk_disable(struct clk_hw *hw) + unsigned long flags; + int ret; + +- /* The lab has no separate clk gate sharing this power-state word. */ +- if (ps->keep_on) ++ /* Held like the power domain until sync_state. */ ++ if (READ_ONCE(ps->keep_on)) + return; + + spin_lock_irqsave(&ps->lock, flags); +@@ -451,6 +456,61 @@ err_remove: + return ret; + } + ++/* ++ * Every consumer of this provider has probed: end a warm boot's hold and ++ * leave the domain as a cold boot would have it, with auto-PM on if it is ++ * still powered, then let genpd power it off if nothing uses it. From here ++ * on runtime PM and system sleep gate the domain as on a cold boot. ++ */ ++static void apple_pmgr_ps_sync_state(struct device *dev) ++{ ++ struct apple_pmgr_ps *ps = dev_get_drvdata(dev); ++ unsigned long flags; ++ u32 reg = 0; ++ ++ if (!ps) ++ return; ++ if (READ_ONCE(ps->keep_on)) { ++ spin_lock_irqsave(&ps->lock, flags); ++ WRITE_ONCE(ps->keep_on, false); ++ if (!ps->manual_pm && apple_pmgr_ps_is_active(ps)) ++ regmap_update_bits(ps->regmap, ps->offset, ++ APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, ++ APPLE_PMGR_AUTO_ENABLE); ++ regmap_read(ps->regmap, ps->offset, ®); ++ spin_unlock_irqrestore(&ps->lock, flags); ++ dev_info(dev, "PS %s: warm hold released reg=%08x\n", ps->name, reg); ++ } ++ if (ps->has_genpd) ++ of_genpd_sync_state(dev->of_node); ++} ++ ++/* ++ * 1 while a warm boot still holds this provider's domain on (power-off ++ * returns -EBUSY), 0 once released or on a cold boot, -EPROBE_DEFER while ++ * the provider is not bound. ++ */ ++int apple_pmgr_ps_power_held(struct device_node *np) ++{ ++ struct platform_device *pdev = of_find_device_by_node(np); ++ struct apple_pmgr_ps *ps; ++ int ret = -EPROBE_DEFER; ++ ++ if (!pdev) ++ return ret; ++ device_lock(&pdev->dev); ++ if (pdev->dev.driver && ++ !strcmp(pdev->dev.driver->name, "apple-pmgr-pwrstate")) { ++ ps = platform_get_drvdata(pdev); ++ if (ps) ++ ret = READ_ONCE(ps->keep_on); ++ } ++ device_unlock(&pdev->dev); ++ put_device(&pdev->dev); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_pmgr_ps_power_held); ++ + static const struct of_device_id apple_pmgr_ps_of_match[] = { + { .compatible = "apple,t8103-pmgr-pwrstate" }, + { .compatible = "apple,pmgr-pwrstate" }, +@@ -464,6 +524,7 @@ static struct platform_driver apple_pmgr_ps_driver = { + .driver = { + .name = "apple-pmgr-pwrstate", + .of_match_table = apple_pmgr_ps_of_match, ++ .sync_state = apple_pmgr_ps_sync_state, + }, + }; + +diff --git a/include/linux/soc/apple/pmgr-pwrstate.h b/include/linux/soc/apple/pmgr-pwrstate.h +new file mode 100644 +index 000000000..8e56d0de0 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-pwrstate.h +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_PWRSTATE_H ++#define _LINUX_SOC_APPLE_PMGR_PWRSTATE_H ++ ++#include ++ ++struct device_node; ++ ++/* ++ * 1 while a warm (kexec) boot still holds the domain of this PMGR power-state ++ * node on, 0 when it does not (released at the provider's sync_state, or a ++ * cold boot), -EPROBE_DEFER while the provider is not bound. ++ */ ++#if IS_ENABLED(CONFIG_APPLE_PMGR_PWRSTATE) ++int apple_pmgr_ps_power_held(struct device_node *np); ++#else ++static inline int apple_pmgr_ps_power_held(struct device_node *np) ++{ ++ return 0; ++} ++#endif ++ ++#endif diff --git a/kernel/patches/drivers/pmdomain/0004-pmdomain-apple-retain-the-warm-boot-hold-for-the-who.patch b/kernel/patches/drivers/pmdomain/0004-pmdomain-apple-retain-the-warm-boot-hold-for-the-who.patch new file mode 100644 index 00000000..58000619 --- /dev/null +++ b/kernel/patches/drivers/pmdomain/0004-pmdomain-apple-retain-the-warm-boot-hold-for-the-who.patch @@ -0,0 +1,102 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:37:37 +0100 +Subject: [PATCH] pmdomain: apple: retain the warm boot hold for the whole boot + +Releasing every inherited S8000 domain at sync_state was followed by +whole-SoC locks after kexec. Keep the hold until reboot while retained +hardware dependencies remain unresolved. System sleep is gated off for +the J71t release. + +Keep the hold-status API for consumers and the existing cold-boot genpd +sync_state path. The GPU AUTO-cycle handling and witnessed power-off +policy remain unchanged. + +Assisted-by: LLM +--- + drivers/pmdomain/apple/pmgr-pwrstate.c | 30 +++++++------------------ + include/linux/soc/apple/pmgr-pwrstate.h | 4 ++-- + 2 files changed, 10 insertions(+), 24 deletions(-) + +diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c +index 4f7086330..9994805a0 100644 +--- a/drivers/pmdomain/apple/pmgr-pwrstate.c ++++ b/drivers/pmdomain/apple/pmgr-pwrstate.c +@@ -63,8 +63,7 @@ struct apple_pmgr_ps { + bool manual_pm; + bool warm; + /* +- * Warm (kexec) boot only: hold the inherited state until this +- * provider's sync_state, i.e. until every consumer has probed. ++ * Warm (kexec) boot only: hold the inherited state for the whole boot. + */ + bool keep_on; + }; +@@ -184,7 +183,7 @@ static void apple_pmgr_clk_disable(struct clk_hw *hw) + unsigned long flags; + int ret; + +- /* Held like the power domain until sync_state. */ ++ /* Held like the power domain for the whole warm boot. */ + if (READ_ONCE(ps->keep_on)) + return; + +@@ -457,37 +456,24 @@ err_remove: + } + + /* +- * Every consumer of this provider has probed: end a warm boot's hold and +- * leave the domain as a cold boot would have it, with auto-PM on if it is +- * still powered, then let genpd power it off if nothing uses it. From here +- * on runtime PM and system sleep gate the domain as on a cold boot. ++ * Retained hardware can still depend on an unclaimed domain after all ++ * consumers have probed. Keep the warm hold for this entire boot: releasing ++ * all domains here was followed by whole-SoC locks after kexec on S8000. ++ * Cold boots retain the existing genpd sync_state path. + */ + static void apple_pmgr_ps_sync_state(struct device *dev) + { + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); +- unsigned long flags; +- u32 reg = 0; + +- if (!ps) ++ if (!ps || READ_ONCE(ps->keep_on)) + return; +- if (READ_ONCE(ps->keep_on)) { +- spin_lock_irqsave(&ps->lock, flags); +- WRITE_ONCE(ps->keep_on, false); +- if (!ps->manual_pm && apple_pmgr_ps_is_active(ps)) +- regmap_update_bits(ps->regmap, ps->offset, +- APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, +- APPLE_PMGR_AUTO_ENABLE); +- regmap_read(ps->regmap, ps->offset, ®); +- spin_unlock_irqrestore(&ps->lock, flags); +- dev_info(dev, "PS %s: warm hold released reg=%08x\n", ps->name, reg); +- } + if (ps->has_genpd) + of_genpd_sync_state(dev->of_node); + } + + /* + * 1 while a warm boot still holds this provider's domain on (power-off +- * returns -EBUSY), 0 once released or on a cold boot, -EPROBE_DEFER while ++ * returns -EBUSY), 0 on a cold boot, -EPROBE_DEFER while + * the provider is not bound. + */ + int apple_pmgr_ps_power_held(struct device_node *np) +diff --git a/include/linux/soc/apple/pmgr-pwrstate.h b/include/linux/soc/apple/pmgr-pwrstate.h +index 8e56d0de0..41f52d614 100644 +--- a/include/linux/soc/apple/pmgr-pwrstate.h ++++ b/include/linux/soc/apple/pmgr-pwrstate.h +@@ -8,8 +8,8 @@ struct device_node; + + /* + * 1 while a warm (kexec) boot still holds the domain of this PMGR power-state +- * node on, 0 when it does not (released at the provider's sync_state, or a +- * cold boot), -EPROBE_DEFER while the provider is not bound. ++ * node on for the whole boot, 0 on a cold boot, -EPROBE_DEFER while the ++ * provider is not bound. + */ + #if IS_ENABLED(CONFIG_APPLE_PMGR_PWRSTATE) + int apple_pmgr_ps_power_held(struct device_node *np); diff --git a/kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch b/kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch new file mode 100644 index 00000000..08928664 --- /dev/null +++ b/kernel/patches/drivers/power/0026-power-supply-apple-tristar-charger-cap-and-log-the-U.patch @@ -0,0 +1,82 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] power: supply: apple_tristar_charger: cap and log the USB + host budget + +An Apple host can grant more current than the standard USB budget, and +the gadget reports the granted budget to the charger. A stale report +from an earlier connection was already dropped, but a report above the +charger's maximum was passed on as is. + +Cap the reported budget and the input current limit at the charger's +maximum, re-evaluate the source when only the budget changes, and log +the reported and accepted values with the connection epoch. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 24 +++++++++++++++----- + 1 file changed, 18 insertions(+), 6 deletions(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index a953cedcb..c50bfc861 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -649,7 +649,12 @@ static int tristar_identify(struct tristar_charger *chg, struct tristar_ident *i + */ + static unsigned int tristar_usb_ma(struct tristar_charger *chg) + { +- return chg->usb_epoch == READ_ONCE(chg->epoch) ? chg->usb_ma : 0; ++ if (chg->usb_epoch != READ_ONCE(chg->epoch)) { ++ dev_dbg(chg->dev, "USB budget stale: report epoch %u, source epoch %u\n", ++ chg->usb_epoch, READ_ONCE(chg->epoch)); ++ return 0; ++ } ++ return chg->usb_ma; + } + + /* +@@ -1140,7 +1145,8 @@ static int tristar_get_property(struct power_supply *psy, + break; + case POWER_SUPPLY_PROP_CURRENT_MAX: + /* What the identified source or USB host allows; 0 when unknown. */ +- val->intval = max(chg->source_ma, tristar_usb_ma(chg)) * 1000; ++ val->intval = min(max(chg->source_ma, tristar_usb_ma(chg)), ++ chg->max_ma) * 1000; + break; + default: + mutex_unlock(&chg->lock); +@@ -1163,23 +1169,29 @@ static int tristar_set_property(struct power_supply *psy, + const union power_supply_propval *val) + { + struct tristar_charger *chg = power_supply_get_drvdata(psy); +- unsigned int old_code; ++ unsigned int old_code, old_ma, report_ma; + + if (psp != POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT) + return -EINVAL; + + mutex_lock(&chg->lock); + old_code = chg->code; +- chg->usb_ma = chg->present && chg->usb_on ? max(val->intval, 0) / 1000 : 0; ++ old_ma = tristar_usb_ma(chg); ++ report_ma = max(val->intval, 0) / 1000; ++ chg->usb_ma = chg->present && chg->usb_on ? ++ min(report_ma, chg->max_ma) : 0; + chg->usb_epoch = READ_ONCE(chg->epoch); +- dev_dbg(chg->dev, "USB host allows %u mA\n", chg->usb_ma); ++ dev_info_ratelimited(chg->dev, ++ "USB budget: reported %u mA, accepted %u mA, present %u, data %u, epoch %u\n", ++ report_ma, chg->usb_ma, chg->present, chg->usb_on, ++ chg->usb_epoch); + if (!chg->stopping) { + tristar_update_target(chg); + mod_delayed_work(chg->wq, &chg->poll_work, 0); + } + mutex_unlock(&chg->lock); + +- if (READ_ONCE(chg->code) != old_code) ++ if (READ_ONCE(chg->code) != old_code || old_ma != READ_ONCE(chg->usb_ma)) + power_supply_changed(psy); + return 0; + } diff --git a/kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch b/kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch new file mode 100644 index 00000000..7a81d907 --- /dev/null +++ b/kernel/patches/drivers/power/0027-power-supply-apple-tristar-select-J71t-USB-wake-even.patch @@ -0,0 +1,56 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] power: supply: apple-tristar: select J71t USB wake event + from match data + +The charger treats one TriStar event as the plug event that may wake the +system. J71t reports plug changes on event 0x49 instead of the default +one. Take the plug event from the match data and give +apple,j71t-tristar-charger 0x49. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index a953cedcb..8a869725d 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -124,6 +124,7 @@ struct tristar_charger { + struct completion adc_done; + int irqs[TRISTAR_IRQS_MAX]; + unsigned int nirqs; ++ unsigned int plug_event; + int plug_irq; /* wakes the system, 0 without one */ + u32 status_base; + u32 control_base; +@@ -1321,13 +1322,15 @@ static int tristar_probe(struct platform_device *pdev) + if (ret <= TRISTAR_IRQ_SOURCE || ret > TRISTAR_IRQS_MAX) + return dev_err_probe(dev, -EINVAL, + "expected PMU line, ADC and source event interrupts\n"); ++ chg->plug_event = device_get_match_data(dev) ? ++ *(const unsigned int *)device_get_match_data(dev) : TRISTAR_EVENT_PLUG; + chg->nirqs = ret; + for (i = 0; i < chg->nirqs; i++) { + chg->irqs[i] = platform_get_irq(pdev, i); + if (chg->irqs[i] < 0) + return chg->irqs[i]; + if (i >= TRISTAR_IRQ_SOURCE && +- irqd_to_hwirq(irq_get_irq_data(chg->irqs[i])) == TRISTAR_EVENT_PLUG) ++ irqd_to_hwirq(irq_get_irq_data(chg->irqs[i])) == chg->plug_event) + chg->plug_irq = chg->irqs[i]; + } + +@@ -1454,7 +1457,10 @@ static int tristar_resume(struct device *dev) + + static DEFINE_SIMPLE_DEV_PM_OPS(tristar_pm_ops, tristar_suspend, tristar_resume); + ++static const unsigned int j71t_plug_event = 0x49; ++ + static const struct of_device_id tristar_of_match[] = { ++ { .compatible = "apple,j71t-tristar-charger", .data = &j71t_plug_event }, + { .compatible = "apple,tristar-charger" }, + { } + }; diff --git a/kernel/patches/drivers/power/0028-dt-bindings-power-supply-apple-tristar-charger-Add-U.patch b/kernel/patches/drivers/power/0028-dt-bindings-power-supply-apple-tristar-charger-Add-U.patch new file mode 100644 index 00000000..52997a82 --- /dev/null +++ b/kernel/patches/drivers/power/0028-dt-bindings-power-supply-apple-tristar-charger-Add-U.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Fri, 30 Oct 2026 16:48:16 +0100 +Subject: [PATCH] dt-bindings: power: supply: apple,tristar-charger: Add USB + role switch + +On the iPad 5 the Lightning port is either the USB device (DWC2) or, for +the Lightning-to-USB camera adapter, a USB host (EHCI/OHCI), and a role +switch moves it between the two. The adapter announces itself over the +Lightning ID bus, which the charger's Tristar (CBTL1610) masters, so the +charger is where the decision is made. + +The charger already sets the device controller's role through its port. +Add an optional phandle to the role switch that gives the PHY to the +host controllers; while the port is a host, the device controller stays +off. A board without it (the iPad 6) keeps the port a USB device and +identifies no accessory. + +Assisted-by: LLM +--- + .../bindings/power/supply/apple,tristar-charger.yaml | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/Documentation/devicetree/bindings/power/supply/apple,tristar-charger.yaml b/Documentation/devicetree/bindings/power/supply/apple,tristar-charger.yaml +index c839ad0ac..59bc52c42 100644 +--- a/Documentation/devicetree/bindings/power/supply/apple,tristar-charger.yaml ++++ b/Documentation/devicetree/bindings/power/supply/apple,tristar-charger.yaml +@@ -60,6 +60,17 @@ properties: + $ref: /schemas/types.yaml#/definitions/phandle + description: The Tristar (CBTL1610) port controller on its I2C bus. + ++ apple,usb-role-switch: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: | ++ The role switch that gives the Lightning port's PHY to the USB host ++ controllers, such as the Apple S8000 USB role switch. When present, the ++ charger also identifies Lightning accessories over the Tristar ID bus ++ and makes the port a USB host while a camera adapter is attached, as ++ iOS does; the adapter then is the power source, and the device ++ controller linked through port stays off. Without it, the port stays a ++ USB device and no accessory is identified. ++ + input-current-limit-microamp: + description: Board maximum input current. + diff --git a/kernel/patches/drivers/power/0029-power-supply-apple_tristar_charger-Add-the-Lightning.patch b/kernel/patches/drivers/power/0029-power-supply-apple_tristar_charger-Add-the-Lightning.patch new file mode 100644 index 00000000..bc0d7bd9 --- /dev/null +++ b/kernel/patches/drivers/power/0029-power-supply-apple_tristar_charger-Add-the-Lightning.patch @@ -0,0 +1,700 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Fri, 30 Oct 2026 16:52:29 +0100 +Subject: [PATCH] power: supply: apple_tristar_charger: Add the Lightning + ID-bus master + +A Lightning accessory announces itself on the ID bus: asked for its +digital ID, it answers with six bytes that say what it is and which pins +it uses. The Tristar (CBTL1610) is the bus master. The host loads a +request, reversed, into the TX window that ends at register 0x60, reads +it back, starts the transfer through 0x61 and polls the event register +0x0d; the Tristar clocks the wire, frames BREAKs, appends the request +CRC and leaves the response in the RX window at 0x20. + +Add the master as the commands iOS sends, in its order and with its I2C +transaction shapes: GetID (0x74) after acknowledging a latched pin fault +and releasing a latched ID-bus reset hold; for the camera adapter the +follow-up iOS runs before USB host mode, the interface information +(0x76), the module and accessory serials (0x78, 0x7a) and two SystemInfo +packets (0x84) that the adapter normally leaves unanswered, where iOS's +timeout recovery (engine abort, BREAK before the next transfer) applies; +and the power handshake, GetState (0x72) then the power request (0x70). +Serial numbers read from the accessory are not kept. + +The ID-bus master is a separate object of the charger module, under the +new APPLE_TRISTAR_ACCESSORY option. The module keeps its name; only the +composite object is spelled with dashes, so that the charger's source +file keeps its name. + +The camera adapter sequence was exercised on an iPad 5 with an earlier +kernel. This shared-driver integration still needs hardware validation. + +Assisted-by: LLM +--- + drivers/power/supply/Kconfig | 11 + + drivers/power/supply/Makefile | 4 +- + drivers/power/supply/apple_tristar_idbus.c | 563 +++++++++++++++++++++ + drivers/power/supply/apple_tristar_idbus.h | 46 ++ + 4 files changed, 623 insertions(+), 1 deletion(-) + create mode 100644 drivers/power/supply/apple_tristar_idbus.c + create mode 100644 drivers/power/supply/apple_tristar_idbus.h + +diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig +index 6afca3ebe..50e012f32 100644 +--- a/drivers/power/supply/Kconfig ++++ b/drivers/power/supply/Kconfig +@@ -392,6 +392,17 @@ config APPLE_TRISTAR_CHARGER + This driver can also be built as a module. If so, the module will be + called apple_tristar_charger. + ++config APPLE_TRISTAR_ACCESSORY ++ bool "Lightning accessory identification" ++ depends on APPLE_TRISTAR_CHARGER ++ depends on USB_ROLE_SWITCH=y || USB_ROLE_SWITCH=APPLE_TRISTAR_CHARGER ++ help ++ Say Y here to let the Tristar charger master the Lightning ID bus, ++ over which Lightning accessories identify themselves: it asks an ++ accessory for its digital ID and runs the exchange iOS runs with ++ the camera adapter before USB host mode, including the power ++ handshake. ++ + config BATTERY_BQ27XXX_DT_UPDATES_NVM + bool "BQ27xxx support for update of NVM/flash data memory" + depends on BATTERY_BQ27XXX_I2C +diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile +index fedeaf842..654359c0d 100644 +--- a/drivers/power/supply/Makefile ++++ b/drivers/power/supply/Makefile +@@ -55,7 +55,9 @@ obj-$(CONFIG_BATTERY_BQ27XXX_HDQ) += bq27xxx_battery_hdq.o + obj-$(CONFIG_CHARGER_APPLE_SN2400) += apple_sn2400_charger.o + obj-$(CONFIG_APPLE_BELLATRIX_POWER) += apple_bellatrix_power.o + obj-$(CONFIG_APPLE_D2365_CHARGER) += apple_d2365_charger.o +-obj-$(CONFIG_APPLE_TRISTAR_CHARGER) += apple_tristar_charger.o ++obj-$(CONFIG_APPLE_TRISTAR_CHARGER) += apple-tristar-charger.o ++apple-tristar-charger-y := apple_tristar_charger.o ++apple-tristar-charger-$(CONFIG_APPLE_TRISTAR_ACCESSORY) += apple_tristar_idbus.o + obj-$(CONFIG_BATTERY_DA9030) += da9030_battery.o + obj-$(CONFIG_BATTERY_DA9052) += da9052-battery.o + obj-$(CONFIG_CHARGER_DA9150) += da9150-charger.o +diff --git a/drivers/power/supply/apple_tristar_idbus.c b/drivers/power/supply/apple_tristar_idbus.c +new file mode 100644 +index 000000000..8fb45b9a0 +--- /dev/null ++++ b/drivers/power/supply/apple_tristar_idbus.c +@@ -0,0 +1,563 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Lightning ID-bus master of the Tristar (CBTL1610) port controller. ++ * ++ * A Lightning accessory announces itself on the ID bus: asked for its ++ * digital ID, it answers with six bytes that say what it is and which pins ++ * it uses. The Tristar clocks the wire, frames BREAKs and appends the ++ * request CRC itself; the host drives it over I2C, as iOS does: ++ * ++ * 1. prepare the engine: 0x60 = 0x01; ++ * 2. write the request, reversed, into the TX window that ends at 0x60, ++ * with the RX length byte ((rx_len - 1) << 1) at 0x60 itself; ++ * 3. read the window back, and send nothing if it differs; ++ * 4. GO: 0x61 = ((tx_len - 1) << 1) | 1; ++ * 5. poll the event register 0x0d every 10 ms, for at most 100 ms; ++ * 6. read the response forward from the RX window at 0x20: the opcode, ++ * rx_len - 1 data bytes and a CRC over the first rx_len bytes. ++ * ++ * The RX window is never written: a write there overwrites the retained ++ * response. Only the control register 0x05, the pin-fault register 0x0a and ++ * the TX window 0x3c-0x61 are written; the mux registers 0x01-0x03 belong ++ * to the charger and the role switch. ++ * ++ * Copyright The Orchard Linux contributors ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple_tristar_idbus.h" ++ ++#define TRISTAR_IDBUS_REG_DXCTRL 0x01 ++#define TRISTAR_IDBUS_REG_ACCMUX 0x02 ++/* bit 0 VBUS sink (the role switch's), bit 2 BREAK, bit 3 ID-bus reset hold */ ++#define TRISTAR_IDBUS_REG_CTRL 0x05 ++#define TRISTAR_IDBUS_CTRL_BREAK 0x04 ++#define TRISTAR_IDBUS_CTRL_RESET 0x08 ++#define TRISTAR_IDBUS_REG_FAULT 0x0a ++#define TRISTAR_IDBUS_REG_STATUS 0x0c ++#define TRISTAR_IDBUS_STATUS_VBUS 0x03 ++#define TRISTAR_IDBUS_REG_EVENT 0x0d ++#define TRISTAR_IDBUS_REG_REVISION 0x0f ++#define TRISTAR_IDBUS_REG_AIDSW 0x17 ++#define TRISTAR_IDBUS_REG_RX 0x20 ++#define TRISTAR_IDBUS_REG_TX_END 0x60 ++#define TRISTAR_IDBUS_REG_GO 0x61 ++#define TRISTAR_IDBUS_TX_PREPARE 0x01 ++/* The revision whose ID-bus engine this file drives */ ++#define TRISTAR_IDBUS_CBTL1610_REV 0x8a ++ ++#define TRISTAR_IDBUS_EV_RESET 0x04 ++#define TRISTAR_IDBUS_EV_DIGITAL_ID 0x08 ++#define TRISTAR_IDBUS_EV_DONE 0x10 ++#define TRISTAR_IDBUS_EV_ERROR 0x20 ++#define TRISTAR_IDBUS_EV_TIMEOUT 0x40 ++#define TRISTAR_IDBUS_EV_FAULT 0x80 ++#define TRISTAR_IDBUS_EV_DONE_MASK 0x38 ++ ++/* Commands and their responses */ ++#define TRISTAR_IDBUS_GET_ID 0x74 ++#define TRISTAR_IDBUS_ID 0x75 ++#define TRISTAR_IDBUS_HOSTID 0x0002 /* the host's system ID, as iOS sends it */ ++#define TRISTAR_IDBUS_POWER 0x70 ++#define TRISTAR_IDBUS_POWER_ACK 0x71 ++#define TRISTAR_IDBUS_GET_STATE 0x72 ++#define TRISTAR_IDBUS_STATE 0x73 ++#define TRISTAR_IDBUS_GET_INTERFACE 0x76 ++#define TRISTAR_IDBUS_INTERFACE 0x77 ++#define TRISTAR_IDBUS_GET_MODULE_SN 0x78 ++#define TRISTAR_IDBUS_MODULE_SN 0x79 ++#define TRISTAR_IDBUS_GET_ACCESSORY_SN 0x7a ++#define TRISTAR_IDBUS_ACCESSORY_SN 0x7b ++#define TRISTAR_IDBUS_SYSTEM_INFO 0x84 ++#define TRISTAR_IDBUS_INFO_MODEL 0 ++#define TRISTAR_IDBUS_INFO_OSVERSION 1 ++/* 0x70 flag that iOS sets only when it retries */ ++#define TRISTAR_IDBUS_POWER_RETRY 0x08 ++ ++#define TRISTAR_IDBUS_MAX_TX (TRISTAR_IDBUS_INFO_MAX + 4) ++#define TRISTAR_IDBUS_MAX_FRAME 22 /* a serial: opcode, 20 bytes, CRC */ ++#define TRISTAR_IDBUS_ID_FRAME_LEN 8 ++ ++#define TRISTAR_IDBUS_POLL_MS 10 ++#define TRISTAR_IDBUS_POLL_STEPS 10 /* 100 ms per transfer */ ++#define TRISTAR_IDBUS_SETTLE_MS 20 ++ ++/* Read before every GetID, in this order; reading 0x0d also clears stale events. */ ++static const u8 tristar_idbus_baseline[] = { ++ TRISTAR_IDBUS_REG_STATUS, TRISTAR_IDBUS_REG_EVENT, TRISTAR_IDBUS_REG_CTRL, ++ TRISTAR_IDBUS_REG_DXCTRL, TRISTAR_IDBUS_REG_ACCMUX, TRISTAR_IDBUS_REG_AIDSW, ++ TRISTAR_IDBUS_REG_FAULT, ++}; ++ ++/* CRC-8, polynomial 0x31 reflected (0x8c), LSB first, seed 0xff. */ ++static u8 tristar_idbus_crc8(const u8 *buf, unsigned int len) ++{ ++ unsigned int i; ++ u8 crc = 0xff; ++ ++ while (len--) { ++ crc ^= *buf++; ++ for (i = 0; i < 8; i++) ++ crc = (crc >> 1) ^ ((crc & 1) ? 0x8c : 0); ++ } ++ return crc; ++} ++ ++/* ++ * A response is its opcode, rx_len - 1 data bytes and a CRC over the first ++ * rx_len bytes. A zero @type accepts any opcode with a good CRC, as iOS does ++ * for the SystemInfo replies. ++ */ ++static int tristar_idbus_check_frame(const u8 *frame, unsigned int rx_len, u8 type) ++{ ++ if (type && frame[0] != type) ++ return -EPROTO; ++ if (tristar_idbus_crc8(frame, rx_len) != frame[rx_len]) ++ return -EBADMSG; ++ return 0; ++} ++ ++/* ---- I2C access, in the transaction shapes iOS uses ---- */ ++ ++/* The register selector and STOP, then a separate read. */ ++static int tristar_idbus_reg_read(struct i2c_client *client, u8 reg) ++{ ++ union i2c_smbus_data data; ++ int ret; ++ ++ i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_WRITE, reg, I2C_SMBUS_BYTE, NULL); ++ if (!ret) ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_READ, 0, I2C_SMBUS_BYTE, &data); ++ i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ return ret ?: data.byte; ++} ++ ++/* ++ * The first I/O error is kept in ib->io_error and refuses every later access ++ * of the sequence, so nothing is written after a failure. ++ */ ++static int tristar_idbus_fail(struct tristar_idbus *ib, int err) ++{ ++ if (!ib->io_error) ++ ib->io_error = err; ++ return err; ++} ++ ++static int tristar_idbus_read(struct tristar_idbus *ib, u8 reg) ++{ ++ int ret; ++ ++ if (ib->io_error) ++ return ib->io_error; ++ ret = tristar_idbus_reg_read(ib->client, reg); ++ if (ret < 0) ++ return tristar_idbus_fail(ib, ret); ++ if (reg == TRISTAR_IDBUS_REG_CTRL) ++ ib->last_ctrl = ret; ++ return ret; ++} ++ ++static int tristar_idbus_write(struct tristar_idbus *ib, u8 reg, u8 val) ++{ ++ int ret; ++ ++ if (ib->io_error) ++ return ib->io_error; ++ ret = i2c_smbus_write_byte_data(ib->client, reg, val); ++ return ret ? tristar_idbus_fail(ib, ret) : 0; ++} ++ ++/* ++ * Read-modify-write that holds the bus throughout: the role switch writes ++ * the VBUS-sink bit of 0x05. Returns the value read back afterwards. ++ */ ++static int tristar_idbus_update(struct tristar_idbus *ib, u8 reg, u8 clear, u8 set) ++{ ++ struct i2c_client *client = ib->client; ++ union i2c_smbus_data data; ++ int ret; ++ ++ if (ib->io_error) ++ return ib->io_error; ++ i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_WRITE, reg, I2C_SMBUS_BYTE, NULL); ++ if (!ret) ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_READ, 0, I2C_SMBUS_BYTE, &data); ++ if (!ret) { ++ data.byte = (data.byte & ~clear) | set; ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_WRITE, reg, I2C_SMBUS_BYTE_DATA, &data); ++ } ++ i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ if (ret) ++ return tristar_idbus_fail(ib, ret); ++ return tristar_idbus_read(ib, reg); ++} ++ ++/* The address pointer and STOP, then a separate read of @len bytes. */ ++static int tristar_idbus_block_read(struct tristar_idbus *ib, u8 reg, u8 *buf, ++ unsigned int len) ++{ ++ struct i2c_client *client = ib->client; ++ struct i2c_msg sel = { ++ .addr = client->addr, ++ .flags = client->flags & I2C_M_TEN, ++ .len = 1, ++ .buf = ®, ++ }; ++ struct i2c_msg rx = { ++ .addr = client->addr, ++ .flags = (client->flags & I2C_M_TEN) | I2C_M_RD, ++ .len = len, ++ .buf = buf, ++ }; ++ int ret; ++ ++ if (ib->io_error) ++ return ib->io_error; ++ i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ ret = __i2c_transfer(client->adapter, &sel, 1); ++ if (ret == 1) ++ ret = __i2c_transfer(client->adapter, &rx, 1); ++ i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ if (ret == 1) ++ return 0; ++ return tristar_idbus_fail(ib, ret < 0 ? ret : -EIO); ++} ++ ++/* The address and the data in one write message. */ ++static int tristar_idbus_block_write(struct tristar_idbus *ib, u8 reg, const u8 *buf, ++ unsigned int len) ++{ ++ u8 msg[1 + TRISTAR_IDBUS_MAX_TX + 1]; ++ int ret; ++ ++ if (ib->io_error) ++ return ib->io_error; ++ if (len > TRISTAR_IDBUS_MAX_TX + 1) ++ return -EINVAL; ++ msg[0] = reg; ++ memcpy(msg + 1, buf, len); ++ ret = i2c_master_send(ib->client, msg, len + 1); ++ if (ret == (int)len + 1) ++ return 0; ++ return tristar_idbus_fail(ib, ret < 0 ? ret : -EIO); ++} ++ ++static int tristar_idbus_read_event(struct tristar_idbus *ib) ++{ ++ int ev = tristar_idbus_read(ib, TRISTAR_IDBUS_REG_EVENT); ++ ++ if (ev >= 0) ++ ib->last_event = ev; ++ return ev; ++} ++ ++static int tristar_idbus_check_revision(struct tristar_idbus *ib) ++{ ++ int ret = tristar_idbus_read(ib, TRISTAR_IDBUS_REG_REVISION); ++ ++ if (ret < 0) ++ return ret; ++ return ret == TRISTAR_IDBUS_CBTL1610_REV ? 0 : -ENODEV; ++} ++ ++/* ---- Transfers ---- */ ++ ++/* A pin fault (0x0d bit 7): acknowledge it by writing back what 0x0a reads. */ ++static int tristar_idbus_ack_fault(struct tristar_idbus *ib) ++{ ++ int ret = tristar_idbus_update(ib, TRISTAR_IDBUS_REG_FAULT, 0, 0); ++ ++ return ret < 0 ? ret : 0; ++} ++ ++/* 0x05 |= 0x04; the read-back is the 20 us the BREAK needs. */ ++static int tristar_idbus_send_break(struct tristar_idbus *ib) ++{ ++ int ret = tristar_idbus_update(ib, TRISTAR_IDBUS_REG_CTRL, 0, ++ TRISTAR_IDBUS_CTRL_BREAK); ++ ++ return ret < 0 ? ret : 0; ++} ++ ++/* ++ * One command and its response; see the top of this file. Completion is the ++ * transfer-done code or, for GetID, the digital-ID event. Returns 0, the ++ * response's header or CRC verdict, or -errno. ++ */ ++static int tristar_idbus_transfer(struct tristar_idbus *ib, const u8 *tx, ++ unsigned int tx_len, unsigned int rx_len, u8 rx_type, ++ bool digital_id, u8 *frame) ++{ ++ u8 win[TRISTAR_IDBUS_MAX_TX + 1], back[TRISTAR_IDBUS_MAX_TX + 1]; ++ u8 base = TRISTAR_IDBUS_REG_TX_END - tx_len, go = ((tx_len - 1) << 1) | 1; ++ unsigned int i, step; ++ int ev, ret; ++ ++ if (!tx_len || tx_len > TRISTAR_IDBUS_MAX_TX || !rx_len || ++ rx_len >= TRISTAR_IDBUS_MAX_FRAME) ++ return -EINVAL; ++ for (i = 0; i < tx_len; i++) ++ win[i] = tx[tx_len - 1 - i]; ++ win[tx_len] = (rx_len - 1) << 1; ++ ++ if (ib->break_pending) { ++ ib->break_pending = false; ++ ret = tristar_idbus_send_break(ib); ++ if (ret) ++ return ret; ++ } ++ ret = tristar_idbus_write(ib, TRISTAR_IDBUS_REG_TX_END, TRISTAR_IDBUS_TX_PREPARE); ++ if (!ret) ++ ret = tristar_idbus_block_write(ib, base, win, tx_len + 1); ++ if (!ret) ++ ret = tristar_idbus_block_read(ib, base, back, tx_len + 1); ++ if (ret) ++ return ret; ++ if (memcmp(back, win, tx_len + 1)) ++ return -EREMOTEIO; ++ ret = tristar_idbus_write(ib, TRISTAR_IDBUS_REG_GO, go); ++ if (ret) ++ return ret; ++ ++ for (step = 0; ; step++) { ++ ev = tristar_idbus_read_event(ib); ++ if (ev < 0) ++ return ev; ++ if (ev & TRISTAR_IDBUS_EV_FAULT) { ++ ret = tristar_idbus_ack_fault(ib); ++ if (ret) ++ return ret; ++ } ++ if ((digital_id && (ev & TRISTAR_IDBUS_EV_DIGITAL_ID)) || ++ (ev & TRISTAR_IDBUS_EV_DONE_MASK) == TRISTAR_IDBUS_EV_DONE) ++ break; ++ if (ev & TRISTAR_IDBUS_EV_ERROR) ++ return -EIO; ++ if (ev & TRISTAR_IDBUS_EV_RESET) ++ return -ECONNRESET; ++ if ((ev & TRISTAR_IDBUS_EV_TIMEOUT) || step == TRISTAR_IDBUS_POLL_STEPS) ++ return -ETIMEDOUT; ++ fsleep(TRISTAR_IDBUS_POLL_MS * USEC_PER_MSEC); ++ } ++ ++ ret = tristar_idbus_block_read(ib, TRISTAR_IDBUS_REG_RX, frame, rx_len + 1); ++ if (ret) ++ return ret; ++ /* With the digital ID, iOS also reads the status (orientation). */ ++ if (digital_id) { ++ ret = tristar_idbus_read(ib, TRISTAR_IDBUS_REG_STATUS); ++ if (ret < 0) ++ return ret; ++ } ++ return tristar_idbus_check_frame(frame, rx_len, rx_type); ++} ++ ++/** ++ * tristar_idbus_status - read the Tristar status register ++ * @ib: ID-bus master ++ * ++ * Bits 0 and 1 are VBUS, bit 6 the plug orientation, bit 7 an attached plug. ++ * ++ * Return: the register value, -ENODEV for another Tristar revision, or ++ * -errno. ++ */ ++int tristar_idbus_status(struct tristar_idbus *ib) ++{ ++ int ret = tristar_idbus_reg_read(ib->client, TRISTAR_IDBUS_REG_REVISION); ++ ++ if (ret < 0) ++ return ret; ++ if (ret != TRISTAR_IDBUS_CBTL1610_REV) ++ return -ENODEV; ++ return tristar_idbus_reg_read(ib->client, TRISTAR_IDBUS_REG_STATUS); ++} ++ ++/** ++ * tristar_idbus_identify - ask the accessory for its digital ID ++ * @ib: ID-bus master ++ * @id: TRISTAR_IDBUS_ID_LEN bytes for the ID ++ * ++ * Acknowledges a latched pin fault and releases a latched ID-bus reset hold ++ * first. One GetID, without a reorientation: a reorientation can move a live ++ * USB session to the other lanes. ++ * ++ * Return: 0 with @id filled, or -errno. ++ */ ++int tristar_idbus_identify(struct tristar_idbus *ib, u8 *id) ++{ ++ static const u8 tx[] = { ++ TRISTAR_IDBUS_GET_ID, TRISTAR_IDBUS_HOSTID >> 8, TRISTAR_IDBUS_HOSTID & 0xff, ++ }; ++ u8 frame[TRISTAR_IDBUS_ID_FRAME_LEN]; ++ unsigned int i; ++ int ret, val; ++ ++ ib->io_error = 0; ++ ib->break_pending = false; ++ ret = tristar_idbus_check_revision(ib); ++ if (ret) ++ return ret; ++ ++ for (i = 0; i < ARRAY_SIZE(tristar_idbus_baseline); i++) { ++ u8 reg = tristar_idbus_baseline[i]; ++ ++ val = reg == TRISTAR_IDBUS_REG_EVENT ? tristar_idbus_read_event(ib) : ++ tristar_idbus_read(ib, reg); ++ if (val < 0) ++ return val; ++ } ++ /* Then the RX window as the last exchange left it, read only. */ ++ ret = tristar_idbus_block_read(ib, TRISTAR_IDBUS_REG_RX, frame, sizeof(frame)); ++ if (ret) ++ return ret; ++ if (ib->last_event & TRISTAR_IDBUS_EV_FAULT) { ++ ret = tristar_idbus_ack_fault(ib); ++ if (ret) ++ return ret; ++ } ++ if (ib->last_ctrl & TRISTAR_IDBUS_CTRL_RESET) { ++ val = tristar_idbus_update(ib, TRISTAR_IDBUS_REG_CTRL, ++ TRISTAR_IDBUS_CTRL_RESET, 0); ++ if (val < 0) ++ return val; ++ fsleep(TRISTAR_IDBUS_SETTLE_MS * USEC_PER_MSEC); ++ if (val & TRISTAR_IDBUS_CTRL_RESET) ++ return -EBUSY; ++ } ++ ++ ret = tristar_idbus_transfer(ib, tx, sizeof(tx), TRISTAR_IDBUS_ID_FRAME_LEN - 1, ++ TRISTAR_IDBUS_ID, true, frame); ++ if (!ret) ++ memcpy(id, frame + 1, TRISTAR_IDBUS_ID_LEN); ++ return ret; ++} ++ ++/* ++ * One SystemInfo packet. The adapter normally does not answer; iOS then ++ * advances when the transfer times out. That timeout runs iOS's recovery: ++ * the engine is aborted (0x60 = 0x01) and the next transfer starts with a ++ * BREAK. Any other failure, or a reply with a bad CRC, is an error. ++ */ ++static int tristar_idbus_system_info(struct tristar_idbus *ib, u8 type, ++ const u8 *data, unsigned int len) ++{ ++ u8 tx[TRISTAR_IDBUS_MAX_TX], reply[2]; ++ int ret; ++ ++ len = min(len, TRISTAR_IDBUS_INFO_MAX); ++ tx[0] = TRISTAR_IDBUS_SYSTEM_INFO; ++ tx[1] = type; ++ tx[2] = len >> 8; ++ tx[3] = len; ++ memcpy(tx + 4, data, len); ++ ret = tristar_idbus_transfer(ib, tx, len + 4, 1, 0, false, reply); ++ if (ret != -ETIMEDOUT || ib->io_error) ++ return ret; ++ ret = tristar_idbus_write(ib, TRISTAR_IDBUS_REG_TX_END, TRISTAR_IDBUS_TX_PREPARE); ++ if (!ret) ++ ib->break_pending = true; ++ return ret; ++} ++ ++/** ++ * tristar_idbus_accessory_info - the exchange iOS runs after the digital ID ++ * @ib: ID-bus master ++ * @model: the host's model number, sent as SystemInfo type 0 ++ * @model_len: its length; at most TRISTAR_IDBUS_INFO_MAX bytes are sent ++ * @osversion: the host's OS version, sent as SystemInfo type 1 ++ * @osversion_len: its length; at most TRISTAR_IDBUS_INFO_MAX bytes are sent ++ * ++ * For the camera adapter iOS reads the interface information (0x76), the ++ * module serial (0x78) and the accessory serial (0x7a), then sends the two ++ * SystemInfo packets (0x84). The adapter expects this exchange before the ++ * power handshake. The serials are not kept. ++ * ++ * Return: 0, or -errno. ++ */ ++int tristar_idbus_accessory_info(struct tristar_idbus *ib, ++ const u8 *model, unsigned int model_len, ++ const u8 *osversion, unsigned int osversion_len) ++{ ++ static const struct { ++ u8 cmd, rx_len, rx_type; ++ } reads[] = { ++ { TRISTAR_IDBUS_GET_INTERFACE, 11, TRISTAR_IDBUS_INTERFACE }, ++ { TRISTAR_IDBUS_GET_MODULE_SN, 21, TRISTAR_IDBUS_MODULE_SN }, ++ { TRISTAR_IDBUS_GET_ACCESSORY_SN, 21, TRISTAR_IDBUS_ACCESSORY_SN }, ++ }; ++ u8 frame[TRISTAR_IDBUS_MAX_FRAME]; ++ unsigned int i; ++ int ret; ++ ++ ib->io_error = 0; ++ ib->break_pending = false; ++ ret = tristar_idbus_check_revision(ib); ++ for (i = 0; i < ARRAY_SIZE(reads) && !ret; i++) ++ ret = tristar_idbus_transfer(ib, &reads[i].cmd, 1, reads[i].rx_len, ++ reads[i].rx_type, false, frame); ++ memzero_explicit(frame, sizeof(frame)); ++ if (!ret) ++ ret = tristar_idbus_system_info(ib, TRISTAR_IDBUS_INFO_MODEL, ++ model, model_len); ++ if (!ret) ++ ret = tristar_idbus_system_info(ib, TRISTAR_IDBUS_INFO_OSVERSION, ++ osversion, osversion_len); ++ /* A pending BREAK is kept for the power handshake that follows. */ ++ if (ret) ++ ib->break_pending = false; ++ return ret; ++} ++ ++/** ++ * tristar_idbus_power_handshake - agree on accessory power ++ * @ib: ID-bus master ++ * ++ * With VBUS present: GetState (0x72), answered by 0x73 and four state ++ * bytes, then the power request (0x70) echoing the first two state bytes ++ * without the retry flag, answered by 0x71. No retries here. ++ * ++ * Return: 0, -ENODEV without VBUS, -EBUSY with the ID bus held in reset, ++ * or -errno. ++ */ ++int tristar_idbus_power_handshake(struct tristar_idbus *ib) ++{ ++ static const u8 get_state = TRISTAR_IDBUS_GET_STATE; ++ u8 state[6], request[3], ack[2]; ++ int ret, val; ++ ++ ib->io_error = 0; ++ ret = tristar_idbus_check_revision(ib); ++ if (ret) ++ return ret; ++ val = tristar_idbus_read(ib, TRISTAR_IDBUS_REG_STATUS); ++ if (val < 0) ++ return val; ++ if ((val & TRISTAR_IDBUS_STATUS_VBUS) != TRISTAR_IDBUS_STATUS_VBUS) ++ return -ENODEV; ++ val = tristar_idbus_read(ib, TRISTAR_IDBUS_REG_CTRL); ++ if (val < 0) ++ return val; ++ if (val & TRISTAR_IDBUS_CTRL_RESET) ++ return -EBUSY; ++ ret = tristar_idbus_transfer(ib, &get_state, 1, 5, TRISTAR_IDBUS_STATE, false, state); ++ if (ret) ++ return ret; ++ request[0] = TRISTAR_IDBUS_POWER; ++ request[1] = state[1] & ~TRISTAR_IDBUS_POWER_RETRY; ++ request[2] = state[2]; ++ return tristar_idbus_transfer(ib, request, sizeof(request), 1, ++ TRISTAR_IDBUS_POWER_ACK, false, ack); ++} +diff --git a/drivers/power/supply/apple_tristar_idbus.h b/drivers/power/supply/apple_tristar_idbus.h +new file mode 100644 +index 000000000..7a47af814 +--- /dev/null ++++ b/drivers/power/supply/apple_tristar_idbus.h +@@ -0,0 +1,46 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Lightning ID-bus master of the Tristar (CBTL1610) port controller. ++ * ++ * Copyright The Orchard Linux contributors ++ */ ++#ifndef _APPLE_TRISTAR_IDBUS_H ++#define _APPLE_TRISTAR_IDBUS_H ++ ++#include ++ ++struct i2c_client; ++ ++#define TRISTAR_IDBUS_ID_LEN 6 ++/* Longest SystemInfo string: 32 data bytes after opcode, type and length */ ++#define TRISTAR_IDBUS_INFO_MAX 32 ++ ++/** ++ * struct tristar_idbus - ID-bus master state ++ * @client: the Tristar ++ * @io_error: first I/O error of the current command sequence; every later ++ * register access of that sequence is refused ++ * @break_pending: a SystemInfo packet timed out; the next transfer starts ++ * with an ID-bus BREAK ++ * @last_ctrl: the control register as last read ++ * @last_event: the event register as last read ++ * ++ * The caller serializes all use of one instance with the other users of the ++ * Tristar. ++ */ ++struct tristar_idbus { ++ struct i2c_client *client; ++ int io_error; ++ bool break_pending; ++ u8 last_ctrl; ++ u8 last_event; ++}; ++ ++int tristar_idbus_status(struct tristar_idbus *ib); ++int tristar_idbus_identify(struct tristar_idbus *ib, u8 *id); ++int tristar_idbus_accessory_info(struct tristar_idbus *ib, ++ const u8 *model, unsigned int model_len, ++ const u8 *osversion, unsigned int osversion_len); ++int tristar_idbus_power_handshake(struct tristar_idbus *ib); ++ ++#endif /* _APPLE_TRISTAR_IDBUS_H */ diff --git a/kernel/patches/drivers/power/0030-power-supply-apple_tristar_charger-Host-mode-for-the.patch b/kernel/patches/drivers/power/0030-power-supply-apple_tristar_charger-Host-mode-for-the.patch new file mode 100644 index 00000000..99819c6b --- /dev/null +++ b/kernel/patches/drivers/power/0030-power-supply-apple_tristar_charger-Host-mode-for-the.patch @@ -0,0 +1,1328 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Fri, 30 Oct 2026 17:12:29 +0100 +Subject: [PATCH] power: supply: apple_tristar_charger: Host mode for the + camera adapter + +The Lightning-to-USB camera adapter (digital ID 11 f0) has the iPad act +as USB host. iOS identifies it on every attach, runs the post-ID +exchange with accessory power on, makes the port a host, then agrees on +power over the ID bus; a charger in the adapter's own port then supplies +up to 2.1 A while the iPad is the host. On unplug the port returns to +the USB device controller. + +Add that policy to the charger, active only when the device tree names +the port's role switch (apple,usb-role-switch). Every 250 ms it reads +the Tristar status (VBUS, orientation, plug present) inside the role +switch's monitor, so that no role change overlaps the read. With a plug +attached it asks for the digital ID 300 ms after the plug edge and then +every 2 s (10 s while a USB host has configured the gadget). For the +adapter it switches accessory power on, runs the post-ID exchange, +claims the next host entry and requests the host role, then runs the +power handshake once strict VBUS appears (two tries, 15 s deadline) and +keeps the role, re-checking the ID every 5 s. Unplug detection follows +iOS: a fully empty status at once (three reads); while the adapter is +ours, plug-present clear with VBUS lingering for 2 s (an unplug in host +mode can read 0x25); plug-present back after 0.5 s or more is a replug, +which releases the role and identifies again. A different ID in host +mode (a fast swap to a computer's cable) releases the role as well. + +The charger and the policy share the Tristar under the charger's lock. +While the policy holds the port (from the host request until the role is +given back, and around each accessory-power or role change), the charger +does not sample the mux; it takes the adapter's allowance as the source, +2.1 A after a good power handshake and 500 mA without one, through its +usual ramp and battery checks, and identifies the source again once the +policy lets go. Role-switch calls are never made with the charger's lock +held, since the role switch calls back into the charger with its own +lock held. The policy stops with the charger's poll on remove, shutdown +and suspend, and gives a held host role back on unbind. The policy looks +the role switch up from its own poll, so charging never waits for the +role switch to bind. The charger owns the device controller's role (off +until the source is identified). While the policy holds the port, that +role stays off even once the source is identified, so the device +controller never pulls up while the PHY serves the host controllers; the +first poll after the policy lets go turns it on, or identifies the +source again if it was the adapter. Stopping the charger always turns it +on. + +A board without the role switch (the iPad 6) gets no new polling and no +extra Tristar traffic; with the option off, the charger is unchanged. + +The interface between the policy and the role switch driver, which +implements it, is added here as include/linux/usb/apple-s8000-role.h: +the policy claims the next host entry, switches accessory power and lets +the role switch sample the Tristar status while it holds the role. +Without the role switch driver the header's stubs return -EOPNOTSUPP. + +The camera adapter sequence was exercised on an iPad 5 with an earlier +kernel. This shared-driver integration still needs hardware validation. + +Assisted-by: LLM +--- + drivers/power/supply/Kconfig | 14 +- + drivers/power/supply/Makefile | 3 +- + .../power/supply/apple_tristar_accessory.c | 883 ++++++++++++++++++ + .../power/supply/apple_tristar_accessory.h | 55 ++ + drivers/power/supply/apple_tristar_charger.c | 85 +- + include/linux/usb/apple-s8000-role.h | 48 + + 6 files changed, 1079 insertions(+), 9 deletions(-) + create mode 100644 drivers/power/supply/apple_tristar_accessory.c + create mode 100644 drivers/power/supply/apple_tristar_accessory.h + create mode 100644 include/linux/usb/apple-s8000-role.h + +diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig +index 50e012f32..45bdd88d1 100644 +--- a/drivers/power/supply/Kconfig ++++ b/drivers/power/supply/Kconfig +@@ -396,12 +396,16 @@ config APPLE_TRISTAR_ACCESSORY + bool "Lightning accessory identification" + depends on APPLE_TRISTAR_CHARGER + depends on USB_ROLE_SWITCH=y || USB_ROLE_SWITCH=APPLE_TRISTAR_CHARGER ++ depends on USB_ROLES_APPLE_S8000=y || USB_ROLES_APPLE_S8000=APPLE_TRISTAR_CHARGER + help +- Say Y here to let the Tristar charger master the Lightning ID bus, +- over which Lightning accessories identify themselves: it asks an +- accessory for its digital ID and runs the exchange iOS runs with +- the camera adapter before USB host mode, including the power +- handshake. ++ Say Y here to let the Tristar charger identify Lightning ++ accessories over the Lightning ID bus and make the port a USB host ++ while a Lightning-to-USB camera adapter is attached, as iOS does. ++ A charger in the adapter's own port then powers the device. ++ ++ It needs the port's USB role switch (USB_ROLES_APPLE_S8000) and ++ is active only on boards whose device tree names that switch for ++ the charger. + + config BATTERY_BQ27XXX_DT_UPDATES_NVM + bool "BQ27xxx support for update of NVM/flash data memory" +diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile +index 654359c0d..53a6668f3 100644 +--- a/drivers/power/supply/Makefile ++++ b/drivers/power/supply/Makefile +@@ -57,7 +57,8 @@ obj-$(CONFIG_APPLE_BELLATRIX_POWER) += apple_bellatrix_power.o + obj-$(CONFIG_APPLE_D2365_CHARGER) += apple_d2365_charger.o + obj-$(CONFIG_APPLE_TRISTAR_CHARGER) += apple-tristar-charger.o + apple-tristar-charger-y := apple_tristar_charger.o +-apple-tristar-charger-$(CONFIG_APPLE_TRISTAR_ACCESSORY) += apple_tristar_idbus.o ++apple-tristar-charger-$(CONFIG_APPLE_TRISTAR_ACCESSORY) += apple_tristar_accessory.o \ ++ apple_tristar_idbus.o + obj-$(CONFIG_BATTERY_DA9030) += da9030_battery.o + obj-$(CONFIG_BATTERY_DA9052) += da9052-battery.o + obj-$(CONFIG_CHARGER_DA9150) += da9150-charger.o +diff --git a/drivers/power/supply/apple_tristar_accessory.c b/drivers/power/supply/apple_tristar_accessory.c +new file mode 100644 +index 000000000..f3fa81e02 +--- /dev/null ++++ b/drivers/power/supply/apple_tristar_accessory.c +@@ -0,0 +1,883 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Lightning accessory policy of the Tristar charger: USB host mode for the ++ * Lightning-to-USB camera adapter. ++ * ++ * The camera adapter announces itself with digital ID 11 f0 (USB, UART and ++ * power pins). On it iOS runs the post-ID exchange (interface information, ++ * serials, SystemInfo), makes the port a USB host, and then agrees on power ++ * with the adapter over the ID bus; the adapter's hub enumerates at high ++ * speed, and a charger in the adapter's own port may then supply 2.1 A while ++ * the iPad is the host. There is no device-mode phase in between. This ++ * policy does the same through the port's USB role switch, which owns the ++ * host controllers and the Tristar's data and accessory-power routes. When ++ * the adapter is unplugged, the port returns to the USB device controller. ++ * ++ * Every 250 ms the policy reads the Tristar status (0x0c: bits 0-1 VBUS, ++ * bit 6 orientation, bit 7 an attached plug). With a plug attached it asks ++ * for the digital ID 300 ms after the plug edge, then every 2 s, or every ++ * 10 s while a USB host uses the gadget; with the adapter in host mode it ++ * checks the ID every 5 s. An unplug is three empty reads in a row; while ++ * the adapter is ours, bit 7 alone counts (an unplug in host mode can leave ++ * the VBUS bits set) and the wait is 2 s; bit 7 back after 0.5 s or more is ++ * a replug and starts over, as iOS identifies on every attach. ++ * ++ * Sharing the Tristar with the charger: every Tristar access of either side ++ * holds the charger's lock, so ID-bus transfers and mux samples never ++ * interleave. The charger samples the mux only while the policy does not ++ * hold the port: from the request for host mode until it is given back, and ++ * around each change of accessory power or USB role, the charger takes the ++ * adapter's allowance instead of identifying the source (see ++ * tristar_accessory_holds()). The hold is taken under the lock before any ++ * role-switch call, so no mux sample is in flight when the role switch ++ * rewrites the routes. Role-switch calls are never made with the lock held: ++ * the role switch calls back into the charger (the status sample below, the ++ * gadget's vbus_draw) with its own lock held. ++ * ++ * Copyright The Orchard Linux contributors ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple_tristar_accessory.h" ++#include "apple_tristar_idbus.h" ++ ++/* Tristar status 0x0c: both low bits are strict VBUS, bit 7 an attached plug. */ ++#define TRISTAR_AUTO_VBUS 0x03 ++#define TRISTAR_AUTO_PRESENT 0x80 ++#define TRISTAR_AUTO_ORIENTATION 0x40 ++ ++#define TRISTAR_ACCESSORY_POLL_MS 250 ++/* A power handshake needs a digital ID read this recently. */ ++#define TRISTAR_ACCESSORY_ID_WINDOW_MS 30000 ++/* The gadget's draw before configuration; more means a configured session. */ ++#define TRISTAR_ACCESSORY_UNCONFIGURED_MA 100 ++ ++/* ++ * The SystemInfo values iOS sends: the root model number and its OS build. ++ * The adapter does not answer them. Without a model number in the device ++ * tree, send the values this exchange was validated with. ++ */ ++#define TRISTAR_ACCESSORY_MODEL "MP2G2" ++#define TRISTAR_ACCESSORY_OSVERSION "20H350" ++ ++enum tristar_auto_phase { ++ TRISTAR_AUTO_EMPTY, /* nothing attached */ ++ TRISTAR_AUTO_DEVICE, /* attached, device role, GetID pending */ ++ TRISTAR_AUTO_OTHER, /* identified, not a host-mode accessory */ ++ TRISTAR_AUTO_ENTER, /* switch to host requested */ ++ TRISTAR_AUTO_HANDSHAKE, /* host is on; wait for VBUS, then 0x72/0x70 */ ++ TRISTAR_AUTO_HOST, /* host role, owned by this policy */ ++ TRISTAR_AUTO_INHIBIT, /* role taken over or entry failed; wait for unplug */ ++}; ++ ++enum tristar_auto_action { ++ TRISTAR_AUTO_NONE, ++ TRISTAR_AUTO_PROBE, /* GetID */ ++ TRISTAR_AUTO_POWER_HANDSHAKE, ++ TRISTAR_AUTO_TO_HOST, ++ TRISTAR_AUTO_TO_DEVICE, ++}; ++ ++struct tristar_auto_cfg { ++ unsigned int probe_ms; /* GetID period, no active USB link */ ++ unsigned int busy_probe_ms; /* ... while a USB host uses the gadget */ ++ unsigned int verify_ms; /* host role: re-check the ID */ ++ unsigned int settle_ms; /* after a plug edge */ ++ unsigned int detach_polls; /* consecutive empty 0x0c reads */ ++ unsigned int owned_detach_polls; /* ... adapter ours, bit 7 clear but VBUS bits set */ ++ unsigned int owned_replug_polls; /* bit 7 clear this long, then back: a replug */ ++ unsigned int handshake_tries; ++ unsigned int handshake_wait_ms; /* host-on deadline for strict VBUS */ ++ int input_ua; /* host role, after a good power handshake */ ++ int floor_ua; /* host role, no power handshake */ ++}; ++ ++static const struct tristar_auto_cfg tristar_auto_cfg = { ++ .probe_ms = 2000, ++ .busy_probe_ms = 10000, ++ .verify_ms = 5000, ++ .settle_ms = 300, ++ .detach_polls = 3, ++ .owned_detach_polls = 8, ++ .owned_replug_polls = 2, ++ .handshake_tries = 2, ++ .handshake_wait_ms = 15000, ++ /* iOS's input limit with the adapter after its power handshake */ ++ .input_ua = 2100000, ++ .floor_ua = 500000, ++}; ++ ++struct tristar_auto { ++ enum tristar_auto_phase phase; ++ int status; /* last good 0x0c, or -1 */ ++ unsigned int empty; /* consecutive empty 0x0c reads */ ++ bool was_empty; /* the previous read was empty */ ++ unsigned long next_probe; ++ u8 id[TRISTAR_IDBUS_ID_LEN]; ++ bool id_valid; ++ bool adapter; /* id selects host mode */ ++ unsigned int ph_tries; ++ unsigned long ph_deadline; ++ bool ph_ok; ++}; ++ ++struct tristar_auto_in { ++ unsigned long now; ++ int status; /* 0x0c or -errno */ ++ bool host; /* the role switch is in host mode */ ++ bool owned; /* ... on this policy's request */ ++ bool link_active; /* a USB host configured the gadget, not suspended */ ++}; ++ ++/** ++ * struct tristar_accessory - Lightning accessory policy ++ * @dev: the charger ++ * @lock: the charger's lock; serializes every Tristar access and the fields below ++ * @role_np: the role switch's node ++ * @role_sw: the Lightning port's role switch, once it is registered ++ * @work: the policy's poll ++ * @ib: ID-bus master ++ * @autop: policy state ++ * @stopped: no poll is queued and none may queue itself ++ * @status: last Tristar status read, or -errno ++ * @vbus: strict VBUS was seen since the last detach ++ * @detach_role: VBUS went away; give the role back at the next poll ++ * @host: the role switch was in host mode at the last poll ++ * @busy: an accessory-power or role change is in progress ++ * @usb_ma: what the USB host allows the gadget, as the charger last saw it ++ * @mux_fault: the charger could not verify a Tristar mux restore ++ * @id_ok: the last GetID succeeded, at @id_at (jiffies) ++ * @id_at: see @id_ok ++ * @source_id: the digital ID of the attached accessory ++ * @source_id_valid: @source_id holds one ++ * @info_valid: the post-ID exchange ran for this attachment ++ * @model: SystemInfo model number ++ * @model_len: its length ++ */ ++struct tristar_accessory { ++ struct device *dev; ++ struct mutex *lock; /* the charger's: every Tristar access */ ++ struct device_node *role_np; ++ struct usb_role_switch *role_sw; ++ struct delayed_work work; ++ struct tristar_idbus ib; ++ struct tristar_auto autop; ++ bool stopped; ++ int status; ++ bool vbus; ++ bool detach_role; ++ bool host; ++ bool busy; ++ unsigned int usb_ma; ++ bool mux_fault; ++ bool id_ok; ++ unsigned long id_at; ++ u8 source_id[TRISTAR_IDBUS_ID_LEN]; ++ bool source_id_valid; ++ bool info_valid; ++ u8 model[TRISTAR_IDBUS_INFO_MAX]; ++ unsigned int model_len; ++}; ++ ++/* ---- Decisions: no I/O here; the caller runs the action and reports back ---- */ ++ ++static bool tristar_auto_due(unsigned long now, unsigned long when) ++{ ++ return (long)(now - when) >= 0; ++} ++ ++/* The camera adapter, digital ID 11 f0 ..: USB=2 UART=1 PWR=0xf. */ ++static bool tristar_auto_is_host_adapter(const u8 *id) ++{ ++ return id[0] == 0x11 && id[1] == 0xf0; ++} ++ ++static bool tristar_auto_attached(int status) ++{ ++ return status >= 0 && (status & (TRISTAR_AUTO_PRESENT | TRISTAR_AUTO_VBUS)); ++} ++ ++/* ++ * The camera adapter belongs to us from the host request until it is gone. ++ * An unplug in host mode can read 0x0c = 0x25: bit 7 (plug present) clear, ++ * VBUS bit 0 still set. Only bit 7 says the plug is there. ++ */ ++static bool tristar_auto_owned(const struct tristar_auto *ta) ++{ ++ return ta->id_valid && ta->adapter && ++ (ta->phase == TRISTAR_AUTO_ENTER || ta->phase == TRISTAR_AUTO_HANDSHAKE || ++ ta->phase == TRISTAR_AUTO_HOST); ++} ++ ++/* ++ * iOS identifies the accessory again on every attach. Forget the adapter ++ * information whenever the policy enters a phase without an owned adapter. ++ */ ++static bool tristar_auto_reidentify(enum tristar_auto_phase old, ++ enum tristar_auto_phase now) ++{ ++ bool was = old == TRISTAR_AUTO_EMPTY || old == TRISTAR_AUTO_DEVICE; ++ bool is = now == TRISTAR_AUTO_EMPTY || now == TRISTAR_AUTO_DEVICE; ++ ++ return is && !was; ++} ++ ++/* Host entry can briefly remove strict VBUS while the same adapter is present. */ ++static bool tristar_auto_hold_source(const struct tristar_auto *ta, int status) ++{ ++ return status >= 0 && (status & TRISTAR_AUTO_PRESENT) && ta->id_valid && ta->adapter && ++ (ta->phase == TRISTAR_AUTO_ENTER || ta->phase == TRISTAR_AUTO_HANDSHAKE || ++ ta->phase == TRISTAR_AUTO_HOST); ++} ++ ++static const char *tristar_auto_phase_name(enum tristar_auto_phase phase) ++{ ++ static const char *const names[] = { ++ [TRISTAR_AUTO_EMPTY] = "empty", ++ [TRISTAR_AUTO_DEVICE] = "device", ++ [TRISTAR_AUTO_OTHER] = "other", ++ [TRISTAR_AUTO_ENTER] = "enter-host", ++ [TRISTAR_AUTO_HANDSHAKE] = "handshake", ++ [TRISTAR_AUTO_HOST] = "host", ++ [TRISTAR_AUTO_INHIBIT] = "inhibit", ++ }; ++ ++ return (unsigned int)phase < ARRAY_SIZE(names) ? names[phase] : "?"; ++} ++ ++static void tristar_auto_forget(struct tristar_auto *ta) ++{ ++ ta->id_valid = false; ++ ta->adapter = false; ++ ta->ph_ok = false; ++ ta->ph_tries = 0; ++} ++ ++static void tristar_auto_init(struct tristar_auto *ta) ++{ ++ *ta = (struct tristar_auto){ .phase = TRISTAR_AUTO_EMPTY, .status = -1 }; ++} ++ ++/* One poll. The caller then runs the returned action and reports back. */ ++static enum tristar_auto_action ++tristar_auto_step(struct tristar_auto *ta, const struct tristar_auto_cfg *cfg, ++ const struct tristar_auto_in *in) ++{ ++ const int plug_bits = TRISTAR_AUTO_PRESENT | TRISTAR_AUTO_ORIENTATION; ++ bool attached, edge, replug, owned; ++ unsigned int limit, was_empty; ++ int prev; ++ ++ /* An I2C error is no evidence either way. */ ++ if (in->status < 0) ++ return TRISTAR_AUTO_NONE; ++ ++ owned = tristar_auto_owned(ta); ++ attached = owned ? in->status >= 0 && (in->status & TRISTAR_AUTO_PRESENT) : ++ tristar_auto_attached(in->status); ++ was_empty = ta->empty; ++ prev = ta->status; ++ ta->status = in->status; ++ replug = attached && ta->was_empty; ++ ta->was_empty = !attached; ++ ta->empty = attached ? 0 : ta->empty + 1; ++ edge = replug || (prev >= 0 && attached && ((prev ^ in->status) & plug_bits)); ++ ++ /* A fully empty 0x0c is an unplug at once; bit 7 clear with VBUS lingering waits longer. */ ++ limit = tristar_auto_owned(ta) && tristar_auto_attached(in->status) ? ++ cfg->owned_detach_polls : cfg->detach_polls; ++ if (ta->empty >= limit && ta->phase != TRISTAR_AUTO_EMPTY) { ++ bool release = ta->adapter && in->host && in->owned; ++ ++ ta->phase = TRISTAR_AUTO_EMPTY; ++ tristar_auto_forget(ta); ++ return release ? TRISTAR_AUTO_TO_DEVICE : TRISTAR_AUTO_NONE; ++ } ++ ++ /* ++ * A replug in host mode (bit 7 clear for a second or more, then back) ++ * leaves the adapter unidentified. Release the role and attach it ++ * again, as iOS does on every attach. ++ */ ++ if (owned && attached && was_empty >= cfg->owned_replug_polls && ++ ta->phase != TRISTAR_AUTO_EMPTY) { ++ bool release = in->host && in->owned; ++ ++ ta->phase = TRISTAR_AUTO_EMPTY; ++ tristar_auto_forget(ta); ++ return release ? TRISTAR_AUTO_TO_DEVICE : TRISTAR_AUTO_NONE; ++ } ++ ++ switch (ta->phase) { ++ case TRISTAR_AUTO_EMPTY: ++ if (attached) { ++ ta->phase = TRISTAR_AUTO_DEVICE; ++ ta->next_probe = in->now + cfg->settle_ms; ++ } ++ return TRISTAR_AUTO_NONE; ++ case TRISTAR_AUTO_DEVICE: ++ case TRISTAR_AUTO_OTHER: ++ /* A host role someone else asked for is theirs to end. */ ++ if (!attached || in->host) ++ return TRISTAR_AUTO_NONE; ++ if (edge) { ++ ta->phase = TRISTAR_AUTO_DEVICE; ++ ta->next_probe = in->now + cfg->settle_ms; ++ } ++ if (!tristar_auto_due(in->now, ta->next_probe)) ++ return TRISTAR_AUTO_NONE; ++ ta->next_probe = in->now + (in->link_active ? cfg->busy_probe_ms : cfg->probe_ms); ++ return TRISTAR_AUTO_PROBE; ++ case TRISTAR_AUTO_HANDSHAKE: ++ if (!in->host || !in->owned) { ++ ta->phase = TRISTAR_AUTO_INHIBIT; ++ return TRISTAR_AUTO_NONE; ++ } ++ /* Without strict VBUS the handshake would fail: do not spend tries on it. */ ++ if ((in->status & TRISTAR_AUTO_VBUS) == TRISTAR_AUTO_VBUS) ++ return TRISTAR_AUTO_POWER_HANDSHAKE; ++ if (tristar_auto_due(in->now, ta->ph_deadline)) ++ ta->phase = TRISTAR_AUTO_HOST; ++ return TRISTAR_AUTO_NONE; ++ case TRISTAR_AUTO_ENTER: ++ if (in->host && in->owned) { ++ ta->phase = TRISTAR_AUTO_HANDSHAKE; ++ ta->ph_deadline = in->now + cfg->handshake_wait_ms; ++ return TRISTAR_AUTO_NONE; ++ } ++ return TRISTAR_AUTO_TO_HOST; ++ case TRISTAR_AUTO_HOST: ++ if (!in->host || !in->owned) { ++ ta->phase = TRISTAR_AUTO_INHIBIT; ++ return TRISTAR_AUTO_NONE; ++ } ++ if (!attached) ++ return TRISTAR_AUTO_NONE; ++ if (edge || tristar_auto_due(in->now, ta->next_probe)) { ++ ta->next_probe = in->now + cfg->verify_ms; ++ return TRISTAR_AUTO_PROBE; ++ } ++ return TRISTAR_AUTO_NONE; ++ case TRISTAR_AUTO_INHIBIT: ++ break; ++ } ++ return TRISTAR_AUTO_NONE; ++} ++ ++/* GetID finished: ret 0 with a valid id, or -errno. */ ++static enum tristar_auto_action ++tristar_auto_probed(struct tristar_auto *ta, const struct tristar_auto_cfg *cfg, ++ unsigned long now, int ret, const u8 *id) ++{ ++ bool adapter; ++ ++ if (ta->phase != TRISTAR_AUTO_DEVICE && ta->phase != TRISTAR_AUTO_OTHER && ++ ta->phase != TRISTAR_AUTO_HOST) ++ return TRISTAR_AUTO_NONE; ++ if (ret) ++ return TRISTAR_AUTO_NONE; ++ adapter = tristar_auto_is_host_adapter(id); ++ memcpy(ta->id, id, TRISTAR_IDBUS_ID_LEN); ++ ta->id_valid = true; ++ if (ta->phase == TRISTAR_AUTO_HOST) { ++ if (adapter) ++ return TRISTAR_AUTO_NONE; ++ /* A fast swap to another accessory, e.g. a computer's cable. */ ++ ta->adapter = false; ++ ta->ph_ok = false; ++ ta->phase = TRISTAR_AUTO_OTHER; ++ ta->next_probe = now + cfg->probe_ms; ++ return TRISTAR_AUTO_TO_DEVICE; ++ } ++ ta->adapter = adapter; ++ if (!adapter) { ++ ta->phase = TRISTAR_AUTO_OTHER; ++ return TRISTAR_AUTO_NONE; ++ } ++ ta->phase = TRISTAR_AUTO_ENTER; ++ ta->ph_tries = 0; ++ ta->ph_ok = false; ++ return TRISTAR_AUTO_TO_HOST; ++} ++ ++/* ++ * Power handshake finished. Host is already on: iOS enumerates the hub ++ * before its handshake completes. After the last try, keep the port in host ++ * mode at the floor input budget. ++ */ ++static void tristar_auto_handshake_done(struct tristar_auto *ta, ++ const struct tristar_auto_cfg *cfg, int ret) ++{ ++ if (ta->phase != TRISTAR_AUTO_HANDSHAKE) ++ return; ++ ta->ph_tries++; ++ ta->ph_ok = !ret; ++ if (ret && ta->ph_tries < cfg->handshake_tries) ++ return; ++ ta->phase = TRISTAR_AUTO_HOST; ++} ++ ++/* Host entry finished. No retry on failure: the next unplug starts over. */ ++static void tristar_auto_host_done(struct tristar_auto *ta, ++ const struct tristar_auto_cfg *cfg, ++ unsigned long now, int ret) ++{ ++ if (ta->phase != TRISTAR_AUTO_ENTER) ++ return; ++ if (ret) { ++ ta->phase = TRISTAR_AUTO_INHIBIT; ++ return; ++ } ++ ta->phase = TRISTAR_AUTO_HANDSHAKE; ++ ta->next_probe = now + cfg->verify_ms; ++ ta->ph_deadline = now + cfg->handshake_wait_ms; ++} ++ ++/* ++ * Input power while the adapter is ours. iOS: power handshake done, 2100 mA ++ * at 5 V in host mode. Returns false when the ordinary device-mode rules ++ * apply. ++ */ ++static bool tristar_auto_supply(const struct tristar_auto *ta, ++ const struct tristar_auto_cfg *cfg, int status, ++ int *current_ua) ++{ ++ bool online; ++ ++ if (!ta->id_valid || !ta->adapter) ++ return false; ++ if (ta->phase != TRISTAR_AUTO_HANDSHAKE && ta->phase != TRISTAR_AUTO_ENTER && ++ ta->phase != TRISTAR_AUTO_HOST) ++ return false; ++ online = status >= 0 && (status & TRISTAR_AUTO_VBUS) == TRISTAR_AUTO_VBUS; ++ *current_ua = !online ? 0 : ta->ph_ok ? cfg->input_ua : cfg->floor_ua; ++ return true; ++} ++ ++/* ---- Policy ---- */ ++ ++static unsigned long tristar_accessory_now(void) ++{ ++ return ktime_to_ms(ktime_get_boottime()); ++} ++ ++/* A new attachment, or a changed accessory: run the post-ID exchange again. Caller holds lock. */ ++static void tristar_accessory_new_source(struct tristar_accessory *acc) ++{ ++ acc->source_id_valid = false; ++ acc->info_valid = false; ++} ++ ++/* Caller holds lock. */ ++static void tristar_accessory_source_id(struct tristar_accessory *acc, const u8 *id) ++{ ++ if (acc->source_id_valid && memcmp(acc->source_id, id, sizeof(acc->source_id))) ++ tristar_accessory_new_source(acc); ++ memcpy(acc->source_id, id, sizeof(acc->source_id)); ++ acc->source_id_valid = true; ++} ++ ++/* Caller holds lock. */ ++static void tristar_accessory_phase(struct tristar_accessory *acc, enum tristar_auto_phase old) ++{ ++ struct tristar_auto *ta = &acc->autop; ++ ++ if (old != TRISTAR_AUTO_EMPTY && old != ta->phase && ++ (ta->phase == TRISTAR_AUTO_DEVICE || ta->phase == TRISTAR_AUTO_EMPTY)) { ++ tristar_accessory_new_source(acc); ++ tristar_auto_forget(ta); ++ } ++ if (tristar_auto_reidentify(old, ta->phase)) ++ acc->info_valid = false; ++ if (old != ta->phase) ++ dev_dbg(acc->dev, "accessory: %s -> %s\n", tristar_auto_phase_name(old), ++ tristar_auto_phase_name(ta->phase)); ++} ++ ++/* Hold the port for a role-switch call; see the top of this file. */ ++static void tristar_accessory_busy(struct tristar_accessory *acc, bool busy) ++{ ++ mutex_lock(acc->lock); ++ acc->busy = busy; ++ mutex_unlock(acc->lock); ++} ++ ++/* ++ * The role switch runs this with its lock held, in the device role only, so ++ * that no role change overlaps the read. -EUCLEAN keeps the role switch from ++ * entering host mode while the charger has a Tristar mux restore unverified. ++ */ ++static int tristar_accessory_sample(void *data) ++{ ++ struct tristar_accessory *acc = data; ++ int ret; ++ ++ mutex_lock(acc->lock); ++ acc->status = tristar_idbus_status(&acc->ib); ++ ret = acc->mux_fault ? -EUCLEAN : 0; ++ mutex_unlock(acc->lock); ++ return ret; ++} ++ ++static enum tristar_auto_action tristar_accessory_probe_id(struct tristar_accessory *acc) ++{ ++ struct tristar_auto *ta = &acc->autop; ++ u8 id[TRISTAR_IDBUS_ID_LEN] = {}; ++ enum tristar_auto_phase old; ++ enum tristar_auto_action action; ++ bool need_info = false, powered = false, same = false; ++ int ret; ++ ++ mutex_lock(acc->lock); ++ ret = tristar_idbus_identify(&acc->ib, id); ++ acc->id_ok = !ret; ++ if (!ret) { ++ acc->id_at = jiffies; ++ same = ta->id_valid && !memcmp(ta->id, id, sizeof(id)); ++ tristar_accessory_source_id(acc, id); ++ need_info = tristar_auto_is_host_adapter(id) && !acc->info_valid; ++ acc->busy = need_info; ++ } ++ mutex_unlock(acc->lock); ++ ++ /* iOS switches the accessory power on before the post-ID exchange. */ ++ if (need_info) { ++ ret = apple_s8000_usb_role_accessory_power(acc->role_sw, true); ++ powered = !ret; ++ if (!ret) { ++ mutex_lock(acc->lock); ++ ret = tristar_idbus_accessory_info(&acc->ib, acc->model, acc->model_len, ++ (const u8 *)TRISTAR_ACCESSORY_OSVERSION, ++ strlen(TRISTAR_ACCESSORY_OSVERSION)); ++ mutex_unlock(acc->lock); ++ } ++ if (ret && powered) ++ apple_s8000_usb_role_accessory_power(acc->role_sw, false); ++ } ++ ++ mutex_lock(acc->lock); ++ acc->busy = false; ++ if (need_info && !ret) ++ acc->info_valid = true; ++ old = ta->phase; ++ action = tristar_auto_probed(ta, &tristar_auto_cfg, tristar_accessory_now(), ret, id); ++ if (!ret && !same) ++ dev_dbg(acc->dev, "accessory: digital ID %*ph%s\n", (int)sizeof(id), id, ++ ta->adapter ? " (camera adapter)" : ""); ++ else if (ret) ++ dev_dbg(acc->dev, "accessory: identification failed (%d)\n", ret); ++ tristar_accessory_phase(acc, old); ++ mutex_unlock(acc->lock); ++ return action; ++} ++ ++static enum tristar_auto_action ++tristar_accessory_act(struct tristar_accessory *acc, enum tristar_auto_action action) ++{ ++ struct tristar_auto *ta = &acc->autop; ++ enum tristar_auto_phase old; ++ int ret = 0; ++ ++ switch (action) { ++ case TRISTAR_AUTO_PROBE: ++ return tristar_accessory_probe_id(acc); ++ case TRISTAR_AUTO_POWER_HANDSHAKE: ++ mutex_lock(acc->lock); ++ if (!acc->id_ok || time_after(jiffies, acc->id_at + ++ msecs_to_jiffies(TRISTAR_ACCESSORY_ID_WINDOW_MS))) ++ ret = -ENOLINK; ++ else ++ ret = tristar_idbus_power_handshake(&acc->ib); ++ old = ta->phase; ++ tristar_auto_handshake_done(ta, &tristar_auto_cfg, ret); ++ dev_dbg(acc->dev, "accessory: power handshake %s (%d), try %u\n", ++ ret ? "failed" : "done", ret, ta->ph_tries); ++ tristar_accessory_phase(acc, old); ++ mutex_unlock(acc->lock); ++ break; ++ case TRISTAR_AUTO_TO_HOST: ++ /* The policy is in enter-host: the charger leaves the port alone. */ ++ ret = apple_s8000_usb_role_claim(acc->role_sw, true); ++ if (!ret) ++ ret = usb_role_switch_set_role(acc->role_sw, USB_ROLE_HOST); ++ if (ret) ++ apple_s8000_usb_role_claim(acc->role_sw, false); ++ mutex_lock(acc->lock); ++ old = ta->phase; ++ tristar_auto_host_done(ta, &tristar_auto_cfg, tristar_accessory_now(), ret); ++ tristar_accessory_phase(acc, old); ++ mutex_unlock(acc->lock); ++ if (ret) ++ dev_warn(acc->dev, "camera adapter: USB host mode failed: %d\n", ret); ++ else ++ dev_dbg(acc->dev, "camera adapter: USB host mode\n"); ++ break; ++ case TRISTAR_AUTO_TO_DEVICE: ++ tristar_accessory_busy(acc, true); ++ if (apple_s8000_usb_role_accessory(acc->role_sw)) ++ ret = usb_role_switch_set_role(acc->role_sw, USB_ROLE_DEVICE); ++ tristar_accessory_busy(acc, false); ++ if (ret) ++ dev_warn(acc->dev, "accessory: USB device mode failed: %d\n", ret); ++ else ++ dev_dbg(acc->dev, "accessory: USB device mode\n"); ++ break; ++ case TRISTAR_AUTO_NONE: ++ break; ++ } ++ return TRISTAR_AUTO_NONE; ++} ++ ++static void tristar_accessory_step(struct tristar_accessory *acc) ++{ ++ struct tristar_auto *ta = &acc->autop; ++ struct tristar_auto_in in = {}; ++ enum tristar_auto_action action; ++ enum tristar_auto_phase old; ++ unsigned int i; ++ bool detach; ++ int ret; ++ ++ /* The charger does not wait for the role switch; the policy does. */ ++ if (!acc->role_sw) { ++ acc->role_sw = usb_role_switch_find_by_fwnode(of_fwnode_handle(acc->role_np)); ++ if (!acc->role_sw) ++ return; ++ } ++ ++ ret = apple_s8000_usb_role_monitor(acc->role_sw, tristar_accessory_sample, acc); ++ mutex_lock(acc->lock); ++ /* Not in the device role: the role switch did not sample. */ ++ if (ret && ret != -EUCLEAN) ++ acc->status = tristar_idbus_status(&acc->ib); ++ /* VBUS gone, other than during host entry: a detach, whatever the plug bit says. */ ++ if (acc->status >= 0 && acc->vbus && !(acc->status & TRISTAR_AUTO_VBUS) && ++ !tristar_auto_hold_source(ta, acc->status)) { ++ acc->vbus = false; ++ acc->detach_role = true; ++ tristar_accessory_new_source(acc); ++ tristar_auto_forget(ta); ++ ta->phase = TRISTAR_AUTO_EMPTY; ++ ta->was_empty = true; ++ ta->empty = 0; ++ } ++ if (acc->status >= 0 && (acc->status & TRISTAR_AUTO_VBUS) == TRISTAR_AUTO_VBUS) ++ acc->vbus = true; ++ detach = acc->detach_role; ++ acc->detach_role = false; ++ mutex_unlock(acc->lock); ++ if (detach) ++ tristar_accessory_act(acc, TRISTAR_AUTO_TO_DEVICE); ++ ++ in.host = usb_role_switch_get_role(acc->role_sw) == USB_ROLE_HOST; ++ in.owned = in.host && apple_s8000_usb_role_accessory(acc->role_sw); ++ mutex_lock(acc->lock); ++ acc->host = in.host; ++ in.link_active = acc->usb_ma > TRISTAR_ACCESSORY_UNCONFIGURED_MA; ++ in.now = tristar_accessory_now(); ++ in.status = acc->status; ++ old = ta->phase; ++ action = tristar_auto_step(ta, &tristar_auto_cfg, &in); ++ tristar_accessory_phase(acc, old); ++ mutex_unlock(acc->lock); ++ ++ /* A probe can lead to host entry; nothing chains further. */ ++ for (i = 0; i < 4 && action != TRISTAR_AUTO_NONE; i++) ++ action = tristar_accessory_act(acc, action); ++} ++ ++static void tristar_accessory_work(struct work_struct *work) ++{ ++ struct tristar_accessory *acc = container_of(to_delayed_work(work), ++ struct tristar_accessory, work); ++ ++ tristar_accessory_step(acc); ++ mutex_lock(acc->lock); ++ if (!acc->stopped) ++ mod_delayed_work(system_power_efficient_wq, &acc->work, ++ msecs_to_jiffies(TRISTAR_ACCESSORY_POLL_MS)); ++ mutex_unlock(acc->lock); ++} ++ ++/* ---- Interface to the charger ---- */ ++ ++/** ++ * tristar_accessory_holds - whether the policy holds the Lightning port ++ * @acc: accessory policy, or NULL ++ * @ma: what the attached adapter supplies while held, in mA ++ * ++ * While held, the charger must not sample the Tristar mux and takes @ma as ++ * the source's allowance. Caller holds the charger lock. ++ * ++ * Return: a description of the holder, or NULL when the charger may ++ * identify the source itself. ++ */ ++const char *tristar_accessory_holds(struct tristar_accessory *acc, unsigned int *ma) ++{ ++ int ua; ++ ++ if (!acc) ++ return NULL; ++ if (tristar_auto_supply(&acc->autop, &tristar_auto_cfg, acc->status, &ua)) { ++ *ma = ua / 1000; ++ return "Lightning camera adapter, USB host mode"; ++ } ++ *ma = 0; ++ if (acc->busy) ++ return "Lightning accessory, changing USB role"; ++ if (acc->host) ++ return "USB host mode"; ++ return NULL; ++} ++ ++/** ++ * tristar_accessory_report - what the charger knows, once per charger poll ++ * @acc: accessory policy, or NULL ++ * @usb_ma: what the USB host allows the gadget (vbus_draw), 0 without one ++ * @mux_fault: the charger could not verify a Tristar mux restore ++ * ++ * Caller holds the charger lock. ++ */ ++void tristar_accessory_report(struct tristar_accessory *acc, unsigned int usb_ma, ++ bool mux_fault) ++{ ++ if (!acc) ++ return; ++ acc->usb_ma = usb_ma; ++ acc->mux_fault = mux_fault; ++} ++ ++/** ++ * tristar_accessory_start - start or resume the policy ++ * @acc: accessory policy, or NULL ++ */ ++void tristar_accessory_start(struct tristar_accessory *acc) ++{ ++ if (!acc) ++ return; ++ mutex_lock(acc->lock); ++ acc->stopped = false; ++ mod_delayed_work(system_power_efficient_wq, &acc->work, 0); ++ mutex_unlock(acc->lock); ++} ++ ++/** ++ * tristar_accessory_stop - stop the policy ++ * @acc: accessory policy, or NULL ++ * ++ * Returns once no poll runs; a role change in progress has finished and ++ * none starts until tristar_accessory_start(). The port keeps its role. ++ * Caller must not hold the charger lock. ++ */ ++void tristar_accessory_stop(struct tristar_accessory *acc) ++{ ++ if (!acc) ++ return; ++ mutex_lock(acc->lock); ++ acc->stopped = true; ++ mutex_unlock(acc->lock); ++ cancel_delayed_work_sync(&acc->work); ++} ++ ++/* Unbind: give a host role the policy holds back to the USB device controller. */ ++static void tristar_accessory_release(void *data) ++{ ++ struct tristar_accessory *acc = data; ++ ++ tristar_accessory_stop(acc); ++ if (acc->role_sw) { ++ if (apple_s8000_usb_role_accessory(acc->role_sw)) ++ usb_role_switch_set_role(acc->role_sw, USB_ROLE_DEVICE); ++ usb_role_switch_put(acc->role_sw); ++ } ++ of_node_put(acc->role_np); ++} ++ ++/* SystemInfo model: the root model number, if the boot loader passed one. */ ++static void tristar_accessory_model(struct tristar_accessory *acc) ++{ ++ struct device_node *root = of_find_node_by_path("/"); ++ const u8 *prop = NULL; ++ int len = 0; ++ ++ if (root) ++ prop = of_get_property(root, "model-number", &len); ++ if (prop && len > 0) { ++ acc->model_len = min_t(unsigned int, len, sizeof(acc->model)); ++ memcpy(acc->model, prop, acc->model_len); ++ while (acc->model_len && !acc->model[acc->model_len - 1]) ++ acc->model_len--; ++ } ++ of_node_put(root); ++ if (!acc->model_len) { ++ acc->model_len = strlen(TRISTAR_ACCESSORY_MODEL); ++ memcpy(acc->model, TRISTAR_ACCESSORY_MODEL, acc->model_len); ++ } ++} ++ ++/** ++ * tristar_accessory_probe - set up the policy if the board has a role switch ++ * @dev: the charger ++ * @tristar: the Tristar ++ * @lock: the charger's lock, which serializes every Tristar access ++ * @accp: the policy, or NULL when the device tree names no role switch ++ * ++ * The policy does nothing until tristar_accessory_start(). It finds the role ++ * switch from its own poll, so that charging never waits for the role switch. ++ * ++ * Return: 0, or -errno. ++ */ ++int tristar_accessory_probe(struct device *dev, struct i2c_client *tristar, ++ struct mutex *lock, struct tristar_accessory **accp) ++{ ++ struct tristar_accessory *acc; ++ struct device_node *np; ++ bool available; ++ int ret; ++ ++ np = of_parse_phandle(dev->of_node, "apple,usb-role-switch", 0); ++ if (!np) ++ return 0; ++ available = of_device_is_available(np); ++ if (!available || !IS_REACHABLE(CONFIG_USB_ROLES_APPLE_S8000)) { ++ of_node_put(np); ++ if (available) ++ dev_warn(dev, "no Lightning role switch driver; accessories stay unidentified\n"); ++ return 0; ++ } ++ ++ acc = devm_kzalloc(dev, sizeof(*acc), GFP_KERNEL); ++ if (!acc) { ++ of_node_put(np); ++ return -ENOMEM; ++ } ++ acc->dev = dev; ++ acc->lock = lock; ++ acc->role_np = np; ++ acc->ib.client = tristar; ++ acc->status = -ENODATA; ++ acc->stopped = true; ++ tristar_auto_init(&acc->autop); ++ tristar_accessory_model(acc); ++ INIT_DELAYED_WORK(&acc->work, tristar_accessory_work); ++ ret = devm_add_action_or_reset(dev, tristar_accessory_release, acc); ++ if (ret) ++ return ret; ++ ++ *accp = acc; ++ dev_info(dev, "Lightning accessory identification and USB host mode enabled\n"); ++ return 0; ++} +diff --git a/drivers/power/supply/apple_tristar_accessory.h b/drivers/power/supply/apple_tristar_accessory.h +new file mode 100644 +index 000000000..8df74ed37 +--- /dev/null ++++ b/drivers/power/supply/apple_tristar_accessory.h +@@ -0,0 +1,55 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Lightning accessory policy of the Tristar charger: the interface between ++ * the charger and the policy. See apple_tristar_accessory.c. ++ * ++ * Copyright The Orchard Linux contributors ++ */ ++#ifndef _APPLE_TRISTAR_ACCESSORY_H ++#define _APPLE_TRISTAR_ACCESSORY_H ++ ++#include ++#include ++ ++struct device; ++struct i2c_client; ++struct mutex; ++struct tristar_accessory; ++ ++#if IS_ENABLED(CONFIG_APPLE_TRISTAR_ACCESSORY) ++int tristar_accessory_probe(struct device *dev, struct i2c_client *tristar, ++ struct mutex *lock, struct tristar_accessory **accp); ++void tristar_accessory_start(struct tristar_accessory *acc); ++void tristar_accessory_stop(struct tristar_accessory *acc); ++void tristar_accessory_report(struct tristar_accessory *acc, unsigned int usb_ma, ++ bool mux_fault); ++const char *tristar_accessory_holds(struct tristar_accessory *acc, unsigned int *ma); ++#else ++static inline int tristar_accessory_probe(struct device *dev, struct i2c_client *tristar, ++ struct mutex *lock, ++ struct tristar_accessory **accp) ++{ ++ return 0; ++} ++ ++static inline void tristar_accessory_start(struct tristar_accessory *acc) ++{ ++} ++ ++static inline void tristar_accessory_stop(struct tristar_accessory *acc) ++{ ++} ++ ++static inline void tristar_accessory_report(struct tristar_accessory *acc, ++ unsigned int usb_ma, bool mux_fault) ++{ ++} ++ ++static inline const char *tristar_accessory_holds(struct tristar_accessory *acc, ++ unsigned int *ma) ++{ ++ return NULL; ++} ++#endif ++ ++#endif /* _APPLE_TRISTAR_ACCESSORY_H */ +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index 2467752a1..f5c7a0dca 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -54,6 +54,8 @@ + #include + #include + ++#include "apple_tristar_accessory.h" ++ + /* PMIC registers, relative to their banks */ + #define TRISTAR_CHG_STATUS 0x0a /* status bank */ + #define TRISTAR_CHG_STATUS_ONLINE 0x42 +@@ -171,6 +173,10 @@ struct tristar_charger { + u64 step_ns; + u64 guard_ns; + unsigned int epoch_seen; ++ /* Lightning accessory policy, on boards with a role switch */ ++ struct tristar_accessory *acc; ++ /* USB data is on, but the device controller is held off for a USB host role. */ ++ bool usb_held; + }; + + static unsigned int tristar_code_ma(unsigned int code) +@@ -611,6 +617,7 @@ static int tristar_identify_usba(struct tristar_charger *chg, unsigned int orien + static int tristar_identify(struct tristar_charger *chg, struct tristar_ident *id) + { + unsigned int epoch = READ_ONCE(chg->epoch), orientation; ++ const char *holder; + bool usbc; + u8 route, state; + int ret; +@@ -618,6 +625,18 @@ static int tristar_identify(struct tristar_charger *chg, struct tristar_ident *i + id->ma = 0; + id->type = POWER_SUPPLY_USB_TYPE_UNKNOWN; + id->sampled = false; ++ holder = tristar_accessory_holds(chg->acc, &id->ma); ++ if (holder) { ++ /* ++ * The accessory policy holds the port: the mux is not ours to ++ * sample, and no cable is known to compare after a source event. ++ */ ++ id->type = POWER_SUPPLY_USB_TYPE_ACA; ++ id->usbc = false; ++ id->orientation = 0; ++ strscpy(id->what, holder, sizeof(id->what)); ++ return 0; ++ } + ret = cbtl_read(chg, CBTL_MUX_ROUTE, &route, 1); + if (!ret) + ret = cbtl_read(chg, CBTL_MUX_STATE, &state, 1); +@@ -660,13 +679,19 @@ static unsigned int tristar_usb_ma(struct tristar_charger *chg) + + /* + * Turn USB data on or hold it off. A failed hold counts as on: the mux is +- * then left alone. Caller holds lock. ++ * then left alone. While the accessory policy holds the port as a USB host, ++ * the device controller stays off even with USB data on; the next poll after ++ * the policy lets go turns it on. Stopping always turns it on. Caller holds ++ * lock. + */ + static void tristar_set_usb(struct tristar_charger *chg, bool on) + { ++ unsigned int ma; ++ bool held = on && !chg->stopping && tristar_accessory_holds(chg->acc, &ma); + int ret = usb_role_switch_set_role(chg->role_sw, +- on ? USB_ROLE_DEVICE : USB_ROLE_NONE); ++ on && !held ? USB_ROLE_DEVICE : USB_ROLE_NONE); + ++ chg->usb_held = held && !ret; + if (ret) + dev_err(chg->dev, "cannot turn USB data %s (%d)\n", on ? "on" : "off", ret); + chg->usb_on = on || ret; +@@ -699,6 +724,15 @@ static void tristar_update_target(struct tristar_charger *chg) + } + } + ++/* Turn on a device controller held off for a host role the policy gave back. Caller holds lock. */ ++static void tristar_unhold_usb(struct tristar_charger *chg) ++{ ++ unsigned int ma; ++ ++ if (chg->usb_held && !tristar_accessory_holds(chg->acc, &ma)) ++ tristar_set_usb(chg, true); ++} ++ + /* Caller holds lock. */ + static void tristar_new_attachment(struct tristar_charger *chg, u64 now) + { +@@ -925,6 +959,37 @@ static void tristar_no_source(struct tristar_charger *chg, unsigned int status, + tristar_set_usb(chg, true); + } + ++/* ++ * While the accessory policy holds the port (the camera adapter in USB host ++ * mode), the adapter is the source: follow what it supplies. Once the policy ++ * lets go, identify the source again, or turn on the device controller that ++ * tristar_set_usb() held off. Caller holds lock. ++ */ ++static void tristar_follow_accessory(struct tristar_charger *chg, u64 now) ++{ ++ bool charging = chg->phase == TRISTAR_RAMPING || chg->phase == TRISTAR_CHARGING; ++ const char *holder; ++ unsigned int ma; ++ ++ if (!IS_ENABLED(CONFIG_APPLE_TRISTAR_ACCESSORY)) ++ return; ++ holder = tristar_accessory_holds(chg->acc, &ma); ++ if (!holder) { ++ if (charging && chg->usb_type == POWER_SUPPLY_USB_TYPE_ACA) ++ tristar_new_attachment(chg, now); ++ else ++ tristar_unhold_usb(chg); ++ return; ++ } ++ if (!charging || (chg->usb_type == POWER_SUPPLY_USB_TYPE_ACA && chg->source_ma == ma)) ++ return; ++ strscpy(chg->source, holder, sizeof(chg->source)); ++ chg->source_ma = ma; ++ chg->usb_type = POWER_SUPPLY_USB_TYPE_ACA; ++ tristar_update_target(chg); ++ dev_dbg(chg->dev, "%s: %u mA\n", holder, ma); ++} ++ + static void tristar_poll(struct work_struct *work) + { + struct tristar_charger *chg = container_of(to_delayed_work(work), +@@ -975,6 +1040,8 @@ static void tristar_poll(struct work_struct *work) + /* USB data must not depend on charging working. */ + if (!chg->usb_on) + tristar_set_usb(chg, true); ++ else ++ tristar_unhold_usb(chg); + goto again; + } + if (READ_ONCE(chg->epoch) != chg->epoch_seen && chg->phase != TRISTAR_SETTLING) { +@@ -1002,6 +1069,7 @@ static void tristar_poll(struct work_struct *work) + chg->step_ns = 0; + } + } ++ tristar_follow_accessory(chg, now); + switch (chg->phase) { + case TRISTAR_SETTLING: + if ((s64)(now - chg->attach_ns) >= (s64)TRISTAR_LOOK_MS * NSEC_PER_MSEC) +@@ -1017,6 +1085,7 @@ static void tristar_poll(struct work_struct *work) + break; + } + again: ++ tristar_accessory_report(chg->acc, chg->usb_ma, chg->phase == TRISTAR_FAULT); + mod_delayed_work(chg->wq, &chg->poll_work, + msecs_to_jiffies(TRISTAR_POLL_MS)); + if (chg->code != old_code || chg->online != old_online) +@@ -1212,7 +1281,9 @@ static const struct power_supply_desc tristar_desc = { + BIT(POWER_SUPPLY_USB_TYPE_DCP) | + BIT(POWER_SUPPLY_USB_TYPE_CDP) | + BIT(POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID) | +- BIT(POWER_SUPPLY_USB_TYPE_C), ++ BIT(POWER_SUPPLY_USB_TYPE_C) | ++ (IS_ENABLED(CONFIG_APPLE_TRISTAR_ACCESSORY) ? ++ BIT(POWER_SUPPLY_USB_TYPE_ACA) : 0), + .properties = tristar_properties, + .num_properties = ARRAY_SIZE(tristar_properties), + .get_property = tristar_get_property, +@@ -1260,6 +1331,7 @@ static void tristar_stop(struct tristar_charger *chg, const char *reason) + for (i = 0; i < chg->nirqs; i++) + disable_irq(chg->irqs[i]); + cancel_delayed_work_sync(&chg->poll_work); ++ tristar_accessory_stop(chg->acc); + mutex_lock(&chg->lock); + if (tristar_set_code(chg, TRISTAR_CHG_HANDOFF_CODE)) + dev_crit(chg->dev, "%s: handoff restore failed\n", reason); +@@ -1354,6 +1426,9 @@ static int tristar_probe(struct platform_device *pdev) + if (!chg->wq) + return -ENOMEM; + INIT_DELAYED_WORK(&chg->poll_work, tristar_poll); ++ ret = tristar_accessory_probe(dev, chg->tristar, &chg->lock, &chg->acc); ++ if (ret) ++ return ret; + + /* Never adopt a limit we cannot explain; anything higher drops to the fallback. */ + ret = regmap_read(chg->regmap, chg->control_base + TRISTAR_CHG_INPUT_LIMIT, &code); +@@ -1402,6 +1477,7 @@ static int tristar_probe(struct platform_device *pdev) + mutex_unlock(&chg->lock); + + mod_delayed_work(chg->wq, &chg->poll_work, 0); ++ tristar_accessory_start(chg->acc); + return 0; + } + +@@ -1446,11 +1522,13 @@ static int tristar_suspend(struct device *dev) + if (ret) + return ret; + cancel_delayed_work_sync(&chg->poll_work); ++ tristar_accessory_stop(chg->acc); + if (chg->plug_irq && device_may_wakeup(dev)) { + ret = enable_irq_wake(chg->plug_irq); + if (ret) + dev_warn(dev, "cannot wake on plug events: %d\n", ret); + } ++ + return 0; + } + +@@ -1464,6 +1542,7 @@ static int tristar_resume(struct device *dev) + chg->stopping = false; + mutex_unlock(&chg->lock); + mod_delayed_work(chg->wq, &chg->poll_work, 0); ++ tristar_accessory_start(chg->acc); + return 0; + } + +diff --git a/include/linux/usb/apple-s8000-role.h b/include/linux/usb/apple-s8000-role.h +new file mode 100644 +index 000000000..c240e217a +--- /dev/null ++++ b/include/linux/usb/apple-s8000-role.h +@@ -0,0 +1,48 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Apple S8000 (A9) Lightning port role switch: host role owned by an accessory. ++ * ++ * The Lightning accessory policy (apple_tristar_charger) claims the next host ++ * entry before it calls usb_role_switch_set_role(). A claimed host role has ++ * no revert timeout and uses the accessory power gates; it ends when the ++ * policy, or anyone else, sets the device role. ++ */ ++#ifndef __LINUX_USB_APPLE_S8000_ROLE_H ++#define __LINUX_USB_APPLE_S8000_ROLE_H ++ ++#include ++#include ++#include ++ ++struct usb_role_switch; ++ ++#if IS_REACHABLE(CONFIG_USB_ROLES_APPLE_S8000) ++int apple_s8000_usb_role_monitor(struct usb_role_switch *sw, ++ int (*sample)(void *data), void *data); ++int apple_s8000_usb_role_claim(struct usb_role_switch *sw, bool claim); ++int apple_s8000_usb_role_accessory_power(struct usb_role_switch *sw, bool on); ++bool apple_s8000_usb_role_accessory(struct usb_role_switch *sw); ++#else ++static inline int apple_s8000_usb_role_monitor(struct usb_role_switch *sw, ++ int (*sample)(void *data), void *data) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline int apple_s8000_usb_role_claim(struct usb_role_switch *sw, bool claim) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline int apple_s8000_usb_role_accessory_power(struct usb_role_switch *sw, bool on) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline bool apple_s8000_usb_role_accessory(struct usb_role_switch *sw) ++{ ++ return false; ++} ++#endif ++ ++#endif /* __LINUX_USB_APPLE_S8000_ROLE_H */ diff --git a/kernel/patches/drivers/power/0031-power-supply-apple_tristar_charger-release-later-USB.patch b/kernel/patches/drivers/power/0031-power-supply-apple_tristar_charger-release-later-USB.patch new file mode 100644 index 00000000..9053a692 --- /dev/null +++ b/kernel/patches/drivers/power/0031-power-supply-apple_tristar_charger-release-later-USB.patch @@ -0,0 +1,64 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:08:20 +0000 +Subject: [PATCH] power: supply: apple_tristar_charger: release later USB holds + +Track each successful data hold and release a transitional attachment +after the existing deadline. Keep fallback charging limits and let the +role switch preserve accessory ownership. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 24 +++++++++++++++++++- + 1 file changed, 23 insertions(+), 1 deletion(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index 2467752a1..86215a4c8 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -156,6 +156,7 @@ struct tristar_charger { + unsigned int source_ma; + /* USB data is on for this attachment: the mux must stay untouched. */ + bool usb_on; ++ u64 usb_off_ns; /* USB data last held off */ + bool source_seen; /* an attachment since probe */ + /* What the USB host allows (vbus_draw), valid for usb_epoch only. */ + unsigned int usb_ma; +@@ -670,8 +671,28 @@ static void tristar_set_usb(struct tristar_charger *chg, bool on) + if (ret) + dev_err(chg->dev, "cannot turn USB data %s (%d)\n", on ? "on" : "off", ret); + chg->usb_on = on || ret; +- if (!chg->usb_on) ++ if (!chg->usb_on) { + chg->usb_ma = 0; ++ chg->usb_off_ns = ktime_get_ns(); ++ } ++} ++ ++/* ++ * An attachment already seen can leave the PMIC transitional (e.g. 0x62). ++ * Main's no-source-since-probe fallback deliberately excludes that case. ++ * Do not hold USB data forever: release after the last hold's deadline, ++ * leaving the charging allowance at fallback. The role setter still honors ++ * an accessory's ownership of the port. Caller holds lock. ++ */ ++static void tristar_release_usb(struct tristar_charger *chg, u64 now, ++ unsigned int status) ++{ ++ if (chg->usb_on || ++ (s64)(now - chg->usb_off_ns) < (s64)TRISTAR_HOLD_MS * NSEC_PER_MSEC) ++ return; ++ dev_info(chg->dev, "PMIC status %#04x, no source identified: USB data on\n", ++ status); ++ tristar_set_usb(chg, true); + } + + /* The highest grant the source allows, up to the board maximum. */ +@@ -965,6 +986,7 @@ static void tristar_poll(struct work_struct *work) + if (chg->code > TRISTAR_CHG_FALLBACK_CODE) + tristar_withdraw(chg, TRISTAR_CHG_FALLBACK_CODE); + tristar_no_source(chg, status, now); ++ tristar_release_usb(chg, now, status); + goto again; + } + if (!chg->present) { diff --git a/kernel/patches/drivers/power/0032-power-supply-apple_tristar-restart-the-USB-hold-on-r.patch b/kernel/patches/drivers/power/0032-power-supply-apple_tristar-restart-the-USB-hold-on-r.patch new file mode 100644 index 00000000..a0d16a2b --- /dev/null +++ b/kernel/patches/drivers/power/0032-power-supply-apple_tristar-restart-the-USB-hold-on-r.patch @@ -0,0 +1,29 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:11:48 +0100 +Subject: [PATCH] power: supply: apple_tristar: restart the USB hold on + reconnect + +Refresh the USB hold timestamp on every empty-input observation. A cable +reinserted after a long unplug must finish identification before data is +enabled. Keep the bounded fallback for persistent transitional states. +This applies to every board using the charger, including iPad 6. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index 6a9842eba..6e8e8368a 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -1031,6 +1031,8 @@ static void tristar_poll(struct work_struct *work) + } + chg->online = status != 0; + if (!status) { ++ /* Identification starts after the empty period, not at unplug. */ ++ chg->usb_off_ns = now; + if (chg->present) { + tristar_set_usb(chg, false); + chg->present = false; diff --git a/kernel/patches/drivers/power/0033-power-supply-apple_tristar_charger-Run-the-poll-on-an-u.patch b/kernel/patches/drivers/power/0033-power-supply-apple_tristar_charger-Run-the-poll-on-an-u.patch new file mode 100644 index 00000000..f09d0a28 --- /dev/null +++ b/kernel/patches/drivers/power/0033-power-supply-apple_tristar_charger-Run-the-poll-on-an-u.patch @@ -0,0 +1,29 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 10:20:00 +0100 +Subject: [PATCH] power: supply: apple_tristar_charger: Run the poll on an + unbound queue + +The charger's own queue asked for neither WQ_PERCPU nor WQ_UNBOUND, so +every boot warned in alloc_workqueue(). The poll sleeps on I2C and the +HDQ gauge and keeps no per-CPU state: make the queue unbound. + +Assisted-by: LLM +--- + drivers/power/supply/apple_tristar_charger.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/drivers/power/supply/apple_tristar_charger.c b/drivers/power/supply/apple_tristar_charger.c +index 6e8e8368a..63168f8cb 100644 +--- a/drivers/power/supply/apple_tristar_charger.c ++++ b/drivers/power/supply/apple_tristar_charger.c +@@ -1446,7 +1446,8 @@ static int tristar_probe(struct platform_device *pdev) + if (ret) + return ret; + init_completion(&chg->adc_done); +- chg->wq = devm_alloc_workqueue(dev, "tristar-charger", WQ_HIGHPRI | WQ_FREEZABLE, 1); ++ chg->wq = devm_alloc_workqueue(dev, "tristar-charger", ++ WQ_UNBOUND | WQ_HIGHPRI | WQ_FREEZABLE, 1); + if (!chg->wq) + return -ENOMEM; + INIT_DELAYED_WORK(&chg->poll_work, tristar_poll); diff --git a/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch b/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch new file mode 100644 index 00000000..af7dc5ff --- /dev/null +++ b/kernel/patches/drivers/pstore/0001-dt-bindings-pstore-retain-the-tethered-warm-boot-sta.patch @@ -0,0 +1,68 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: pstore: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + .../devicetree/bindings/arm/apple,pstore.yaml | 31 +++++++++++++++++++ + .../bindings/reserved-memory/ramoops.yaml | 4 +++ + 2 files changed, 35 insertions(+) + create mode 100644 Documentation/devicetree/bindings/arm/apple,pstore.yaml + +diff --git a/Documentation/devicetree/bindings/arm/apple,pstore.yaml b/Documentation/devicetree/bindings/arm/apple,pstore.yaml +new file mode 100644 +index 000000000..241dee447 +--- /dev/null ++++ b/Documentation/devicetree/bindings/arm/apple,pstore.yaml +@@ -0,0 +1,31 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/arm/apple,pstore.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: Apple retained pstore allocation ++ ++maintainers: ++ - brkosebay ++ ++description: | ++ Select early allocation on the root node. The generated ramoops node ++ carries apple,preserve-in-capture so a capture kernel leaves evidence intact. ++ Its address is carried in the kexec FDT, never reallocated on capture boots. ++ ++select: ++ properties: ++ compatible: ++ contains: ++ enum: ++ - apple,s8000 ++ - apple,s8003 ++ required: ++ - compatible ++properties: ++ apple,reserve-pstore: ++ type: boolean ++ description: Reserve an aligned 2 MiB no-map ramoops area before paging. ++ ++additionalProperties: true +diff --git a/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml b/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml +index 45cc39ecc..0b349c0ba 100644 +--- a/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml ++++ b/Documentation/devicetree/bindings/reserved-memory/ramoops.yaml +@@ -30,6 +30,10 @@ allOf: + - $ref: reserved-memory.yaml + + properties: ++ apple,preserve-in-capture: ++ type: boolean ++ description: Preserve the previous kernel records by refusing capture-kernel probe. ++ + compatible: + const: ramoops + diff --git a/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..1bac3a8f --- /dev/null +++ b/kernel/patches/drivers/pstore/0002-pstore-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,180 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] pstore: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + arch/arm64/include/asm/mmu.h | 1 + + arch/arm64/kernel/setup.c | 1 + + arch/arm64/mm/init.c | 88 ++++++++++++++++++++++++++++++++++++ + fs/pstore/ram.c | 8 ++++ + 4 files changed, 98 insertions(+) + +diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h +index 5e1211c54..bd22a913d 100644 +--- a/arch/arm64/include/asm/mmu.h ++++ b/arch/arm64/include/asm/mmu.h +@@ -62,6 +62,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void) + + extern void arm64_memblock_init(void); + extern void paging_init(void); ++extern void apple_soc_publish_pstore(void); + extern void bootmem_init(void); + extern void create_mapping_noalloc(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot); +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index 29c6100f0..45c23e3d9 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -332,6 +332,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) + arm64_memblock_init(); + + paging_init(); ++ apple_soc_publish_pstore(); + + acpi_table_upgrade(); + +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index fbf215ecc..3e458f58e 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -22,6 +22,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +@@ -193,6 +195,91 @@ static int __init early_mem(char *p) + } + early_param("mem", early_mem); + ++static phys_addr_t apple_soc_pstore_fdt __initdata; ++ ++/* ++ * Reserve before paging/CMA/crashkernel, without touching the linear map: ++ * paging_init() has not built it yet. Prepare the replacement FDT through a ++ * temporary early mapping and publish its linear alias only after paging. ++ * Never choose a fresh address in kdump; reuse the first kernel's region. ++ */ ++static void __init apple_soc_reserve_pstore(void) ++{ ++ void *fdt = initial_boot_params, *copy; ++ phys_addr_t base, copy_pa; ++ fdt64_t reg[2]; ++ char name[40]; ++ int parent, node, size, ret; ++ ++ if (!IS_BUILTIN(CONFIG_PSTORE_RAM) || !fdt || ++ !fdt_getprop(fdt, 0, "apple,reserve-pstore", NULL)) ++ return; ++ if (fdt_node_offset_by_compatible(fdt, -1, "ramoops") >= 0) ++ return; ++ if (is_kdump_kernel()) { ++ pr_warn("Apple pstore: no inherited region in capture kernel\n"); ++ return; ++ } ++ parent = fdt_path_offset(fdt, "/reserved-memory"); ++ if (parent < 0 || fdt_address_cells(fdt, parent) != 2 || ++ fdt_size_cells(fdt, parent) != 2) ++ return; ++ ++ /* Existing firmware, kernel and initrd reservations are already known. */ ++ base = memblock_phys_alloc(SZ_2M, SZ_2M); ++ if (!base) ++ return; ++ size = fdt_totalsize(fdt) + SZ_4K; ++ copy_pa = memblock_phys_alloc(size, SMP_CACHE_BYTES); ++ if (!copy_pa) ++ goto free_region; ++ copy = early_memremap(copy_pa, size); ++ if (!copy) ++ goto free_copy; ++ ret = fdt_open_into(fdt, copy, size); ++ if (ret) ++ goto unmap_copy; ++ parent = fdt_path_offset(copy, "/reserved-memory"); ++ snprintf(name, sizeof(name), "ramoops@%llx", (u64)base); ++ node = fdt_add_subnode(copy, parent, name); ++ if (node < 0) ++ goto unmap_copy; ++ reg[0] = cpu_to_fdt64(base); ++ reg[1] = cpu_to_fdt64(SZ_2M); ++ if (fdt_setprop(copy, node, "compatible", "ramoops", sizeof("ramoops")) || ++ fdt_setprop(copy, node, "reg", reg, sizeof(reg)) || ++ fdt_setprop(copy, node, "no-map", NULL, 0) || ++ fdt_setprop_u32(copy, node, "record-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "console-size", SZ_1M) || ++ fdt_setprop_u32(copy, node, "max-reason", 2) || ++ fdt_setprop(copy, node, "apple,preserve-in-capture", NULL, 0) || ++ fdt_add_mem_rsv(copy, base, SZ_2M)) ++ goto unmap_copy; ++ /* No cached linear alias of ramoops' write-combined persistent mapping. */ ++ if (memblock_mark_nomap(base, SZ_2M)) ++ goto unmap_copy; ++ early_memunmap(copy, size); ++ apple_soc_pstore_fdt = copy_pa; ++ pr_info("Apple pstore: reserved 2 MiB at %pa, carried in kexec FDT\n", &base); ++ return; ++ ++unmap_copy: ++ early_memunmap(copy, size); ++free_copy: ++ memblock_phys_free(copy_pa, size); ++free_region: ++ memblock_phys_free(base, SZ_2M); ++ pr_warn("Apple pstore: reservation failed\n"); ++} ++ ++/* The linear map now exists; unflattening must see the prepared FDT. */ ++void __init apple_soc_publish_pstore(void) ++{ ++ if (apple_soc_pstore_fdt) ++ early_init_dt_verify(phys_to_virt(apple_soc_pstore_fdt), ++ apple_soc_pstore_fdt); ++} ++ + void __init arm64_memblock_init(void) + { + s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); +@@ -298,6 +385,7 @@ void __init arm64_memblock_init(void) + } + + early_init_fdt_scan_reserved_mem(); ++ apple_soc_reserve_pstore(); + } + + void __init bootmem_init(void) +diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c +index 2eb0d4fa2..b26a3a0d5 100644 +--- a/fs/pstore/ram.c ++++ b/fs/pstore/ram.c +@@ -9,6 +9,7 @@ + #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + + #include ++#include + #include + #include + #include +@@ -729,6 +730,13 @@ static int ramoops_probe(struct platform_device *pdev) + phys_addr_t paddr; + int err = -EINVAL; + ++ /* Leave the original panic/console records intact across phoenix. */ ++ if (is_kdump_kernel() && of_property_read_bool(pdev->dev.of_node, ++ "apple,preserve-in-capture")) { ++ pr_info("capture kernel: preserving Apple records for the full boot\n"); ++ return -ENODEV; ++ } ++ + /* + * Only a single ramoops area allowed at a time, so fail extra + * probes. diff --git a/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch b/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch new file mode 100644 index 00000000..e1ea9505 --- /dev/null +++ b/kernel/patches/drivers/rtc/0001-rtc-preserve-PMU-and-RTC-interrupt-ownership.patch @@ -0,0 +1,206 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] rtc: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + drivers/rtc/rtc-apple-pmic.c | 127 ++++++++++++++++++++++++++++++++++- + 1 file changed, 126 insertions(+), 1 deletion(-) + +diff --git a/drivers/rtc/rtc-apple-pmic.c b/drivers/rtc/rtc-apple-pmic.c +index ee56618c8..d31d4c716 100644 +--- a/drivers/rtc/rtc-apple-pmic.c ++++ b/drivers/rtc/rtc-apple-pmic.c +@@ -12,6 +12,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -24,22 +25,30 @@ + #include + + #define RTC_COUNTER_SIZE 6 ++#define RTC_ALARM_SIZE 4 ++#define RTC_CONTROL_RUN BIT(0) ++#define RTC_CONTROL_ALARM BIT(6) + + /** + * struct apple_pmic_rtc_regs - RTC Hardware registers + * @control: RTC Control + * @counter: Time Counter ++ * @alarm: Alarm seconds counter ++ * @has_alarm: Alarm registers are present + */ + + struct apple_pmic_rtc_regs { + u32 control; + u32 counter; ++ u32 alarm; ++ bool has_alarm; + }; + + /** + * struct apple_pmic_rtc - RTC driver internal structure + * @rtc: RTC device + * @regmap: regmap used to access registers ++ * @dev: device used for logging and wake setup + * @nvmem_cell: nvmem cell for offset + * @regs: hardware registers + * @base: RTC base address +@@ -183,17 +192,107 @@ static int apple_pmic_rtc_read_time(struct device *dev, struct rtc_time *tm) + return 0; + } + ++static int apple_pmic_rtc_alarm_irq_enable(struct device *dev, ++ unsigned int enabled) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ ++ /* AppleD2365PMU 0xfffffff006bbbc1c writes these exact values. */ ++ return regmap_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_RUN | ++ (enabled ? RTC_CONTROL_ALARM : 0)); ++} ++ ++static int apple_pmic_rtc_read_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ unsigned int control; ++ time64_t secs; ++ int rc; ++ ++ rc = regmap_bulk_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ rc = regmap_read(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, &control); ++ if (rc) ++ return rc; ++ ++ secs = (time64_t)get_unaligned_le32(buf) + ap_rtc->offset; ++ rtc_time64_to_tm(secs, &alarm->time); ++ alarm->enabled = !!(control & RTC_CONTROL_ALARM); ++ alarm->pending = 0; ++ return 0; ++} ++ ++static int apple_pmic_rtc_set_alarm(struct device *dev, ++ struct rtc_wkalrm *alarm) ++{ ++ struct apple_pmic_rtc *ap_rtc = dev_get_drvdata(dev); ++ u8 buf[RTC_ALARM_SIZE]; ++ time64_t secs; ++ int rc; ++ ++ if (!alarm->enabled) ++ return apple_pmic_rtc_alarm_irq_enable(dev, 0); ++ ++ secs = rtc_tm_to_time64(&alarm->time) - ap_rtc->offset; ++ if (secs <= 0 || secs > U32_MAX) ++ return -ERANGE; ++ put_unaligned_le32(secs, buf); ++ ++ /* iOS programs 0x508 first, then enables the one-shot at 0x500. */ ++ rc = regmap_bulk_write(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->alarm, ++ buf, sizeof(buf)); ++ if (rc) ++ return rc; ++ return apple_pmic_rtc_alarm_irq_enable(dev, 1); ++} ++ ++static irqreturn_t apple_pmic_rtc_alarm_irq(int irq, void *data) ++{ ++ struct apple_pmic_rtc *ap_rtc = data; ++ int rc; ++ ++ /* The PMU event read is the acknowledge; iOS then clears alarm bit 6. */ ++ rc = regmap_update_bits(ap_rtc->regmap, ++ ap_rtc->base + ap_rtc->regs->control, ++ RTC_CONTROL_ALARM, 0); ++ if (rc) ++ dev_warn_ratelimited(ap_rtc->dev, ++ "failed to disable fired alarm: %d\n", rc); ++ rtc_update_irq(ap_rtc->rtc, 1, RTC_IRQF | RTC_AF); ++ return IRQ_HANDLED; ++} ++ + static const struct rtc_class_ops apple_pmic_rtc_ops = { + .read_time = apple_pmic_rtc_read_time, + .set_time = apple_pmic_rtc_set_time, + }; + ++static const struct rtc_class_ops apple_pmic_rtc_alarm_ops = { ++ .read_time = apple_pmic_rtc_read_time, ++ .set_time = apple_pmic_rtc_set_time, ++ .read_alarm = apple_pmic_rtc_read_alarm, ++ .set_alarm = apple_pmic_rtc_set_alarm, ++ .alarm_irq_enable = apple_pmic_rtc_alarm_irq_enable, ++}; ++ + static const struct apple_pmic_rtc_regs amber_regs = { + .counter = 0x6 + }; + + static const struct apple_pmic_rtc_regs antigua_regs = { +- .counter = 0x2 ++ .control = 0x0, ++ .counter = 0x2, ++ .alarm = 0x8, ++ .has_alarm = true, + }; + + static const struct apple_pmic_rtc_regs calpe_regs = { +@@ -230,6 +329,7 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + struct device_node *node = pdev->dev.of_node; + const struct of_device_id *match; + struct apple_pmic_rtc *ap_rtc; ++ int irq; + int rc; + + match = of_match_node(apple_pmic_id_table, node); +@@ -264,6 +364,31 @@ static int apple_pmic_rtc_probe(struct platform_device *pdev) + ap_rtc->rtc->ops = &apple_pmic_rtc_ops; + ap_rtc->rtc->range_min = INT_MIN; + ap_rtc->rtc->range_max = INT_MAX; ++ if (ap_rtc->regs->has_alarm) { ++ irq = platform_get_irq_optional(pdev, 0); ++ if (irq == -EPROBE_DEFER) ++ return irq; ++ if (irq > 0) { ++ rc = devm_request_threaded_irq(&pdev->dev, irq, NULL, ++ apple_pmic_rtc_alarm_irq, ++ IRQF_ONESHOT, ++ "apple-pmic-rtc", ap_rtc); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot request alarm IRQ\n"); ++ device_init_wakeup(&pdev->dev, true); ++ rc = devm_pm_set_wake_irq(&pdev->dev, irq); ++ if (rc) ++ return dev_err_probe(&pdev->dev, rc, ++ "cannot set alarm wake IRQ\n"); ++ ap_rtc->rtc->ops = &apple_pmic_rtc_alarm_ops; ++ dev_info(&pdev->dev, ++ "D2365 alarm at 0x%03x on PMU event IRQ %d\n", ++ ap_rtc->base + ap_rtc->regs->alarm, irq); ++ } else if (irq != -ENXIO) { ++ return irq; ++ } ++ } + + return devm_rtc_register_device(ap_rtc->rtc); + } diff --git a/kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch b/kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch new file mode 100644 index 00000000..ad082c85 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0014-dt-bindings-soc-apple-select-certified-AOP-reload-th.patch @@ -0,0 +1,116 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: soc-apple: select certified AOP reload through + DT + +A retained AOP must not be reset as though it were a fresh boot. +Describe the compatible and retained-reload property. + +Assisted-by: LLM +--- + .../bindings/soc/apple/apple,t8010-aop.yaml | 58 +++++++++++++++++-- + 1 file changed, 54 insertions(+), 4 deletions(-) + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +index 14726afa0..df6621d74 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml +@@ -4,22 +4,28 @@ + $id: http://devicetree.org/schemas/soc/apple/apple,t8010-aop.yaml# + $schema: http://devicetree.org/meta-schemas/core.yaml# + +-title: Apple T8010 Always-On Processor ++title: Apple S8000 and T8010 Always-On Processor + + maintainers: + - Dalton Alexandre + + description: +- The Always-On Processor (AOP) of the A10 is an ARMv7 coprocessor that +- runs RTKit firmware and serves the motion sensors (accelerometer, ++ The Always-On Processor (AOP) of the A9 and A10 is an ARMv7 coprocessor ++ that runs RTKit firmware and serves the motion sensors (accelerometer, + gyroscope, compass, pressure) and the always-on microphones. It runs from + local SRAM, with a further firmware segment in system memory, and talks + to the application processor through an A7 IOP mailbox. Its firmware is + board-specific and is loaded by the operating system. + + properties: ++ apple,certified-reload: ++ type: boolean ++ description: Firmware supports the retained shutdown certificate for warm reload. ++ + compatible: +- const: apple,t8010-aop ++ enum: ++ - apple,s8000-aop ++ - apple,t8010-aop + + reg: + items: +@@ -50,6 +56,38 @@ properties: + The chip's unique ID (ECID), filled in by the bootloader. The + firmware's boot arguments carry it. + ++ apple,aop-target: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The AOP target number from the Apple device tree, filled in by the ++ bootloader. The S8000 firmware's boot arguments carry it. ++ ++ apple,chip-id: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip ID (e.g. 0x8003), filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ apple,chip-revision: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ The SoC's chip revision, filled in by the bootloader. The S8000 ++ firmware's boot arguments carry it. ++ ++ memory-region: ++ maxItems: 1 ++ description: Reserved 16 KiB shutdown transcript page for J71t warm reload. ++ ++ apple,observe-domains: ++ $ref: /schemas/types.yaml#/definitions/phandle-array ++ minItems: 5 ++ maxItems: 5 ++ description: AOP GPIO, AOP, bus interface, CPU and filter in lock order. ++ ++ apple,spi0-domain: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: Read-only SPI0 power-state provider observation. ++ + patternProperties: + "^(accel|gyro|compass|pressure)$": + type: object +@@ -58,6 +96,9 @@ patternProperties: + properties: + compatible: + enum: ++ - apple,s8000-aop-accel ++ - apple,s8000-aop-gyro ++ - apple,s8000-aop-pressure + - apple,t8010-aop-accel + - apple,t8010-aop-gyro + - apple,t8010-aop-compass +@@ -65,6 +106,15 @@ patternProperties: + + mount-matrix: + description: The sensor's mounting rotation in the device. ++ ++ patternProperties: ++ "^apple,(low-temp-accel-offset|(accel|gyro)-(sensitivity|interrupt)-calibration|(accel|gyro)-range-sensitivity-cal|gyro-trim-calibration|pressure-(offset-calibration|accel-sensitivity|heater-resistance))$": ++ $ref: /schemas/types.yaml#/definitions/uint8-array ++ maxItems: 64 ++ description: ++ The unit's factory calibration of the sensor, copied by the ++ bootloader from the property of the same name (without the ++ prefix) in the Apple device tree. The S8000 firmware takes it. + required: + - compatible + additionalProperties: false diff --git a/kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch b/kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch new file mode 100644 index 00000000..279d4de7 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0015-soc-apple-aop-select-certified-AOP-reload-through-DT.patch @@ -0,0 +1,3181 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] soc: apple: aop: select certified AOP reload through DT + +A retained AOP must not be reset as though it were a fresh boot. Select +retained reload through device-tree data, preserving ordinary +initialization for other hardware. + +Assisted-by: LLM +--- + MAINTAINERS | 2 + + drivers/soc/apple/Kconfig | 12 +- + drivers/soc/apple/aop-warm-record.h | 63 ++ + drivers/soc/apple/aop-warm-spi0.c.h | 1339 ++++++++++++++++++++++++ + drivers/soc/apple/aop-warm-spi0.h | 543 ++++++++++ + drivers/soc/apple/aop.c | 729 ++++++++++++- + drivers/soc/apple/rtkit.c | 50 + + include/linux/soc/apple/aop.h | 28 + + include/linux/soc/apple/pmgr-observe.h | 27 + + include/linux/soc/apple/rtkit.h | 2 + + 10 files changed, 2758 insertions(+), 37 deletions(-) + create mode 100644 drivers/soc/apple/aop-warm-record.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.c.h + create mode 100644 drivers/soc/apple/aop-warm-spi0.h + create mode 100644 include/linux/soc/apple/pmgr-observe.h + +diff --git a/MAINTAINERS b/MAINTAINERS +index a4a69643c..12c292825 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -2068,6 +2068,8 @@ APPLE T8010 ALWAYS-ON PROCESSOR + M: Dalton Alexandre + S: Maintained + F: Documentation/devicetree/bindings/soc/apple/apple,t8010-aop.yaml ++F: drivers/iio/gyro/apple_aop_gyro.c ++F: drivers/iio/pressure/apple_aop_pressure.c + F: drivers/soc/apple/aop.c + F: include/linux/soc/apple/aop.h + +diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig +index e68a49f1b..c0af67b62 100644 +--- a/drivers/soc/apple/Kconfig ++++ b/drivers/soc/apple/Kconfig +@@ -39,16 +39,16 @@ config APPLE_RTKIT + Say 'y' here if you have an Apple SoC. + + config APPLE_AOP_T8010 +- tristate "Apple T8010 Always-On Processor" ++ tristate "Apple S8000 and T8010 Always-On Processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help +- The Always-On Processor (AOP) of the Apple A10 runs the firmware for +- the motion sensors and the always-on microphones. This driver loads +- and starts that firmware and provides its services to the sensor +- drivers. ++ The Always-On Processor (AOP) of the Apple A9 and A10 runs the ++ firmware for the motion sensors and the always-on microphones. This ++ driver loads and starts that firmware and provides its services to ++ the sensor drivers. + +- Say 'y' or 'm' here if you have an A10-based iPad or iPhone. ++ Say 'y' or 'm' here if you have an A9- or A10-based iPad or iPhone. + + config APPLE_PMP_T8015_V1 + tristate "Apple T8015 PMP" +diff --git a/drivers/soc/apple/aop-warm-record.h b/drivers/soc/apple/aop-warm-record.h +new file mode 100644 +index 000000000..237d78d17 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-record.h +@@ -0,0 +1,63 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#include ++#include ++#define A9_HO_PAGE 0x4000U ++#define A9_HO_LOG_OFFSET 0x800U ++#define A9_SL_MAGIC 0x4c533941U /* "A9SL" */ ++#define A9_SL_VERSION 1U ++ ++enum a9_ho_step { ++ A9_HO_STEP_BEGIN, ++ A9_HO_STEP_RATES, ++ A9_HO_STEP_AFK_SENT, ++ A9_HO_STEP_AFK_ACKED, ++ A9_HO_STEP_DISABLED, ++ A9_HO_STEP_AP_SENT, ++ A9_HO_STEP_AP_ACKED, ++ A9_HO_STEP_IOP_SENT, ++ A9_HO_STEP_IOP_ACKED, ++ A9_HO_STEP_RAM_STABLE, ++ A9_HO_STEP_MAILBOX_IDLE, ++ A9_HO_STEP_HALTED, ++ A9_HO_STEP_DRAINED, ++ A9_HO_STEPS, ++}; ++ ++static const char * const a9_ho_step_names[A9_HO_STEPS] = { ++ "begin", "rates-zero", "afk-sent", "afk-acked", "endpoint-disable", ++ "ap-sent", "ap-ack", "iop-sent", "iop-ack", "ram-stable", "mailbox-idle", ++ "halt", "drain", ++}; ++ ++struct a9_sl_header { ++ u32 magic, version, capacity, length; ++ u32 lines, truncated, checksum, mode; ++ u64 begin_ns; ++ u8 writer[16]; ++ char entry[64]; ++}; ++ ++_Static_assert(sizeof(struct a9_sl_header) == 120, "shutdown log ABI"); ++ ++#define A9_SL_CAPACITY (A9_HO_PAGE - A9_HO_LOG_OFFSET - sizeof(struct a9_sl_header)) ++ ++/* FNV-1a; incremental over appended text. */ ++static inline u32 a9_sl_hash(u32 hash, const char *text, size_t length) ++{ ++ size_t i; ++ ++ for (i = 0; i < length; i++) ++ hash = (hash ^ (u8)text[i]) * 16777619U; ++ return hash; ++} ++ ++#define A9_SL_HASH_SEED 2166136261U ++ ++/* A replayable previous transcript: complete header, bounded, intact. */ ++static inline bool a9_sl_valid(const struct a9_sl_header *h, const char *text) ++{ ++ return h->magic == A9_SL_MAGIC && h->version == A9_SL_VERSION && ++ h->capacity == A9_SL_CAPACITY && h->length <= h->capacity && ++ !h->entry[sizeof(h->entry) - 1] && ++ a9_sl_hash(A9_SL_HASH_SEED, text, h->length) == h->checksum; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.c.h b/drivers/soc/apple/aop-warm-spi0.c.h +new file mode 100644 +index 000000000..3c421d207 +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.c.h +@@ -0,0 +1,1339 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Ported from a9/aop-release: wake16, release6-d254a9f790. */ ++static void a9_aop_spi0_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_REGS; i++) ++ values[i] = readl(base + a9_spi0_offsets[i]); ++} ++ ++/* ++ * AOPSPIMaster_reset (0102f2f4) under the AOP-domain PMGR guard with IRQs ++ * off and the AOP CPU halted. Its provider power transition 3 is an ++ * AOP-local call with no AP register equivalent and is not replayed. A poll ++ * timeout still restores the saved control word. ++ */ ++static void a9_aop_spi0_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, s->before); ++ s->saved = s->before[A9_SPI0_I_CONTROL]; ++ s->asserted = a9_spi0_base_word(s->saved) | A9_SPI0_RESET; ++ writel(s->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ s->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(s->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(s->saved, control); ++ s->reset_done_ns = ktime_get_ns(); ++ a9_aop_spi0_snapshot(s->base, s->after); ++ if (!ret && (s->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ s->error = ret; ++} ++ ++/* ++ * Wake8 device evidence: SCM was 0 before/after the first controller reset, ++ * then became 3 at the channel-3 stop store. There is no firmware-derived ++ * SCM status write. Replay AOPSPIMaster_reset after the teardown so the ++ * controller reset, rather than an invented SCM write, is the one variable. ++ */ ++static void a9_aop_spi0_post_reset_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ void __iomem *control = s->base + A9_SPI0_CONTROL; ++ u32 value = 0; ++ int ret; ++ ++ a9_aop_spi0_snapshot(s->base, r->before); ++ r->saved = r->before[A9_SPI0_I_CONTROL]; ++ r->asserted = a9_spi0_base_word(r->saved) | A9_SPI0_RESET; ++ writel(r->asserted, control); ++ ret = readl_poll_timeout_atomic(control, value, value & A9_SPI0_RESET, ++ 1, A9_SPI0_POLL_US); ++ if (!ret) ++ ret = readl_poll_timeout_atomic(control, value, ++ !(value & A9_SPI0_ENABLE), 1, ++ A9_SPI0_POLL_US); ++ r->polled = value; ++ if (!ret) { ++ udelay(A9_SPI0_DELAY_US); ++ writel(a9_spi0_base_word(r->saved), control); ++ udelay(A9_SPI0_DELAY_US); ++ } ++ writel(r->saved, control); ++ a9_aop_spi0_snapshot(s->base, r->after); ++ if (!ret && (r->after[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)) ++ ret = -EIO; ++ r->error = ret; ++} ++ ++static void a9_aop_spi0_ch3_snapshot(void __iomem *base, u32 *values) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_CH3_REGS; i++) ++ values[i] = readl(base + a9_spi0_ch3_offsets[i]); ++} ++ ++static void a9_aop_spi0_trace_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int channel, i; ++ ++ s->trace.control = readl(s->base + A9_SPI0_CONTROL); ++ s->trace.clock = readl(s->base + A9_SPI0_CLOCK); ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->trace.scm[i] = readl(s->base + a9_spi0_scm_offsets[i]); ++ a9_aop_spi0_ch3_snapshot(s->base, s->trace.ch3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) ++ for (i = 0; i < A9_SPI0_CHANNEL_REGS; i++) ++ s->trace.channel[channel][i] = ++ readl(s->base + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel + ++ a9_spi0_channel_offsets[i]); ++} ++ ++/* Warm path only, CPU halted. Log outside the PMGR guard; no SCM writes. */ ++static int a9_aop_spi0_trace(struct apple_aop *aop, const char *stage) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *c = s->trace.ch3, *m = s->trace.scm; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_trace_read, s); ++ if (ret) { ++ s->intc.failed = "failed-spi0-trace-guard"; ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=%d: guard refused\n", stage, ret); ++ return ret; ++ } ++ dev_info(aop->dev, "SPI0 SCM stage=%s errno=0 ctl=%08x clk=%08x scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ stage, s->trace.control, s->trace.clock, m[0], m[1], m[2], m[3], ++ c[0], c[1], c[2], c[3], c[4], c[5], c[6]); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ v = s->trace.channel[channel]; ++ dev_info(aop->dev, "SPI0 SCM channel stage=%s ch=%02u ctl=%08x route=%08x rate=%08x cfg=%08x interval=%08x cursor=%08x desc=%08x pending0=%u pending1=%u\n", ++ stage, channel, v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ !!(c[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(c[A9_CH3_I_8D4] & BIT(2 * channel))); ++ } ++ return 0; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s); ++ ++/* ++ * Wake10 safety gate, strengthened in wake11 to retain every selected-channel ++ * teardown invariant. A fault observed by a copy sample remains terminal even ++ * if it is transient and the final read happens to clear. ++ */ ++static int a9_aop_spi0_gate(struct apple_aop *aop, const char *stage, ++ const char *check, bool include_copy) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *m, *c; ++ int ret; ++ ++ ret = a9_aop_spi0_trace(aop, stage); ++ if (ret) ++ return ret; ++ m = s->trace.scm; ++ c = s->trace.ch3; ++ if (m[0] || m[1]) ++ s->gate_failed = "failed-spi0-scm"; ++ else if (c[A9_CH3_I_8D0] || c[A9_CH3_I_8D4]) ++ s->gate_failed = "failed-spi0-fault"; ++ else if (include_copy && s->copy_faulted) ++ s->gate_failed = "failed-spi0-copy-sample"; ++ else { ++ s->gate_failed = a9_spi0_ch3_verify(c, s->teardown_stable & BIT(3)); ++ if (!s->gate_failed) ++ s->gate_failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = s->gate_failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 %s check errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x copy_faulted=%u result=%s\n", ++ check, ret, m[0], m[1], c[A9_CH3_I_8D0], c[A9_CH3_I_8D4], ++ s->copy_faulted, s->gate_failed ?: "pass"); ++ return ret; ++} ++ ++#define A9_SPI0_COPY_CHUNK 256U ++#define A9_SPI0_COPY_SAMPLE_US 5000U ++ ++struct a9_aop_spi0_copy_probe { ++ void __iomem *base; ++ u32 scm_lo, scm_hi, fault, fault_8d4; ++}; ++ ++static void a9_aop_spi0_copy_probe_read(void *data) ++{ ++ struct a9_aop_spi0_copy_probe *p = data; ++ ++ p->scm_lo = readl(p->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(p->base + A9_SPI0_SCM_HI); ++ p->fault = readl(p->base + A9_SPI0_FAULT); ++ p->fault_8d4 = readl(p->base + A9_SPI0_FAULT_8D4); ++} ++ ++/* ++ * Read-only samples spaced by at least 5 ms across both halves of the SRAM ++ * copy. Once a nonzero state is seen it remains a terminal fact, but the copy ++ * completes so later samples and the full after-copy channel snapshot expose ++ * whether the source is level, transient or moves between channels. ++ */ ++static int a9_aop_spi0_copy_sample(struct apple_aop *aop, const char *phase, ++ size_t offset, size_t size, u64 start_ns, ++ u64 *last_us, unsigned int *sample, ++ bool force) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_copy_probe p = { .base = s->base }; ++ u64 elapsed_us = div_u64(ktime_get_boottime_ns() - start_ns, ++ NSEC_PER_USEC); ++ bool first_fault, initial = !*sample; ++ int ret; ++ ++ if (!force && *sample && elapsed_us - *last_us < A9_SPI0_COPY_SAMPLE_US) ++ return 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_copy_probe_read, &p); ++ first_fault = !s->copy_faulted && ++ (p.scm_lo || p.scm_hi || p.fault || p.fault_8d4); ++ if (!ret && first_fault) ++ s->copy_faulted = true; ++ dev_info(aop->dev, "SPI0 COPY sample=%u phase=%s elapsed_us=%llu offset=0x%zx size=0x%zx scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ *sample, phase, elapsed_us, offset, size, p.scm_lo, p.scm_hi, ++ p.fault, p.fault_8d4, ret ? "guard" : s->copy_faulted ? "fault" : "pass"); ++ *last_us = elapsed_us; ++ (*sample)++; ++ if (ret) ++ return ret; ++ if (first_fault) { ++ ret = a9_aop_spi0_trace(aop, "copy-fault"); ++ if (ret) ++ return ret; ++ } ++ /* The offset-zero sample is the last no-write gate before the copy. */ ++ return !initial || !s->copy_faulted ? 0 : -EIO; ++} ++ ++static int a9_aop_copy_verify_wake10(struct apple_aop *aop, const u8 *src, ++ size_t size) ++{ ++ u32 chunk[A9_SPI0_COPY_CHUNK / 4]; ++ u64 start_ns = ktime_get_boottime_ns(), last_us = 0; ++ unsigned int sample = 0; ++ size_t done, n; ++ int ret; ++ ++ ret = a9_aop_spi0_copy_sample(aop, "write", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __iowrite32_copy(aop->sram + done, src + done, n / 4); ++ ret = a9_aop_spi0_copy_sample(aop, "write", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_copy_sample(aop, "verify", 0, size, start_ns, &last_us, &sample, true); ++ if (ret) ++ return ret; ++ for (done = 0; done < size; done += n) { ++ n = min_t(size_t, size - done, sizeof(chunk)); ++ __ioread32_copy(chunk, aop->sram + done, n / 4); ++ if (memcmp(chunk, src + done, n)) ++ return -EIO; ++ ret = a9_aop_spi0_copy_sample(aop, "verify", done + n, size, start_ns, ++ &last_us, &sample, done + n == size); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void __iomem *a9_aop_spi0_channel_base(struct a9_aop_spi0 *s, ++ unsigned int channel) ++{ ++ return s->base + A9_SPI0_CHANNEL_BASE + A9_SPI0_CHANNEL_STRIDE * channel; ++} ++ ++/* ++ * Wake15: which other channels were RUN with a counting interval when this ++ * stop was issued, and which of them advanced across the stop. Read only; it ++ * separates a stop that had a ticking peer from the stop of the last live ++ * channel in the log. It never gates anything in-kernel. ++ */ ++static void a9_aop_spi0_peers_begin(struct a9_aop_spi0 *s, unsigned int channel, ++ u32 *interval, u32 *run) ++{ ++ unsigned int i; ++ u32 ctl; ++ ++ *run = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ interval[i] = 0; ++ if (i == channel) ++ continue; ++ ctl = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ interval[i] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ if ((ctl & A9_SPI0_CH3_RUN) && interval[i]) ++ *run |= BIT(i); ++ } ++} ++ ++static void a9_aop_spi0_stop_timing(struct a9_aop_spi0 *s, ++ struct a9_aop_spi0_ch3 *c, ++ const u32 *interval) ++{ ++ unsigned int i; ++ ++ c->peers_ticking = 0; ++ for (i = 0; i < A9_SPI0_CHANNELS; i++) { ++ void __iomem *base = a9_aop_spi0_channel_base(s, i); ++ ++ if ((c->peers_run & BIT(i)) && ++ readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]) != ++ interval[i]) ++ c->peers_ticking |= BIT(i); ++ } ++ c->reset_to_stop_us = c->stop_ns > s->reset_done_ns ? ++ div_u64(c->stop_ns - s->reset_done_ns, NSEC_PER_USEC) : 0; ++} ++ ++/* ++ * Wake12/13 captured channel 7, and wake13 channel 3, stopped but never idle: ++ * for 20 complete slices the control/progress word, config, interval, cursor, ++ * SCM and both fault banks were unchanged. This predicate is intentionally ++ * narrower than "RUN clear and no progress": the channel must be one of the ++ * two device-observed periodic clients, must carry that client's route, rate ++ * and config before the stop, and every sample must be byte-identical with SCM ++ * 3/0 (every earlier non-idle stop, which followed a channel-3 stop that had ++ * latched it) or 0/0 (a first stop that latches nothing), no fault, no cursor ++ * movement and a retained nonzero interval. ++ * It only permits the callback masks, descriptor clear, config acknowledge and ++ * the established controller reset; every later SCM, fault and copy gate stays. ++ */ ++static const char *a9_aop_spi0_stable_why(struct a9_aop_spi0_ch3 *c, ++ const struct a9_aop_channel_diag_slice *diag, ++ unsigned int channel) ++{ ++ const struct a9_aop_ch3_slice *first; ++ unsigned int i; ++ u32 route; ++ ++ if (channel == A9_SPI0_PHOSPHORUS_CHANNEL) ++ route = A9_SPI0_PHOSPHORUS_ROUTE; ++ else if (channel == A9_SPI0_MPU_ASYNC_CHANNEL) ++ route = A9_SPI0_MPU_ASYNC_ROUTE; ++ else ++ return "channel"; ++ if (c->poll != -ETIMEDOUT || c->slices != A9_SPI0_CH3_SLICES) ++ return "no-timeout"; ++ if (!(c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) || ++ c->before[A9_CH3_I_ROUTE] != route || ++ c->before[A9_CH3_I_RATE] != A9_SPI0_PHOSPHORUS_RATE || ++ c->before[A9_CH3_I_C3C] != A9_SPI0_PHOSPHORUS_CFG || ++ c->before[A9_CH3_I_DESC] || (c->stop & A9_SPI0_CH3_RUN)) ++ return "before-identity"; ++ if (c->before[A9_CH3_I_8D4] || (c->before[A9_CH3_I_8D0] & ~BIT(2 * channel))) ++ return "before-fault"; ++ ++ first = &c->slice[0]; ++ if ((first->c3 & A9_SPI0_STABLE_CTL_MASK) != A9_SPI0_STABLE_CTL_STOPPED || ++ (first->c3 & (A9_SPI0_CH3_RUN | A9_SPI0_CH3_IDLE | A9_SPI0_CH3_IRQ1))) ++ return "stopped-control"; ++ if ((first->scm_lo != 0 && first->scm_lo != 3) || first->scm_hi) ++ return "scm"; ++ if (first->fault || diag[0].fault_8d4) ++ return "sample-fault"; ++ if (diag[0].cfg != A9_SPI0_PHOSPHORUS_CFG || diag[0].cursor || !diag[0].interval) ++ return "cfg-cursor-interval"; ++ ++ for (i = 1; i < A9_SPI0_CH3_SLICES; i++) { ++ const struct a9_aop_ch3_slice *p = &c->slice[i]; ++ ++ if (p->c3 != first->c3 || p->scm_lo != first->scm_lo || ++ p->scm_hi != first->scm_hi || p->fault != first->fault || ++ diag[i].cfg != diag[0].cfg || ++ diag[i].interval != diag[0].interval || ++ diag[i].cursor != diag[0].cursor || ++ diag[i].fault_8d4 != diag[0].fault_8d4) ++ return "progress"; ++ } ++ if (c->slice[A9_SPI0_CH3_SLICES - 1].ns < ++ (u64)A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US * NSEC_PER_USEC) ++ return "short"; ++ return NULL; ++} ++ ++/* ++ * AOPSPIMaster_teardownChannel (0102e4f8) for channel 3, then the fault ++ * handler's acknowledge (0102f20a), under the same guard as the reset with ++ * the CPU halted. Every store is a read-modify-write except the descriptor, ++ * which the firmware zeroes. The idle wait runs in 1 ms slices, up to twice ++ * the firmware's own 10 ms SCMto bound; each slice ends with read-only ++ * samples of C3, the SCM state and +0x8d0 (wake15: and config, interval, ++ * cursor and +0x8d4). An idle timeout stops before the masks unless the ++ * stable-stop predicate holds. The acknowledge bit self-clears; +0x8d0 bit 6 ++ * shows whether it took effect. ++ */ ++static void a9_aop_spi0_ch3_locked(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ void __iomem *c3 = s->base + A9_SPI0_CH3; ++ void __iomem *c3c = s->base + A9_SPI0_CH3_0C; ++ void __iomem *base = a9_aop_spi0_channel_base(s, 3); ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_ch3_snapshot(s->base, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(c3) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, c3); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, 3, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(c3, value, value & A9_SPI0_CH3_IDLE, ++ 1, A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->ch3_diag[c->slices]; ++ d->cfg = readl(c3c); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->ch3_diag, 3); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(3); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(c3) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, c3); ++ c->class1 = readl(c3) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, c3); ++ writel(0, s->base + A9_SPI0_CH3_DESC); ++ c->ack = readl(c3c) | A9_SPI0_CH3_ACK; ++ writel(c->ack, c3c); ++ } ++ a9_aop_spi0_ch3_snapshot(s->base, c->after); ++ c->failed = c->poll && !c->stable ? "failed-ch3-teardown-idle" : ++ a9_spi0_ch3_verify(c->after, c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++ ++static void a9_aop_spi0_channel_snapshot(struct a9_aop_spi0 *s, ++ unsigned int channel, u32 *values) ++{ ++ void __iomem *base = a9_aop_spi0_channel_base(s, channel); ++ ++ values[A9_CH3_I_C3] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]); ++ values[A9_CH3_I_ROUTE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_ROUTE]); ++ values[A9_CH3_I_RATE] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_RATE]); ++ values[A9_CH3_I_C3C] = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]); ++ values[A9_CH3_I_DESC] = readl(base + A9_SPI0_CHANNEL_DESC); ++ values[A9_CH3_I_8D0] = readl(s->base + A9_SPI0_FAULT); ++ values[A9_CH3_I_8D4] = readl(s->base + A9_SPI0_FAULT_8D4); ++} ++ ++static const char *a9_spi0_channel_state_verify(u32 control, u32 desc, ++ u32 fault, u32 fault_8d4, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ u32 pending = BIT(2 * channel); ++ ++ if (control & A9_SPI0_CH3_RUN) ++ return "failed-channel-teardown-run"; ++ if (control & A9_SPI0_CH3_IRQ0) ++ return "failed-channel-teardown-class0"; ++ if (control & A9_SPI0_CH3_IRQ1) ++ return "failed-channel-teardown-class1"; ++ if (!(control & A9_SPI0_CH3_IDLE) && !stable_stop) ++ return "failed-channel-teardown-idle"; ++ if (desc) ++ return "failed-channel-teardown-desc"; ++ if ((fault | fault_8d4) & pending) ++ return "failed-channel-teardown-pending"; ++ return NULL; ++} ++ ++static const char *a9_spi0_channel_verify(const u32 *after, ++ unsigned int channel, ++ bool stable_stop) ++{ ++ return a9_spi0_channel_state_verify(after[A9_CH3_I_C3], ++ after[A9_CH3_I_DESC], ++ after[A9_CH3_I_8D0], ++ after[A9_CH3_I_8D4], channel, ++ stable_stop); ++} ++ ++/* ++ * Wake11 applies the channel-3 firmware sequence unchanged to another slot. ++ * Wake12 only appends read-only scheduler samples to that established poll. ++ */ ++static void a9_aop_spi0_channel_locked(void *data) ++{ ++ struct a9_aop_spi0_channel_op *op = data; ++ struct a9_aop_spi0 *s = op->spi0; ++ struct a9_aop_spi0_ch3 *c = op->state; ++ void __iomem *base = a9_aop_spi0_channel_base(s, op->channel); ++ void __iomem *control = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CTL]; ++ void __iomem *config = base + a9_spi0_channel_offsets[A9_CHANNEL_I_CFG]; ++ struct a9_aop_ch3_slice *p; ++ struct a9_aop_channel_diag_slice *d; ++ u32 interval[A9_SPI0_CHANNELS]; ++ u32 value = 0; ++ u64 start; ++ ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->before); ++ c->run_before = c->before[A9_CH3_I_C3] & A9_SPI0_CH3_RUN; ++ c->stop = readl(control) & ~A9_SPI0_CH3_STOP; ++ c->stop_ns = ktime_get_ns(); ++ writel(c->stop, control); ++ start = ktime_get_ns(); ++ a9_aop_spi0_peers_begin(s, op->channel, interval, &c->peers_run); ++ c->poll = -ETIMEDOUT; ++ for (c->slices = 0; c->poll && c->slices < A9_SPI0_CH3_SLICES; c->slices++) { ++ p = &c->slice[c->slices]; ++ c->poll = readl_poll_timeout_atomic(control, value, ++ value & A9_SPI0_CH3_IDLE, 1, ++ A9_SPI0_CH3_SLICE_US); ++ p->ns = ktime_get_ns() - start; ++ p->c3 = value; ++ p->scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ p->scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ p->fault = readl(s->base + A9_SPI0_FAULT); ++ d = &s->channel_diag[c->slices]; ++ d->cfg = readl(config); ++ d->interval = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_INTERVAL]); ++ d->cursor = readl(base + a9_spi0_channel_offsets[A9_CHANNEL_I_CURSOR]); ++ d->fault_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ } ++ c->idle = value; ++ a9_aop_spi0_stop_timing(s, c, interval); ++ c->stable_why = a9_aop_spi0_stable_why(c, s->channel_diag, op->channel); ++ c->stable = !c->stable_why; ++ if (c->stable) ++ s->teardown_stable |= BIT(op->channel); ++ if (!c->poll || c->stable) { ++ /* A normal idle or the captured stopped state: mask callbacks. */ ++ c->idle_us = div_u64(c->slice[c->slices - 1].ns, NSEC_PER_USEC); ++ c->class0 = readl(control) & ~A9_SPI0_CH3_CLASS0; ++ writel(c->class0, control); ++ c->class1 = readl(control) & ~A9_SPI0_CH3_CLASS1; ++ writel(c->class1, control); ++ writel(0, base + A9_SPI0_CHANNEL_DESC); ++ c->ack = readl(config) | A9_SPI0_CH3_ACK; ++ writel(c->ack, config); ++ } ++ a9_aop_spi0_channel_snapshot(s, op->channel, c->after); ++ c->failed = c->poll && !c->stable ? "failed-channel-teardown-idle" : ++ a9_spi0_channel_verify(c->after, op->channel, ++ c->stable); ++ if (!c->failed) ++ c->error = 0; ++ else if (c->stable) ++ c->error = -EIO; ++ else ++ c->error = c->poll ?: -EIO; ++} ++ ++/* Called only if ps_spi0 is fully on; reads nothing. */ ++static void a9_aop_spi0_pmgr_on(void *data) ++{ ++ *(bool *)data = true; ++} ++ ++/* AP PMGR ps_spi0, read-only: never requested, resumed or written. ++ * 0 = on, -EHOSTDOWN = off (raw still filled), -ENODEV = no node. ++ */ ++static int a9_aop_spi0_pmgr(struct apple_aop *aop, u32 *raw) ++{ ++ struct apple_pmgr_observation state = {}; ++ bool on = false; ++ int ret; ++ ++ if (!aop->spi0_pmgr) ++ return -ENODEV; ++ ret = apple_pmgr_observe(&aop->spi0_pmgr, 1, &state, ++ a9_aop_spi0_pmgr_on, &on); ++ *raw = state.raw; ++ return ret ?: on ? 0 : -EIO; ++} ++ ++/* ++ * After a reset that cleared +0x8d0 bit 6, before the image copy: CPU halted, ++ * AP notifications masked. A refusal sets ch3.failed and nothing is loaded. ++ */ ++static int a9_aop_spi0_ch3_teardown(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *b = c->before, *a = c->after; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ unsigned int i; ++ bool source_progress = false; ++ int ret; ++ ++ aop->experiment_step = "spi0-ch3-teardown"; ++ dev_info(aop->dev, "SPI0 CH3TD begin pa=%llx cpu=halted: first channel-3 store follows\n", ++ A9_SPI0_PA + A9_SPI0_CH3); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, "spi0-ch3-teardown"); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_ch3_locked, s); ++ if (ret) { ++ c->failed = "failed-ch3-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH3TD errno=%d ran=0: guard refused, channel 3 not touched\n", ++ ret); ++ return ret; ++ } ++ c->ran = true; ++ dev_info(aop->dev, "SPI0 CH3TD before c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH3TD idle-sample slice=%u at_us=%llu c3=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ i, div_u64(p->ns, NSEC_PER_USEC), p->c3, p->scm_lo, ++ p->scm_hi, p->fault, !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->ch3_diag[i]; ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ i, p->c3, d->cfg, d->interval, d->cursor, p->scm_lo, ++ p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->ch3_diag[0].interval || ++ d->cursor != s->ch3_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH3TD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->ch3_diag[0].interval, ++ s->ch3_diag[c->slices - 1].interval, ++ s->ch3_diag[0].cursor, ++ s->ch3_diag[c->slices - 1].cursor, source_progress, ++ c->stable ? "stable-stop-reset-only" : "refused"); ++ /* beyond_fw_bound=1: idle, but later than the firmware would wait. */ ++ dev_info(aop->dev, "SPI0 CH3TD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ c->stop, c->idle, c->poll); ++ else if (c->stable) ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->poll, c->class0, c->class1, c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH3TD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 c3c=%08x\n", ++ c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH3TD after c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH3TD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ /* c3c_bit5 is the pulse's readback, logged only (0 at 19:50). */ ++ dev_info(aop->dev, "SPI0 CH3TD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x c3c_bit5=%u ch3_pending=%u result=%s\n", ++ c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), ++ a[A9_CH3_I_DESC], !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), c->failed ?: ++ c->stable ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static int a9_aop_spi0_channel_teardown(struct apple_aop *aop, ++ unsigned int channel) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->channel; ++ struct a9_aop_spi0_channel_op op = { ++ .spi0 = s, ++ .state = c, ++ .channel = channel, ++ }; ++ const u32 *b, *a; ++ const struct a9_aop_ch3_slice *p; ++ const struct a9_aop_channel_diag_slice *d; ++ char checkpoint[32]; ++ unsigned int i; ++ bool source_progress = false; ++ bool stable_stop; ++ int ret; ++ ++ memset(c, 0, sizeof(*c)); ++ memset(s->channel_diag, 0, sizeof(s->channel_diag)); ++ s->teardown_failed_channel = channel; ++ aop->experiment_step = "spi0-channel-teardown"; ++ snprintf(checkpoint, sizeof(checkpoint), "spi0-ch%u-teardown", channel); ++ dev_info(aop->dev, "SPI0 CH%uTD begin pa=%llx cpu=halted: first channel-%u store follows\n", ++ channel, A9_SPI0_PA + A9_SPI0_CHANNEL_BASE + ++ A9_SPI0_CHANNEL_STRIDE * channel, channel); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint_noflush(aop, checkpoint); ++ if (ret) { ++ c->failed = "failed-channel-teardown-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_channel_locked, &op); ++ if (ret) { ++ c->failed = "failed-channel-teardown-guard"; ++ dev_info(aop->dev, "SPI0 CH%uTD errno=%d ran=0: guard refused, channel %u not touched\n", ++ channel, ret, channel); ++ return ret; ++ } ++ c->ran = true; ++ stable_stop = s->teardown_stable & BIT(channel); ++ b = c->before; ++ a = c->after; ++ dev_info(aop->dev, "SPI0 CH%uTD before ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, b[0], b[1], b[2], b[3], b[4], b[5], b[6]); ++ for (i = 0; i < c->slices; i++) { ++ p = &c->slice[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD idle-sample slice=%u at_us=%llu ctl=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x idle=%u\n", ++ channel, i, div_u64(p->ns, NSEC_PER_USEC), p->c3, ++ p->scm_lo, p->scm_hi, p->fault, ++ !!(p->c3 & A9_SPI0_CH3_IDLE)); ++ if (c->poll) { ++ d = &s->channel_diag[i]; ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic slice=%u ctl=%08x cfg=%08x interval=%08x cursor=%08x scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x\n", ++ channel, i, p->c3, d->cfg, d->interval, d->cursor, ++ p->scm_lo, p->scm_hi, p->fault, d->fault_8d4); ++ if (i && (p->c3 != c->slice[0].c3 || ++ d->interval != s->channel_diag[0].interval || ++ d->cursor != s->channel_diag[0].cursor)) ++ source_progress = true; ++ } ++ } ++ if (c->poll && c->slices) ++ dev_info(aop->dev, "SPI0 CH%uTD diagnostic mode=scheduler-read-only samples=%u ctl_hi=%02x->%02x interval=%08x->%08x cursor=%08x->%08x source_progress=%u result=%s\n", ++ channel, c->slices, ++ c->slice[0].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ c->slice[c->slices - 1].c3 >> A9_SPI0_CHANNEL_PROGRESS_SHIFT, ++ s->channel_diag[0].interval, ++ s->channel_diag[c->slices - 1].interval, ++ s->channel_diag[0].cursor, ++ s->channel_diag[c->slices - 1].cursor, ++ source_progress, ++ stable_stop ? "stable-stop-reset-only" : "refused"); ++ dev_info(aop->dev, "SPI0 CH%uTD idle-poll run_before=%u slices=%u/%u idle_us=%llu bound_us=%u fw_bound_us=%u beyond_fw_bound=%u result=%s\n", ++ channel, c->run_before, c->slices, A9_SPI0_CH3_SLICES, c->idle_us, ++ A9_SPI0_CH3_SLICES * A9_SPI0_CH3_SLICE_US, A9_SPI0_CH3_IDLE_FW_US, ++ !c->poll && c->idle_us > A9_SPI0_CH3_IDLE_FW_US, ++ c->poll ? "timeout" : "idle"); ++ if (c->poll && !stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d masks=skipped: idle bit 1 never set, no further store\n", ++ channel, c->stop, c->idle, c->poll); ++ else if (stable_stop) ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=%d stable_stop=1 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->poll, c->class0, c->class1, ++ c->ack); ++ else ++ dev_info(aop->dev, "SPI0 CH%uTD writes stop=%08x idle_poll=%08x poll_errno=0 class0=%08x class1=%08x desc=00000000 cfg=%08x\n", ++ channel, c->stop, c->idle, c->class0, c->class1, c->ack); ++ dev_info(aop->dev, "SPI0 CH%uTD after ctl=%08x route=%08x rate=%08x cfg=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ channel, a[0], a[1], a[2], a[3], a[4], a[5], a[6]); ++ dev_info(aop->dev, "SPI0 CH%uTD timing reset_to_stop_us=%llu late=%u peers_run=%04x peers_ticking=%04x stable=%u checkpoint=noflush stable_why=%s\n", ++ channel, c->reset_to_stop_us, c->reset_to_stop_us > A9_SPI0_LATE_STOP_US, ++ c->peers_run, c->peers_ticking, c->stable, ++ c->stable ? "stable" : c->poll ? c->stable_why : "idle"); ++ dev_info(aop->dev, "SPI0 CH%uTD check errno=%d run=%u class0=%u class1=%u idle=%u desc=%08x cfg_bit5=%u pending0=%u pending1=%u result=%s\n", ++ channel, c->error, !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_RUN), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1), ++ !!(a[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE), a[A9_CH3_I_DESC], ++ !!(a[A9_CH3_I_C3C] & A9_SPI0_CH3_ACK), ++ !!(a[A9_CH3_I_8D0] & BIT(2 * channel)), ++ !!(a[A9_CH3_I_8D4] & BIT(2 * channel)), c->failed ?: ++ stable_stop ? "pass-stable-reset-only" : "pass"); ++ return c->error; ++} ++ ++static const char *a9_aop_spi0_selected_verify(struct a9_aop_spi0 *s) ++{ ++ const u32 *v; ++ const char *failed = NULL; ++ unsigned int channel; ++ ++ if (s->teardown_selected != s->teardown_done) ++ return "failed-channel-teardown-incomplete"; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ v = s->trace.channel[channel]; ++ failed = a9_spi0_channel_state_verify(v[A9_CHANNEL_I_CTL], ++ v[A9_CHANNEL_I_DESC], ++ s->trace.ch3[A9_CH3_I_8D0], ++ s->trace.ch3[A9_CH3_I_8D4], channel, ++ s->teardown_stable & BIT(channel)); ++ if (failed) { ++ s->teardown_failed_channel = channel; ++ return failed; ++ } ++ } ++ return NULL; ++} ++ ++/* ++ * Channel 3 remains mandatory and executes first. Wake11 additionally selects ++ * every snapshot slot with run bit 0 (RUN) set and a nonzero interval, which ++ * excludes configured-but-idle slots while selecting any periodic client still ++ * owned by the old firmware. Wake16 returns to wake11's order after wake15's ++ * channel-7-first stop failed: all ten device stops fit "channel 3 idles when ++ * stopped within ~2 ms of the reset (5/5; 0/2 at 11 and 33 ms) and channel 7 ++ * never idles (0/3, at 0.3 ms, 1.9 ms and 15 ms)". Channel 3 goes first and ++ * early; channel 7 is then the stopped-without-idle channel the stable-stop ++ * fingerprint was captured from (wake11/12). ++ */ ++static int a9_aop_spi0_teardown_running(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *v; ++ unsigned int channel; ++ int ret; ++ ++ s->teardown_selected = BIT(3); ++ s->teardown_done = 0; ++ s->teardown_stable = 0; ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3) ++ continue; ++ v = s->trace.channel[channel]; ++ if ((v[A9_CHANNEL_I_CTL] & A9_SPI0_CH3_RUN) && ++ v[A9_CHANNEL_I_INTERVAL]) ++ s->teardown_selected |= BIT(channel); ++ } ++ dev_info(aop->dev, "SPI0 teardown select stage=after-controller-reset selected=%04x mandatory=0008 rule=ch3-or-run-and-interval order=ch3-then-others\n", ++ s->teardown_selected); ++ ret = a9_aop_spi0_ch3_teardown(aop); ++ if (ret) { ++ s->teardown_failed_channel = 3; ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=3 errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, ret, ++ s->ch3.failed ?: "failed-ch3-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(3); ++ for (channel = 0; channel < A9_SPI0_CHANNELS; channel++) { ++ if (channel == 3 || !(s->teardown_selected & BIT(channel))) ++ continue; ++ ret = a9_aop_spi0_channel_teardown(aop, channel); ++ if (ret) { ++ dev_info(aop->dev, "SPI0 teardown refused selected=%04x done=%04x failed_channel=%u errno=%d result=%s\n", ++ s->teardown_selected, s->teardown_done, channel, ret, ++ s->channel.failed ?: "failed-channel-teardown"); ++ return ret; ++ } ++ s->teardown_done |= BIT(channel); ++ } ++ s->teardown_failed_channel = A9_SPI0_CHANNELS; ++ dev_info(aop->dev, "SPI0 teardown complete selected=%04x done=%04x stable=%04x result=pass\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable); ++ return 0; ++} ++ ++/* ++ * Second copy of the firmware controller reset, after the stop has completed. ++ * The preceding after-ch3-teardown trace is the read-only before snapshot; ++ * after-post-teardown-reset is the matching read-only after snapshot. Refuse ++ * the image copy unless SCM is zero and every channel quiet-state gate holds. ++ */ ++static int a9_aop_spi0_post_reset(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_post_reset *r = &s->post_reset; ++ const u32 *b = r->before, *a = r->after; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.ran || ++ s->ch3.error || s->teardown_selected != s->teardown_done || r->ran) { ++ r->failed = "failed-post-reset-state"; ++ return -EPERM; ++ } ++ aop->experiment_step = "spi0-post-teardown-reset"; ++ dev_info(aop->dev, "SPI0 POSTRST begin pa=%llx reason=ch3-stop-latched-scm cpu=halted\n", ++ A9_SPI0_PA); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-post-teardown-reset"); ++ if (ret) { ++ r->failed = "failed-post-reset-checkpoint"; ++ return ret; ++ } ++ } ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_post_reset_locked, s); ++ r->ran = !ret; ++ r->error = ret ?: r->error; ++ dev_info(aop->dev, "SPI0 POSTRST errno=%d ran=%d before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ r->error, r->ran, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 POSTRST after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], r->saved, r->asserted, ++ r->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ if (r->error) { ++ r->failed = "failed-post-reset-controller"; ++ return r->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-post-teardown-reset"); ++ if (ret) ++ return ret; ++ if (s->trace.scm[0] || s->trace.scm[1]) ++ r->failed = "failed-post-reset-scm"; ++ else { ++ r->failed = a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3)); ++ if (!r->failed) ++ r->failed = a9_aop_spi0_selected_verify(s); ++ } ++ ret = r->failed ? -EIO : 0; ++ dev_info(aop->dev, "SPI0 POSTRST check errno=%d scm_lo=%08x scm_hi=%08x ch3_pending=%u result=%s\n", ++ ret, s->trace.scm[0], s->trace.scm[1], ++ !!(s->trace.ch3[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING), ++ r->failed ?: "pass"); ++ dev_info(aop->dev, "SPI0 POSTRST channels selected=%04x done=%04x stable=%04x result=%s\n", ++ s->teardown_selected, s->teardown_done, s->teardown_stable, ++ r->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * The ten INTC words of a9_intc_offsets into state[@at], read only. The ++ * INTC is the AOP wrapper block (0xc1800000 in the firmware, space 2 as ++ * SPI0's verified 0xc1100000 = 0x210100000), which this driver reads ++ * unguarded for the mailbox and CPU control; no separate PMGR domain. ++ */ ++static void a9_aop_intc_snapshot(struct apple_aop *aop, unsigned int at) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ u32 *v = n->state[at]; ++ unsigned int i; ++ ++ for (i = 0; i < A9_INTC_REGS; i++) ++ v[i] = aop_rd(aop, a9_intc_offsets[i]); ++ n->states |= BIT(at); ++ dev_info(aop->dev, "AOP INTC state at=%s reset808=%08x current81c=%08x raise900=%08x ack980=%08x mask_a00=%08x mask_a04=%08x mask_a08=%08x mask_a80=%08x g7_c020=%08x g7_c024=%08x src_masked=%u view=%s\n", ++ a9_intc_at_name(at), v[0], v[1], v[2], v[3], v[4], v[5], v[6], ++ v[7], v[8], v[9], !!(v[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(v)); ++} ++ ++/* ++ * Round 4 (ds-aop-spi0-ch3-round4.md): the old session's SPI0 global IRQ ++ * 0x10014 stays latched in the AOP interrupt controller, and fresh firmware ++ * takes it as soon as SPI lifecycle event 1 enables it, before its own SPI0 ++ * reset (SCMErr:0x3,3). With the CPU halted and before the first SPI0 ++ * access, disable that source (group 1 disable bank, 0102c9ce). The bank ++ * reads back the mask, as the firmware's own reads of it show: bit 20 must ++ * read 1 after the store. ++ */ ++static int a9_aop_intc_disable(struct apple_aop *aop) ++{ ++ struct a9_aop_intc *n = &aop->spi0.intc; ++ const u32 *b = n->state[A9_INTC_AT_PRE_DISABLE]; ++ const u32 *a = n->state[A9_INTC_AT_POST_DISABLE]; ++ int ret; ++ ++ aop->experiment_step = "intc-disable"; ++ dev_info(aop->dev, "AOP INTC disable begin irq=0x10014 pa=%llx value=%08x cpu=halted: first INTC store follows\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "intc-disable"); ++ if (ret) { ++ n->failed = "failed-intc-disable-checkpoint"; ++ return ret; ++ } ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_DISABLE); ++ ret = aop_wr(aop, A9_INTC_DISABLE, A9_INTC_SPI0_SCM); ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_DISABLE); ++ if (ret) { ++ n->failed = "failed-intc-disable"; ++ } else if (!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM)) { ++ n->failed = "failed-intc-disable-readback"; ++ ret = -EIO; ++ } ++ n->disabled = !ret; ++ dev_info(aop->dev, "AOP INTC disable errno=%d before=%08x value=%08x after=%08x mask_a80=%08x src_masked=%u view=%s result=%s\n", ++ ret, b[A9_INTC_I_MASK0], A9_INTC_SPI0_SCM, a[A9_INTC_I_MASK0], ++ a[A9_INTC_I_ENABLE0], !!(a[A9_INTC_I_MASK0] & A9_INTC_SPI0_SCM), ++ a9_intc_view(a), n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Release policy for aop_spi0_reset=1. The reload is device-proven for exactly ++ * one retained state: the previous kernel quiesced and halted the firmware ++ * (wake16, 2026-10-29; decision prev-quiesce-halted). Auto-boot still set, a ++ * previous kernel that did not halt it, or an older kernel in the chain are ++ * different hardware states that no run has reloaded with the final teardown. ++ * Refuse them before the first halt or SPI0 access so the old firmware is ++ * left exactly as found; aop_spi0_untested=1 attempts them anyway. A skip ++ * (DFU cold, bad record, nothing ran) must also find the wrapper at its reset ++ * state: a record-less AOP that something already stopped is refused rather ++ * than cold-loaded. A stale record skips as before. ++ */ ++static int a9_aop_spi0_policy(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ enum a9_spi0_reason reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, ++ &aop->spi0.prev); ++ ++ if (a9_spi0_policy_allows(reason, outbox & AOP_OUTBOX_ENABLE, spi0_untested)) ++ return 0; ++ dev_info(aop->dev, "SPI0 policy decision=refuse reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d: only prev-quiesce-halted reloads and a skip needs the wrapper at its reset state block not touched\n", ++ a9_spi0_reason_name(reason), auto_boot, outbox, ++ a9_spi0_prev_name(aop->spi0.prev.state), aop->spi0.prev.hello, ++ aop->spi0.prev.halted); ++ return -EPERM; ++} ++ ++/* ++ * After the proven halt/drain, before any SRAM write. Only firmware that ran ++ * since the last SoC reset leaves SPI0 state behind: auto-boot 1, or the ++ * previous kernel's transcript bound by +0x4020 (a9_spi0_decide). Without ++ * that evidence the block is never touched. @auto_boot and @outbox are the ++ * boot snapshot from before the stop. Nonzero means: do not load or ++ * release the AOP. ++ */ ++static int a9_aop_spi0_prereset(struct apple_aop *aop, u32 auto_boot, u32 outbox) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ const u32 *b = s->before, *a = s->after; ++ const char *failed = NULL; ++ bool run; ++ int ret; ++ ++ s->auto_boot = auto_boot; ++ s->outbox = outbox; ++ s->reason = a9_spi0_decide(auto_boot, outbox & AOP_OUTBOX_ENABLE, &s->prev); ++ s->decided = true; ++ run = a9_spi0_reason_runs(s->reason); ++ dev_info(aop->dev, "SPI0 reset decision=%s reason=%s auto_boot=%u out4020=%08x prev=%s prev_hello=%d prev_halted=%d prev_steps=0x%x prev_carry=%d%s\n", ++ run ? "run" : "skip", a9_spi0_reason_name(s->reason), auto_boot, ++ outbox, a9_spi0_prev_name(s->prev.state), s->prev.hello, ++ s->prev.halted, s->prev.steps, s->prev.carried, ++ run ? "" : ": block not touched"); ++ if (!run) { ++ s->skipped = a9_spi0_reason_name(s->reason); ++ return 0; ++ } ++ if (!s->base || !aop->force_halted || !aop->force_drained || ++ !a9_aop_force_stopped(aop)) ++ return -EPERM; ++ ret = a9_aop_intc_disable(aop); ++ if (ret) ++ return ret; ++ aop->experiment_step = "spi0-reset"; ++ s->pmgr_before_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_before); ++ /* Reaches ramoops before the first AP access to the block. */ ++ dev_info(aop->dev, "SPI0 reset begin pa=%llx cpu=halted ps_spi0=%08x errno=%d: first AP access follows\n", ++ A9_SPI0_PA, s->pmgr_before, s->pmgr_before_errno); ++ if (cold_defer) { ++ ret = a9_aop_cold_checkpoint(aop, "spi0-reset"); ++ if (ret) ++ return ret; ++ } ++ ret = a9_aop_spi0_trace(aop, "before-controller-reset"); ++ if (ret) ++ return ret; ++ /* ++ * Wake15: from the controller reset to the last channel stop nothing may ++ * drain a console in this task. Holding the console lock makes every ++ * dev_info() below only append to the log buffer; the flushed spi0-reset ++ * checkpoint above already put the begin line into ramoops, and the lines ++ * are emitted by console_unlock() once the stops are stored. ++ */ ++ console_lock(); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_reset_locked, s); ++ s->ran = !ret; ++ s->error = ret ?: s->error; ++ s->pmgr_after_errno = a9_aop_spi0_pmgr(aop, &s->pmgr_after); ++ dev_info(aop->dev, "SPI0 reset errno=%d ran=%d pa=%llx before ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x\n", ++ s->error, s->ran, A9_SPI0_PA, b[0], b[1], b[2], b[3], b[4], b[5]); ++ dev_info(aop->dev, "SPI0 reset after ctl=%08x clk=%08x 8d0=%08x 8d4=%08x ch3=%08x ch3c=%08x saved=%08x asserted=%08x polled=%08x ch3_pending=%u->%u provider3=not-replayed\n", ++ a[0], a[1], a[2], a[3], a[4], a[5], s->saved, s->asserted, ++ s->polled, !!(b[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING), ++ !!(a[A9_SPI0_I_FAULT] & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 pmgr ps_spi0 before=%08x errno=%d after=%08x errno=%d (read-only, never powered)\n", ++ s->pmgr_before, s->pmgr_before_errno, s->pmgr_after, ++ s->pmgr_after_errno); ++ if (s->error) { ++ console_unlock(); ++ return s->error; ++ } ++ ret = a9_aop_spi0_trace(aop, "after-controller-reset"); ++ if (!ret) ++ ret = a9_aop_spi0_teardown_running(aop); ++ console_unlock(); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-ch3-teardown"); ++ if (!ret) ++ failed = a9_aop_spi0_selected_verify(s); ++ if (failed) { ++ s->channel.failed = failed; ++ ret = -EIO; ++ } ++ if (!ret) ++ ret = a9_aop_spi0_post_reset(aop); ++ return ret; ++} ++ ++/* Wake10 samples both SCM words and both fault classes, not only channel 3. */ ++static void a9_aop_spi0_fault_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ ++ s->prerun_scm_lo = readl(s->base + A9_SPI0_SCM_LO); ++ s->prerun_scm_hi = readl(s->base + A9_SPI0_SCM_HI); ++ s->prerun = readl(s->base + A9_SPI0_FAULT); ++ s->prerun_8d4 = readl(s->base + A9_SPI0_FAULT_8D4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ a9_aop_spi0_ch3_snapshot(s->base, s->ch3.reassert); ++} ++ ++/* ++ * Last AP look before RUN, CPU still halted, after the image copy: bit 6 ++ * must stay clear in every one of 31 guarded samples spaced >= 5 ms apart, ++ * spanning >= 150 ms. A reasserted channel-3 bit would crash the fresh ++ * image as before, so the first set sample refuses the release; nothing is ++ * retried. Any refusal sets ch3.failed. ++ */ ++static int a9_aop_spi0_recheck(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_spi0_ch3 *c = &s->ch3; ++ const u32 *r = c->reassert; ++ u64 start = ktime_get_boottime_ns(); ++ unsigned int i; ++ int ret = 0; ++ ++ for (i = 0; i < A9_SPI0_WINDOW_SAMPLES; i++) { ++ if (i) ++ usleep_range(A9_SPI0_WINDOW_US, A9_SPI0_WINDOW_US + 1000); ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_fault_read, s); ++ c->samples = i + 1; ++ c->elapsed_ms = div_u64(ktime_get_boottime_ns() - start, NSEC_PER_MSEC); ++ if (ret) { ++ c->failed = "failed-ch3-window-guard"; ++ break; ++ } ++ if (s->prerun_scm_lo || s->prerun_scm_hi || s->prerun || ++ s->prerun_8d4) { ++ s->reasserted = c->reasserted = true; ++ c->reassert_sample = i; ++ c->reassert_ms = c->elapsed_ms; ++ c->failed = "failed-spi0-window-reasserted"; ++ ret = -EIO; ++ dev_info(aop->dev, "SPI0 WINDOW reasserted sample=%u at_ms=%llu scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x cpu=halted\n", ++ i, c->reassert_ms, s->prerun_scm_lo, s->prerun_scm_hi, ++ s->prerun, s->prerun_8d4); ++ if (s->prerun & A9_SPI0_CH3_PENDING) ++ dev_info(aop->dev, "SPI0 WINDOW channel3 c3=%08x route=%08x rate=%08x c3c=%08x desc=%08x 8d0=%08x 8d4=%08x\n", ++ r[0], r[1], r[2], r[3], r[4], r[5], r[6]); ++ break; ++ } ++ } ++ /* usleep_range() cannot return early; this only trips on a bad clock. */ ++ if (!ret && c->elapsed_ms < A9_SPI0_WINDOW_MS) { ++ c->failed = "failed-ch3-window-short"; ++ ret = -EIO; ++ } ++ c->window_done = !ret; ++ dev_info(aop->dev, "SPI0 CH3TD window errno=%d samples=%u/%u elapsed_ms=%llu min_ms=%u result=%s\n", ++ ret, c->samples, A9_SPI0_WINDOW_SAMPLES, c->elapsed_ms, ++ A9_SPI0_WINDOW_MS, ret ? c->failed : "pass"); ++ dev_info(aop->dev, "SPI0 pre-run errno=%d 8d0=%08x ch3_pending=%u cpu=halted\n", ++ ret, s->prerun, !!(s->prerun & A9_SPI0_CH3_PENDING)); ++ dev_info(aop->dev, "SPI0 WINDOW state errno=%d scm_lo=%08x scm_hi=%08x 8d0=%08x 8d4=%08x result=%s\n", ++ ret, s->prerun_scm_lo, s->prerun_scm_hi, s->prerun, ++ s->prerun_8d4, ret ? c->failed : "pass"); ++ if (ret && c->reasserted) ++ a9_aop_spi0_trace(aop, "window-fault"); ++ return ret; ++} ++ ++/* SCM state and the channel-3 interval pair, read-only, into scm[scm_at]. */ ++static void a9_aop_spi0_scm_read(void *data) ++{ ++ struct a9_aop_spi0 *s = data; ++ unsigned int i; ++ ++ for (i = 0; i < A9_SPI0_SCM_REGS; i++) ++ s->intc.scm[s->intc.scm_at][i] = readl(s->base + a9_spi0_scm_offsets[i]); ++} ++ ++/* ++ * After the clean pre-RUN window, CPU still halted: the firmware's group-1 ++ * pre-handler acknowledge for 0x10014 (0102c6de..0102c6f2) discards the ++ * old-session latch now that its SPI0 cause is quiet. Fresh firmware ++ * re-enables the source in event 1. The INTC words, the SCM state and the ++ * C3 interval pair are logged around it and never gated: with 0x10014 ++ * masked, no readable word shows whether it is still pending. ++ */ ++static int a9_aop_intc_ack(struct apple_aop *aop) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS] = {}; ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ const u32 *p = n->scm[0], *q = n->scm[1]; ++ const u32 *b = n->state[A9_INTC_AT_PRE_ACK]; ++ const u32 *a = n->state[A9_INTC_AT_POST_ACK]; ++ int ret; ++ ++ aop->experiment_step = "intc-ack"; ++ n->scm_at = 0; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) { ++ n->failed = "failed-intc-ack-guard"; ++ dev_info(aop->dev, "AOP INTC pre-ack errno=%d: guard refused, no acknowledge\n", ++ ret); ++ return ret; ++ } ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_PRE_ACK); ++ dev_info(aop->dev, "AOP INTC pre-ack scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x cpu=halted\n", ++ p[0], p[1], p[2], p[3], b[A9_INTC_I_CURRENT]); ++ ret = aop_wr(aop, A9_INTC_ACK, A9_INTC_SPI0_SCM); ++ /* Also the posted-write barrier. */ ++ a9_aop_intc_snapshot(aop, A9_INTC_AT_POST_ACK); ++ n->acked = !ret; ++ dev_info(aop->dev, "AOP INTC ack errno=%d before=%08x value=%08x after=%08x\n", ++ ret, b[A9_INTC_I_ACK], A9_INTC_SPI0_SCM, a[A9_INTC_I_ACK]); ++ if (ret) { ++ n->failed = "failed-intc-ack"; ++ return ret; ++ } ++ n->scm_at = 1; ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, ++ a9_aop_spi0_scm_read, s); ++ if (ret) ++ n->failed = "failed-intc-post-guard"; ++ dev_info(aop->dev, "AOP INTC post-ack errno=%d scm_lo=%08x scm_hi=%08x c3_interval=%08x c3_cursor=%08x current=%08x result=%s\n", ++ ret, q[0], q[1], q[2], q[3], a[A9_INTC_I_CURRENT], n->failed ?: "pass"); ++ return ret; ++} ++ ++/* ++ * Wake7 still took 0x10014 before SPI event 5 despite the targeted ack. ++ * Replay INTC lifecycle event 3 (0102c592/598/59c/5a2): 1/read/0/read at ++ * +0x808. aop_wr includes a posted-write read after each store. This resets ++ * all AOP-private interrupt state; it does not write the SCM status or MPU. ++ * No assumption that SCM must become zero. Only the final reset readback ++ * and the established channel-3 quiet conditions gate RUN. ++ */ ++static int a9_aop_intc_reset(struct apple_aop *aop) ++{ ++ struct a9_aop_spi0 *s = &aop->spi0; ++ struct a9_aop_intc *n = &s->intc; ++ int ret; ++ ++ if (!aop->force_halted || !aop->force_drained || !s->ch3.window_done || ++ !n->acked || n->reset_done || ++ (aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) || readl(aop->control)) { ++ n->failed = "failed-intc-reset-state"; ++ return -EPERM; ++ } ++ ret = a9_aop_spi0_trace(aop, "pre-intc-reset"); ++ if (ret) ++ return ret; ++ aop->experiment_step = "intc-reset"; ++ dev_info(aop->dev, "AOP INTC reset begin pa=%llx sequence=1/read/0/read cpu=halted\n", ++ A9_SPI0_WRAPPER_PA + A9_INTC_RESET); ++ ret = aop_wr(aop, A9_INTC_RESET, 1); ++ n->reset_asserted = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-assert"; ++ return ret; ++ } ++ ret = aop_wr(aop, A9_INTC_RESET, 0); ++ n->reset_after = aop_rd(aop, A9_INTC_RESET); ++ if (ret) { ++ n->failed = "failed-intc-reset-deassert"; ++ return ret; ++ } ++ if (n->reset_after) { ++ n->failed = "failed-intc-reset-readback"; ++ ret = -EIO; ++ } else { ++ ret = a9_aop_spi0_trace(aop, "post-intc-reset"); ++ if (!ret && a9_spi0_ch3_verify(s->trace.ch3, ++ s->teardown_stable & BIT(3))) { ++ n->failed = "failed-intc-reset-ch3"; ++ ret = -EIO; ++ } else if (!ret && a9_aop_spi0_selected_verify(s)) { ++ n->failed = "failed-intc-reset-channel"; ++ ret = -EIO; ++ } ++ } ++ n->reset_done = !ret; ++ dev_info(aop->dev, "AOP INTC reset errno=%d asserted=%08x after=%08x result=%s\n", ++ ret, n->reset_asserted, n->reset_after, n->failed ?: "pass"); ++ return ret; ++} +diff --git a/drivers/soc/apple/aop-warm-spi0.h b/drivers/soc/apple/aop-warm-spi0.h +new file mode 100644 +index 000000000..2470165ce +--- /dev/null ++++ b/drivers/soc/apple/aop-warm-spi0.h +@@ -0,0 +1,543 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * J71t AOP reload/wake experiments, both opt-in and default off. Constants ++ * and evidence types only; the MMIO lives in apple-a9-aop-active.c. ++ * Evidence: docs/periph/ds-aop-reload-crash-and-wake.md. ++ * ++ * aop_spi0_reset=1: every halt + SRAM reload of a live AOP crashed in ++ * AOPSPIMaster_channelFaultIRQ (0102f252, DFAR 0x4c): the AOP-private SPI0 ++ * controller kept channel 3's +0x8d0 bit 6 while the fresh firmware's ++ * channel table was still empty. With the CPU halted, the AP replays the ++ * firmware's own controller reset AOPSPIMaster_reset (0102f2f4), then the ++ * firmware's own channel-3 teardown, and samples bit 6 for >= 150 ms after ++ * the image copy before it releases RUN once. Round 4: the old session's ++ * SPI0 global IRQ 0x10014 stays latched in the AOP interrupt controller ++ * (SCMErr:0x3,3 at the accel STARTEP), so the AP disables that source before ++ * the reset and acknowledges it after the clean window, just before RUN. ++ * Wake7: the channel-3 idle wait gets the firmware's own 10 ms bound twice ++ * over, sampled per 1 ms slice, and the disable is judged by the INTC mask ++ * readback, which the firmware itself reads. ++ * Wake8: after the targeted ack, replay INTC event 3 (+0x808: 1/read/0/read). ++ * This resets retained interrupt delivery, not the read-only SCM state. ++ * Wake9: wake8 device evidence showed the channel-3 stop itself changes SCM ++ * from 0 to 3. Replay the same SPI controller reset once more after teardown; ++ * require the post-reset SCM state to be zero and the teardown state to hold. ++ * Wake10: the second reset clears SCM, but a halted CPU then saw SCM 3 and ++ * +0x8d0 bit 14 (channel 7) during the firmware copy. Snapshot all 16 channel ++ * slots, sample global SCM/fault state across the copy, and gate the copy and ++ * final RUN on a completely quiet readback. ++ * Wake11: that snapshot proved channels 3 and 7 were both running. Keep the ++ * channel-3 sequence unchanged, then apply the same firmware teardown to every ++ * other run-bit + nonzero-interval channel before the post-teardown reset. ++ * Wake12: channel 7 is the phosphorus/BMP284 async client. Its firmware has no ++ * owner-specific stop path, so do not invent another store when the generic ++ * stop times out. Record the remaining scheduler state read-only and refuse. ++ * Wake13/14: every device stop of channel 3 that idled (wake8..wake12) came ++ * 1.5-2.2 ms after the controller reset; the two that did not (wake13/14) came ++ * 11/33 ms after it (pr_flush in the checkpoint), with class-0 state identical to ++ * wake12's passes. Wake15 closed that window (no pr_flush between the reset and ++ * the stops, console lock held), recorded the reset-to-stop latency and the peers ++ * per stop, and accepted one identity-pinned stopped-without-idle channel (3 or ++ * 7) as wake13 did for channel 7. Wake15's channel-7-first order then showed ++ * channel 7 never idles even at 0.3 ms (it is 0/3 over wake11/12/15), with ++ * nothing ticking. Wake16 therefore returns to channel 3 first (early, the only ++ * stop that ever idled) and then channel 7, which takes the stable-stop path ++ * (SCM 0/0 or 3/0, constant); every stop logs why the fingerprint did or did not ++ * hold (stable_why=). Wake14's class-0 prelude stays removed. ++ * ++ * aop_wake_wrapper=1: before 0x220, the AppleA7IOP status-3 resume wrapper ++ * writes (b4b854..b4b8b4). No CPU RUN, no auto-boot, no NMI. ++ */ ++#ifndef _A9_AOP_RELOAD_H ++#define _A9_AOP_RELOAD_H ++ ++#include ++#include ++#include "aop-warm-record.h" ++ ++/* Firmware device spi0: IOP 0x10100000 (space 2, 0x1000) via /arm-io ranges. */ ++#define A9_SPI0_PA 0x210100000ULL ++#define A9_SPI0_SIZE 0x1000U ++#define A9_SPI0_WRAPPER_PA 0x210800000ULL /* the only layout this applies to */ ++#define A9_SPI0_CONTROL 0x000U ++#define A9_SPI0_CLOCK 0x008U /* 0102f2aa programs 0x01b71b00 here */ ++#define A9_SPI0_FAULT 0x8d0U /* bit 2n = channel n, read by 0102f20a */ ++#define A9_SPI0_FAULT_8D4 0x8d4U ++#define A9_SPI0_CHANNEL_BASE 0x100U ++#define A9_SPI0_CHANNEL_STRIDE 0x080U ++#define A9_SPI0_CHANNELS 16U ++#define A9_SPI0_CHANNEL_REGS 7U ++#define A9_SPI0_CHANNEL_DESC 0x018U ++#define A9_SPI0_CH3 0x280U /* base + 0x100 + 0x80 * 3 */ ++#define A9_SPI0_CH3_0C 0x28cU ++#define A9_SPI0_RESET 0x100U /* control bit 8 */ ++#define A9_SPI0_ENABLE 0x001U /* control bit 0 */ ++#define A9_SPI0_CH3_PENDING 0x040U ++/* The firmware spins without bound; the AP gives each poll 1 ms. */ ++#define A9_SPI0_POLL_US 1000U ++#define A9_SPI0_DELAY_US 100U /* 0102b3f8(100, 0), twice */ ++#define A9_SPI0_REGS 6 ++ ++static const u32 a9_spi0_offsets[A9_SPI0_REGS] = { ++ A9_SPI0_CONTROL, A9_SPI0_CLOCK, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++ A9_SPI0_CH3, A9_SPI0_CH3_0C, ++}; ++ ++/* Snapshot indices into a9_spi0_offsets. */ ++enum { A9_SPI0_I_CONTROL, A9_SPI0_I_CLOCK, A9_SPI0_I_FAULT }; ++ ++/* ++ * Channel-3 teardown after the reset (ds-aop-spi0-ch3-round3.md section 5). ++ * The reset leaves C3 = 0x00010447: idle but still enabled, async-armed and ++ * with the class-0 fault IRQ on, so a level source re-latched +0x8d0 bit 6 ++ * within ~105 ms with the CPU halted. AOPSPIMaster_teardownChannel ++ * (0102e4f8) stops the channel and masks both IRQ classes; the fault ++ * handler (0102f20a) then acknowledges with C3C bit 5. ++ */ ++#define A9_SPI0_CH3_ROUTE 0x284U ++#define A9_SPI0_CH3_RATE 0x288U ++#define A9_SPI0_CH3_DESC 0x298U /* channel +0x18, zeroed on teardown */ ++#define A9_SPI0_CH3_RUN 0x001U /* C3 bit 0, channel run/enable */ ++#define A9_SPI0_CH3_IDLE 0x002U /* C3 bit 1, polled after the stop */ ++#define A9_SPI0_CH3_IRQ0 0x040U /* C3 bit 6, +0x8d0 class-0/fault enable */ ++#define A9_SPI0_CH3_IRQ1 0x100U /* C3 bit 8, +0x8d4 class-1 enable */ ++#define A9_SPI0_CHANNEL_PROGRESS_SHIFT 24U /* hardware-updated, not an IRQ class */ ++#define A9_SPI0_CH3_STOP 0x411U /* clear run bit 0 and bits 4, 10 */ ++#define A9_SPI0_CH3_CLASS0 0x450U /* mask class 0: bits 4, 6, 10 */ ++#define A9_SPI0_CH3_CLASS1 0x510U /* mask class 1: bits 4, 8, 10 */ ++#define A9_SPI0_CH3_ACK 0x020U /* C3C bit 5, ORed by 0102f20a; self-clearing */ ++#define A9_SPI0_PHOSPHORUS_CHANNEL 7U ++#define A9_SPI0_PHOSPHORUS_CTL_MASK 0x00ffffffU ++#define A9_SPI0_PHOSPHORUS_CTL_RUN 0x00010445U ++#define A9_SPI0_PHOSPHORUS_CTL_STOPPED 0x00010444U ++#define A9_SPI0_PHOSPHORUS_ROUTE 0x00000040U ++#define A9_SPI0_PHOSPHORUS_RATE 0x019000c8U ++#define A9_SPI0_PHOSPHORUS_CFG 0x00200009U ++/* Channel 3 is the MPU6900 async FIFO client (route 0x10, same rate/config). */ ++#define A9_SPI0_MPU_ASYNC_CHANNEL 3U ++#define A9_SPI0_MPU_ASYNC_ROUTE 0x00000010U ++/* Stopped control word, any top byte, class-0 enable optional (0x40). */ ++#define A9_SPI0_STABLE_CTL_STOPPED 0x00010404U ++#define A9_SPI0_STABLE_CTL_MASK 0x00ffffbfU ++/* Peer sampling and the "late stop" mark are reported, never gated, in-kernel. */ ++#define A9_SPI0_LATE_STOP_US 3000U ++/* ++ * AOPSPIMaster_setChannelEnabled(false) (0102ef7c) stores C3 & ~0x411 only ++ * if bit 0 was set, then spins on bit 1 until master +0x30, and panics ++ * "SCMto:%u" (0x010398da) past it. Event 1 sets +0x30 to 0x2710 us when it ++ * is 0 (0102e6fa, as in the live-cold object), through timer +0x18 = ++ * us * Hz / 1000000 (0102ff3c): a 10 ms bound. The AP polls in 1 ms slices, ++ * up to twice that, sampling C3, the SCM state and +0x8d0 after each slice. ++ */ ++#define A9_SPI0_CH3_IDLE_FW_US 10000U ++#define A9_SPI0_CH3_SLICE_US 1000U ++#define A9_SPI0_CH3_SLICES 20U /* 20 ms */ ++/* At least 150 ms (the 16:27 re-latch took ~105 ms) of samples before RUN. */ ++#define A9_SPI0_WINDOW_SAMPLES 31U ++#define A9_SPI0_WINDOW_US 5000U /* 30 sleeps >= 5 ms each */ ++#define A9_SPI0_WINDOW_MS 150U /* required first-to-last sample span */ ++#define A9_SPI0_CH3_REGS 7 ++ ++static const u32 a9_spi0_ch3_offsets[A9_SPI0_CH3_REGS] = { ++ A9_SPI0_CH3, A9_SPI0_CH3_ROUTE, A9_SPI0_CH3_RATE, A9_SPI0_CH3_0C, ++ A9_SPI0_CH3_DESC, A9_SPI0_FAULT, A9_SPI0_FAULT_8D4, ++}; ++ ++/* Every firmware-visible channel slot, read only in wake10 and selected in wake11. */ ++static const u32 a9_spi0_channel_offsets[A9_SPI0_CHANNEL_REGS] = { ++ 0x00U, 0x04U, 0x08U, 0x0cU, 0x10U, 0x14U, 0x18U, ++}; ++ ++enum { ++ A9_CHANNEL_I_CTL, A9_CHANNEL_I_ROUTE, A9_CHANNEL_I_RATE, A9_CHANNEL_I_CFG, ++ A9_CHANNEL_I_INTERVAL, A9_CHANNEL_I_CURSOR, A9_CHANNEL_I_DESC, ++}; ++ ++/* Snapshot indices into a9_spi0_ch3_offsets. */ ++enum { ++ A9_CH3_I_C3, A9_CH3_I_ROUTE, A9_CH3_I_RATE, A9_CH3_I_C3C, A9_CH3_I_DESC, ++ A9_CH3_I_8D0, A9_CH3_I_8D4, ++}; ++ ++/* One idle-poll slice: C3 as the slice ended, then SCM state and +0x8d0. */ ++struct a9_aop_ch3_slice { ++ u64 ns; /* since the stop store */ ++ u32 c3, scm_lo, scm_hi, fault; ++}; ++ ++/* Wake12 read-only scheduler evidence for an additional-channel stop poll. */ ++struct a9_aop_channel_diag_slice { ++ u32 cfg, interval, cursor, fault_8d4; ++}; ++ ++struct a9_aop_spi0_ch3 { ++ u32 before[A9_SPI0_CH3_REGS], after[A9_SPI0_CH3_REGS]; ++ u32 stop, idle, class0, class1, ack; /* words written / polled */ ++ int poll; /* idle-bit poll result */ ++ bool run_before; /* C3 bit 0 before the stop */ ++ unsigned int slices; /* idle-poll slices used */ ++ struct a9_aop_ch3_slice slice[A9_SPI0_CH3_SLICES]; ++ u64 idle_us; /* stop store to the idle read */ ++ int error; ++ bool ran; /* the guarded teardown executed */ ++ const char *failed; /* distinct refusal reason, or NULL */ ++ /* Wake15: stop latency after the controller reset and the peers' state. */ ++ u64 stop_ns, reset_to_stop_us; ++ u32 peers_run, peers_ticking; /* masks of the other 15 channels */ ++ bool stable; /* accepted stopped-without-idle identity */ ++ const char *stable_why; /* first failed fingerprint condition, NULL if stable */ ++ /* Pre-RUN window after the image copy; a refusal also sets failed. */ ++ unsigned int samples; ++ u64 elapsed_ms; ++ bool window_done, reasserted; ++ unsigned int reassert_sample; ++ u64 reassert_ms; ++ u32 reassert[A9_SPI0_CH3_REGS]; ++}; ++ ++struct a9_aop_spi0; ++ ++struct a9_aop_spi0_channel_op { ++ struct a9_aop_spi0 *spi0; ++ struct a9_aop_spi0_ch3 *state; ++ unsigned int channel; ++}; ++ ++/* ++ * AOP interrupt controller, wrapper offsets (ds-aop-spi0-ch3-round4.md ++ * section 3). IRQ 0x10014 is group 1, source 0x14: bank 0, bit 20. The ++ * group-1 disable/enable banks are write-one commands whose reads return ++ * the mask, 1 = disabled: the sleep command saves +0xa00..+0xa08 (0102c5ce) ++ * before masking all (0102c5da), the restore writes saved to +0xa00 and ++ * ~saved to +0xa80 (0102c9e6/0102c9f4), and aop_interrupt_write_enabled ++ * reads the target bank for the previous state (0102c9c0). So the disable ++ * is judged by bit 20 of the +0xa00 readback (wake6: ffffffff before the ++ * store, every source already masked by the quiesce). +0x81c only reports ++ * an unmasked pending source: the sleep path masks all, writes group 7 ++ * +0xc020/+0xc024 = 2 and spins until +0x81c reads 0 (0102c654), so ++ * all-ones there means the AP is not reading the INTC. +0x808 ends 0 after ++ * the event-3 pulse. +0x900 is the group-1 raise (0102c7c6) and +0x980 its ++ * pre-handler clear (0102c6f2); the firmware never reads them, so their ++ * readbacks are logged only. No register shows a masked source pending. ++ */ ++#define A9_INTC_RESET 0x808U /* event-3 pulse 1 then 0 (0102c592) */ ++#define A9_INTC_CURRENT 0x81cU /* current unmasked source */ ++#define A9_INTC_RAISE 0x900U /* group 1 raise, 0102c7c6 */ ++#define A9_INTC_ACK 0x980U /* group 1 pre-handler ack/clear, 0102c6f2 */ ++#define A9_INTC_DISABLE 0xa00U /* group 1 disable, store at 0102c9ce */ ++#define A9_INTC_ENABLE 0xa80U /* group 1 enable; reads the same mask */ ++#define A9_INTC_G7_DISABLE 0xc020U /* group 7 disable, 2 after sleep */ ++#define A9_INTC_SPI0_SCM 0x00100000U /* IRQ 0x10014 */ ++#define A9_INTC_ALL_ONES 0xffffffffU ++#define A9_INTC_REGS 10 ++ ++static const u32 a9_intc_offsets[A9_INTC_REGS] = { ++ A9_INTC_RESET, A9_INTC_CURRENT, A9_INTC_RAISE, A9_INTC_ACK, ++ A9_INTC_DISABLE, A9_INTC_DISABLE + 4, A9_INTC_DISABLE + 8, ++ A9_INTC_ENABLE, A9_INTC_G7_DISABLE, A9_INTC_G7_DISABLE + 4, ++}; ++ ++/* Snapshot indices into a9_intc_offsets. */ ++enum { ++ A9_INTC_I_RESET, A9_INTC_I_CURRENT, A9_INTC_I_RAISE, A9_INTC_I_ACK, ++ A9_INTC_I_MASK0, A9_INTC_I_MASK1, A9_INTC_I_MASK2, A9_INTC_I_ENABLE0, ++ A9_INTC_I_G7_0, A9_INTC_I_G7_1, ++}; ++ ++/* When the AP snapshots the INTC words. */ ++enum { ++ A9_INTC_AT_PRE_DISABLE, A9_INTC_AT_POST_DISABLE, A9_INTC_AT_PRE_ACK, ++ A9_INTC_AT_POST_ACK, A9_INTC_AT_N, ++}; ++ ++static inline const char *a9_intc_at_name(unsigned int at) ++{ ++ static const char * const names[] = { ++ "pre-disable", "post-disable", "pre-ack", "post-ack", ++ }; ++ ++ return at < A9_INTC_AT_N ? names[at] : "unknown"; ++} ++ ++/* "all-ones" if +0x81c reads ffffffff, which a live INTC cannot report. */ ++static inline const char *a9_intc_view(const u32 *state) ++{ ++ return state[A9_INTC_I_CURRENT] == A9_INTC_ALL_ONES ? "all-ones" : "intc"; ++} ++/* ++ * Read only, never written: the SCM state (also per idle-poll slice) and, ++ * around the acknowledge, the C3 interval pair. ++ */ ++#define A9_SPI0_SCM_LO 0x00cU ++#define A9_SPI0_SCM_HI 0x010U ++#define A9_SPI0_CH3_IVAL 0x290U ++#define A9_SPI0_CH3_CURSOR 0x294U ++#define A9_SPI0_SCM_REGS 4 ++ ++static const u32 a9_spi0_scm_offsets[A9_SPI0_SCM_REGS] = { ++ A9_SPI0_SCM_LO, A9_SPI0_SCM_HI, A9_SPI0_CH3_IVAL, A9_SPI0_CH3_CURSOR, ++}; ++ ++struct a9_aop_intc { ++ u32 state[A9_INTC_AT_N][A9_INTC_REGS]; /* around each store */ ++ unsigned int states; /* bit n: state[n] was read */ ++ u32 scm[2][A9_SPI0_SCM_REGS]; /* before / after the ack */ ++ unsigned int scm_at; /* scm[] row the guarded read fills */ ++ bool disabled, acked; ++ bool reset_done; ++ u32 reset_asserted, reset_after; ++ const char *failed; /* distinct refusal reason, or NULL */ ++}; ++ ++/* Read-only milestones. Wake10 gates the global SCM/fault words, not channels. */ ++struct a9_aop_spi0_trace { ++ u32 control, clock, scm[A9_SPI0_SCM_REGS], ch3[A9_SPI0_CH3_REGS]; ++ u32 channel[A9_SPI0_CHANNELS][A9_SPI0_CHANNEL_REGS]; ++}; ++ ++/* Wake9's second, post-channel-stop replay of AOPSPIMaster_reset. */ ++struct a9_aop_spi0_post_reset { ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved, asserted, polled; ++ int error; ++ bool ran; ++ const char *failed; ++}; ++ ++/* ++ * First failed readback after the teardown, or NULL if all hold. C3C bit 5 ++ * is a pulse: at 19:50 the write of 0x00200029 read back 0x00200009 while ++ * +0x8d0 bit 6 went 1 -> 0, so the acknowledge is judged by bit 6 alone and ++ * the C3C readback is only logged. ++ */ ++static inline const char *a9_spi0_ch3_verify(const u32 *after, bool stable) ++{ ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_RUN) ++ return "failed-ch3-teardown-run"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ0) ++ return "failed-ch3-teardown-class0"; ++ if (after[A9_CH3_I_C3] & A9_SPI0_CH3_IRQ1) ++ return "failed-ch3-teardown-class1"; ++ if (!(after[A9_CH3_I_C3] & A9_SPI0_CH3_IDLE) && !stable) ++ return "failed-ch3-teardown-idle"; ++ if (after[A9_CH3_I_DESC]) ++ return "failed-ch3-teardown-desc"; ++ if (after[A9_CH3_I_8D0] & A9_SPI0_CH3_PENDING) ++ return "failed-ch3-teardown-pending"; ++ return NULL; ++} ++ ++/* ++ * Has firmware run since the last SoC reset? auto-boot alone cannot say: ++ * the outgoing aop_quiesce=1 halt clears it, exactly like a DFU cold start. ++ * The previous kernel's reserved-RAM shutdown transcript says whether its ++ * firmware ran. That DRAM can outlive a DFU; wrapper +0x4020 outbox enable ++ * cannot: set after reset (12/12 DFU cold snapshots), cleared by the first ++ * store of every Linux stop. No SPI0 access is needed to decide. ++ */ ++enum a9_spi0_prev_state { ++ A9_SPI0_PREV_MISSING, /* no transcript magic, or none reserved */ ++ A9_SPI0_PREV_INVALID, /* magic present, header/checksum rejected */ ++ A9_SPI0_PREV_VALID, ++}; ++ ++struct a9_spi0_prev { ++ enum a9_spi0_prev_state state; ++ bool hello; /* "SHUTDOWN begin ... hello=1": its firmware ran */ ++ bool halted; /* "QUIESCE step=complete ... halted=1" + steps bit */ ++ bool carried; /* "SPI0-CARRY retained=1": ran earlier in the chain */ ++ u32 steps; /* "QUIESCE step=complete ... steps=0x..." */ ++}; ++ ++enum a9_spi0_reason { ++ /* Run the reset. */ ++ A9_SPI0_LIVE_AUTO_BOOT, /* auto-boot 1 before the stop */ ++ A9_SPI0_PREV_HALTED, /* previous kernel quiesced and halted it */ ++ A9_SPI0_PREV_LIVE, /* previous kernel's firmware ran, no halt */ ++ A9_SPI0_PREV_CARRIED, /* an earlier kernel in the chain ran it */ ++ /* Skip: the block is not touched. */ ++ A9_SPI0_FRESH, /* no transcript, auto-boot 0: DFU cold */ ++ A9_SPI0_PREV_BAD, /* transcript present but corrupt */ ++ A9_SPI0_PREV_NO_RUN, /* transcript: nothing ran in the chain */ ++ A9_SPI0_STALE, /* run evidence, wrapper at reset state */ ++}; ++ ++static inline const char *a9_spi0_reason_name(enum a9_spi0_reason reason) ++{ ++ static const char * const names[] = { ++ "live-auto-boot", "prev-quiesce-halted", "prev-live", "prev-carried", ++ "fresh-instance", "prev-transcript-invalid", "prev-no-run", ++ "stale-transcript", ++ }; ++ ++ return reason <= A9_SPI0_STALE ? names[reason] : "unknown"; ++} ++ ++static inline const char *a9_spi0_prev_name(enum a9_spi0_prev_state state) ++{ ++ return state == A9_SPI0_PREV_VALID ? "valid" : ++ state == A9_SPI0_PREV_INVALID ? "invalid" : "missing"; ++} ++ ++static inline bool a9_spi0_reason_runs(enum a9_spi0_reason reason) ++{ ++ return reason <= A9_SPI0_PREV_CARRIED; ++} ++ ++/* ++ * Release policy: a reason that would run the reload is allowed by default ++ * only for prev-quiesce-halted, the one state device-proven end to end. A ++ * skip (fresh, bad record, nothing ran) is allowed only while the wrapper is ++ * still at its reset state; stale records skip as before. Untested lifts both. ++ */ ++static inline bool a9_spi0_policy_allows(enum a9_spi0_reason reason, bool outbox_enabled, ++ bool untested) ++{ ++ if (untested || reason == A9_SPI0_PREV_HALTED || reason == A9_SPI0_STALE) ++ return true; ++ if (a9_spi0_reason_runs(reason)) ++ return false; ++ /* ++ * A skip means "nothing ran since the SoC reset", which only holds if the ++ * wrapper is still at its reset state (+0x4020 outbox enable set, 12/12 DFU ++ * cold starts). Without it some earlier kernel stopped the firmware and its ++ * record is lost: a cold load would land on a halted, never-torn-down AOP. ++ */ ++ return outbox_enabled; ++} ++ ++/* Token search inside one transcript line, which holds no NUL. */ ++static inline bool a9_spi0_line_has(const char *line, size_t length, ++ const char *token, size_t n, ++ const char **at) ++{ ++ size_t i; ++ ++ for (i = 0; n <= length && i <= length - n; i++) { ++ if (!memcmp(line + i, token, n)) { ++ *at = line + i + n; ++ return true; ++ } ++ } ++ return false; ++} ++ ++#define a9_spi0_has(line, length, token, at) \ ++ a9_spi0_line_has(line, length, token, sizeof(token) - 1, at) ++ ++static inline bool a9_spi0_line_starts(const char *line, size_t length, ++ const char *prefix, size_t n) ++{ ++ return length >= n && !memcmp(line, prefix, n); ++} ++ ++#define a9_spi0_starts(line, length, prefix) \ ++ a9_spi0_line_starts(line, length, prefix, sizeof(prefix) - 1) ++ ++static inline u32 a9_spi0_hex(const char *p, const char *end) ++{ ++ u32 value = 0; ++ ++ for (; p < end; p++) { ++ if (*p >= '0' && *p <= '9') ++ value = value << 4 | (u32)(*p - '0'); ++ else if (*p >= 'a' && *p <= 'f') ++ value = value << 4 | (u32)(*p - 'a' + 10); ++ else ++ break; ++ } ++ return value; ++} ++ ++/* Our own checksummed transcript; only whole-line prefixes count. */ ++static inline void a9_spi0_prev_parse(struct a9_spi0_prev *p, ++ const char *text, size_t length) ++{ ++ const char *end = text + length, *line, *next, *at; ++ size_t n; ++ ++ for (line = text; line < end; line = next < end ? next + 1 : end) { ++ next = memchr(line, '\n', end - line); ++ if (!next) ++ next = end; ++ n = next - line; ++ if (a9_spi0_starts(line, n, "SHUTDOWN begin ") && ++ a9_spi0_has(line, n, " hello=1 ", &at)) ++ p->hello = true; ++ if (a9_spi0_starts(line, n, "QUIESCE step=complete ")) { ++ if (a9_spi0_has(line, n, " steps=0x", &at)) ++ p->steps = a9_spi0_hex(at, next); ++ p->halted = a9_spi0_has(line, n, " halted=1 ", &at) && ++ (p->steps & (1U << A9_HO_STEP_HALTED)); ++ } ++ if (a9_spi0_starts(line, n, "SPI0-CARRY retained=1 ")) ++ p->carried = true; ++ } ++} ++ ++/* ++ * Skips unless there is positive evidence: running the reset on a fresh ++ * block would be the first AP access to SPI0 with nothing retained to ++ * clear (and ps_spi0 may gate it), while a skip on retained state only ++ * repeats the known, non-hanging accel STARTEP crash. ++ */ ++static inline enum a9_spi0_reason a9_spi0_decide(bool auto_boot, bool outbox_enabled, ++ const struct a9_spi0_prev *p) ++{ ++ if (auto_boot) ++ return A9_SPI0_LIVE_AUTO_BOOT; ++ if (p->state == A9_SPI0_PREV_MISSING) ++ return A9_SPI0_FRESH; ++ if (p->state != A9_SPI0_PREV_VALID) ++ return A9_SPI0_PREV_BAD; ++ if (!p->hello && !p->carried) ++ return A9_SPI0_PREV_NO_RUN; ++ if (outbox_enabled) ++ return A9_SPI0_STALE; ++ if (p->hello) ++ return p->halted ? A9_SPI0_PREV_HALTED : A9_SPI0_PREV_LIVE; ++ return A9_SPI0_PREV_CARRIED; ++} ++ ++struct a9_aop_spi0 { ++ void __iomem *base; /* mapped only with aop_spi0_reset=1 */ ++ u32 before[A9_SPI0_REGS], after[A9_SPI0_REGS]; ++ u32 saved; /* control word before the reset */ ++ u32 asserted; /* saved & ~0x101 | 0x100 */ ++ u32 polled; /* control word when the polls ended */ ++ u32 prerun; /* +0x8d0 just before RUN */ ++ u32 prerun_8d4, prerun_scm_lo, prerun_scm_hi; ++ u32 pmgr_before, pmgr_after; /* AP PMGR ps_spi0 raw, read-only */ ++ int pmgr_before_errno, pmgr_after_errno; ++ int error; ++ bool ran, reasserted, copy_faulted; ++ const char *gate_failed; ++ const char *skipped; ++ struct a9_spi0_prev prev; /* parsed at probe from the transcript */ ++ u32 auto_boot, outbox; /* boot snapshot, before the stop */ ++ enum a9_spi0_reason reason; ++ bool decided; ++ struct a9_aop_spi0_ch3 ch3; /* after a clean reset only */ ++ /* Wake11 reuses one bounded scratch record for each additional channel. */ ++ struct a9_aop_spi0_ch3 channel; ++ u32 teardown_selected, teardown_done; ++ unsigned int teardown_failed_channel; ++ struct a9_aop_spi0_post_reset post_reset; /* after the stop latches SCM */ ++ struct a9_aop_intc intc; /* IRQ 0x10014 around the reset */ ++ struct a9_aop_spi0_trace trace; /* latest read-only milestone */ ++ /* Appended so every pre-wake12 field, including ch3, keeps its layout. */ ++ struct a9_aop_channel_diag_slice channel_diag[A9_SPI0_CH3_SLICES]; ++ /* Wake13: exact phosphorus stopped-without-idle exception, by channel. */ ++ u32 teardown_stable; ++ /* Wake15: channel 3 shares the sampled diagnostics; reset completion time. */ ++ struct a9_aop_channel_diag_slice ch3_diag[A9_SPI0_CH3_SLICES]; ++ u64 reset_done_ns; ++}; ++ ++/* 0102f2f4: bits 8 and 0 clear; the reset write adds bit 8. */ ++static inline u32 a9_spi0_base_word(u32 saved) ++{ ++ return saved & ~(A9_SPI0_RESET | A9_SPI0_ENABLE); ++} ++ ++#endif +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index e8ae90098..d208eacb6 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -1,19 +1,28 @@ + // SPDX-License-Identifier: GPL-2.0-only OR MIT + /* +- * Apple T8010 Always-On Processor (AOP) ++ * Apple S8000 and T8010 Always-On Processor (AOP) + * + * The AOP runs RTKit firmware that the operating system loads: a main image + * into the AOP's SRAM and an external segment into system memory, with boot +- * arguments filled in. Its services talk AFK/EPIC over shared ring buffers on +- * the RTKit application endpoints, as on later SoCs (see the Asahi Linux AOP ++ * arguments filled in. On the S8000 (A9) the AOP sees its SRAM through a window ++ * that has to be set up first, and its firmware takes more boot arguments. ++ * Its services talk AFK/EPIC over shared ring buffers on the RTKit ++ * application endpoints, as on later SoCs (see the Asahi Linux AOP + * driver). Each service with a device tree child node of the same name gets a + * child platform device; its driver receives the service's reports through + * apple_aop_listen() and sends requests with apple_aop_call(). + * +- * The firmware cannot be restarted once stopped, so the AOP starts once per +- * boot and the driver cannot be unbound or unloaded after that. ++ * J71t can reload after a certified quiesce/halt/drain across kexec. Other ++ * retained states are refused. Unbind/unload after starting remains forbidden. + */ + ++#include ++#include ++#include ++#include ++#include ++#include ++#include "aop-warm-spi0.h" + #include + #include + #include +@@ -41,6 +50,14 @@ + #define APPLE_AOP_IRQ_MASK_ALL 0x1111 + #define APPLE_AOP_IRQ_INBOX BIT(0) + #define APPLE_AOP_IRQ_OUTBOX BIT(12) ++/* S8000: the window through which the AOP sees its SRAM */ ++#define APPLE_AOP_REMAP_PA_LO 0x08 ++#define APPLE_AOP_REMAP_PA_HI 0x10 ++#define APPLE_AOP_REMAP_IOVA_LO 0x18 ++#define APPLE_AOP_REMAP_IOVA_HI 0x20 ++#define APPLE_AOP_REMAP_SIZE 0x28 ++#define APPLE_AOP_REMAP_RESERVED 0x30 ++#define APPLE_AOP_REMAP_ENABLE 0x38 + + #define APPLE_AOP_CONTROL_AUTO_BOOT 0x0 + +@@ -54,6 +71,13 @@ + #define APPLE_AOP_BOOTARG_ECID APPLE_AOP_FOURCC('E', 'C', 'I', 'D') + #define APPLE_AOP_BOOTARG_NONC APPLE_AOP_FOURCC('N', 'O', 'N', 'C') + #define APPLE_AOP_BOOTARG_EC0P APPLE_AOP_FOURCC('E', 'C', '0', 'p') ++#define APPLE_AOP_BOOTARG_AOPT APPLE_AOP_FOURCC('A', 'O', 'P', 't') ++#define APPLE_AOP_BOOTARG_NCAL APPLE_AOP_FOURCC('n', 'C', 'a', 'l') ++#define APPLE_AOP_BOOTARG_WRAD APPLE_AOP_FOURCC('W', 'r', 'A', 'd') ++#define APPLE_AOP_BOOTARG_SOC APPLE_AOP_FOURCC('S', 'O', 'C', '_') ++#define APPLE_AOP_BOOTARG_SOCR APPLE_AOP_FOURCC('S', 'O', 'C', 'R') ++#define APPLE_AOP_BOOTARG_STKG APPLE_AOP_FOURCC('S', 'T', 'K', 'G') ++#define APPLE_AOP_MAX_BOOTARGS 10 + + #define APPLE_AOP_HALT_TIMEOUT_US 10000 + +@@ -78,7 +102,9 @@ + #define AFK_OPC_GET_BUF_ACK 0xa1 + #define AFK_OPC_SEND 0xa2 + #define AFK_OPC_START 0xa3 +-#define AFK_OPC_SHUTDOWN_ACK 0xc1 ++#define AFK_OPC_SHUTDOWN 0xa4 ++#define AFK_OPC_SHUTDOWN_ACK 0x87 ++#define AFK_OPC_HIBERNATE_ACK 0xc3 + + #define AFK_BLOCK_STEP 0x40 + #define AFK_MAX_BUFFER SZ_1M +@@ -100,6 +126,12 @@ + #define APPLE_AOP_MAX_CALLS 8 + #define APPLE_AOP_CALL_TIMEOUT msecs_to_jiffies(1000) + ++/* Service properties */ ++#define APPLE_AOP_PROPERTY_MAX 64 ++#define APPLE_AOP_PROP_CALIBRATION 0x3d ++#define APPLE_AOP_CALIBRATION_PARTIAL 1 ++#define APPLE_AOP_CALIBRATION_COMPLETE 2 ++ + struct afk_qe_header { + __le32 magic; + __le32 size; +@@ -160,6 +192,8 @@ struct apple_aop_ep { + + struct mutex lock; /* tx ring, seq and calls */ + u16 seq; ++ bool initialized; ++ struct completion stopped; + struct apple_aop_call *calls[APPLE_AOP_MAX_CALLS]; + }; + +@@ -182,12 +216,25 @@ struct apple_aop_register_work { + struct apple_aop_service service; + }; + ++struct apple_aop_hw { ++ /* Set up the AOP's window onto its SRAM before starting it */ ++ bool sram_window; ++ /* A fresh NONC boot argument */ ++ bool nonce; ++ /* AOPt, SOC_, SOCR, WrAd, STKG and nCal boot arguments */ ++ bool soc_bootargs; ++}; ++ + struct apple_aop { + struct device *dev; ++ const struct apple_aop_hw *hw; + void __iomem *akf; + void __iomem *control; + void __iomem *sram; ++ resource_size_t akf_base; ++ resource_size_t sram_base; + resource_size_t sram_size; ++ u32 target, chip_id, chip_revision; + struct apple_rtkit *rtk; + + void *ext; +@@ -202,6 +249,16 @@ struct apple_aop { + struct mutex children_lock; /* children, removing */ + struct list_head children; + bool removing; ++ u64 accel_reports; ++ int shutdown_error; ++ /* J71t only; all other boards retain their original start/stop path. */ ++ bool warm, hello, iop_crashed, force_halted, force_drained; ++ void __iomem *warm_wrapper; ++ struct device_node *domains[5], *spi0_pmgr; ++ struct a9_aop_spi0 spi0; ++ const char *experiment_step; ++ struct a9_sl_header *shutdown_log; ++ char *shutdown_text; + }; + + static bool apple_aop_is_started(struct apple_aop *aop) +@@ -223,6 +280,409 @@ static bool apple_aop_halt(struct apple_aop *aop) + APPLE_AOP_HALT_TIMEOUT_US); + } + ++static bool a9_aop_ramoops_ready(void) ++{ ++ struct console *con; ++ bool ready = false; ++ int cookie; ++ ++ cookie = console_srcu_read_lock(); ++ for_each_console_srcu(con) { ++ if (!strcmp(con->name, "ramoops") && ++ (console_srcu_read_flags(con) & (CON_ENABLED | CON_SUSPENDED)) == ++ CON_ENABLED) { ++ ready = true; ++ break; ++ } ++ } ++ console_srcu_read_unlock(cookie); ++ return ready; ++} ++ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* J71t warm reload integration. SPI0 machinery is the release6 wake16 code. */ ++#define AOP_DOMAINS 5 ++#define AOP_CPU_CONTROL 0x44 ++#define AOP_CPU_START BIT(4) ++#define AOP_OUTBOX_CONTROL 0x4020 ++#define AOP_INBOX_CONTROL 0x4008 ++#define AOP_OUTBOX_ENABLE BIT(0) ++#define AOP_MBOX_FAULTS GENMASK(19, 18) ++#define cold_defer 1 ++#define spi0_untested false ++ ++static u32 aop_rd(struct apple_aop *aop, u32 offset) ++{ ++ return readl(aop->warm_wrapper + offset); ++} ++ ++static bool a9_aop_force_stopped(struct apple_aop *aop) ++{ ++ return !(aop_rd(aop, AOP_CPU_CONTROL) & AOP_CPU_START) && ++ !readl(aop->control) && ++ !(aop_rd(aop, AOP_OUTBOX_CONTROL) & (AOP_OUTBOX_ENABLE | AOP_MBOX_FAULTS)) && ++ !(aop_rd(aop, AOP_INBOX_CONTROL) & AOP_MBOX_FAULTS); ++} ++ ++struct aop_warm_write { void __iomem *target; u32 value; }; ++static void aop_warm_store(void *data) ++{ ++ struct aop_warm_write *w = data; ++ ++ writel(w->value, w->target); ++ readl(w->target); ++} ++ ++static int aop_wr(struct apple_aop *aop, u32 offset, u32 value) ++{ ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct aop_warm_write w = { aop->warm_wrapper + offset, value }; ++ ++ if ((aop_rd(aop, AOP_INBOX_CONTROL) | aop_rd(aop, AOP_OUTBOX_CONTROL)) & ++ AOP_MBOX_FAULTS) ++ return -EIO; ++ dev_info(aop->dev, "COLD-W step=%s bank=wrapper offset=%04x before=%08x value=%08x\n", ++ aop->experiment_step ?: "warm-stop", offset, readl(w.target), value); ++ { ++ int ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, aop_warm_store, &w); ++ ++ dev_info(aop->dev, "COLD-R offset=%04x after=%08x\n", offset, readl(w.target)); ++ return ret; ++ } ++} ++ ++static int a9_aop_cold_checkpoint_noflush(struct apple_aop *aop, const char *step) ++{ ++ if (!a9_aop_ramoops_ready()) ++ return -ENODEV; ++ dev_info(aop->dev, "COLD checkpoint=%s flush=skipped\n", step); ++ return 0; ++} ++ ++static int a9_aop_cold_checkpoint(struct apple_aop *aop, const char *step) ++{ ++ int ret = a9_aop_cold_checkpoint_noflush(aop, step); ++ ++ if (ret) ++ return ret; ++ if (!pr_flush(1000, false)) ++ return -ETIMEDOUT; ++ mb(); ++ return 0; ++} ++ ++#include "aop-warm-spi0.c.h" ++static int apple_aop_send(struct apple_aop_ep *ep, u8 opcode, u64 payload); ++ ++/* No cmdline override for untested states: default is the release6 policy. */ ++static atomic_t aop_warm_attempted = ATOMIC_INIT(0); ++ ++static int apple_aop_warm_stop(struct apple_aop *aop) ++{ ++ unsigned int fifo, i; ++ u32 offset, val; ++ int ret; ++ ++ ret = aop_wr(aop, 0x4020, aop_rd(aop, 0x4020) & ~BIT(0)); ++ if (!ret) ++ ret = aop_wr(aop, 0x4000, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0xc00, 0x1111); ++ if (!ret) ++ ret = aop_wr(aop, 0x44, aop_rd(aop, 0x44) & ~BIT(4)); ++ if (ret) ++ return ret; ++ writel(0, aop->control); ++ msleep(10); ++ if (!a9_aop_force_stopped(aop)) ++ return -EIO; ++ aop->force_halted = true; ++ for (fifo = 0; fifo < 2; fifo++) { ++ offset = fifo ? 0x4020 : 0x4008; ++ for (i = 0; i < 64; i++) { ++ val = aop_rd(aop, offset); ++ if (!a9_aop_force_stopped(aop) || (val & AOP_MBOX_FAULTS)) ++ return -EIO; ++ if (val & BIT(17)) ++ break; ++ readq(aop->warm_wrapper + (fifo ? 0x4038 : 0x4018)); ++ } ++ if (!(aop_rd(aop, offset) & BIT(17))) ++ return -ETIMEDOUT; ++ } ++ aop->force_drained = true; ++ return 0; ++} ++ ++static void apple_aop_warm_put_node(void *node) ++{ ++ of_node_put(node); ++} ++ ++static void apple_aop_warm_unmap(void *p) ++{ ++ memunmap(p); ++} ++ ++static int apple_aop_warm_prepare(struct apple_aop *aop) ++{ ++ struct device *dev = aop->dev; ++ struct device_node *np = dev->of_node, *mem; ++ struct apple_pmgr_observation states[AOP_DOMAINS]; ++ struct a9_sl_header header; ++ struct resource res; ++ char *text; ++ u32 auto_boot, outbox; ++ unsigned int i; ++ int ret; ++ ++ if (!of_property_read_bool(aop->dev->of_node, "apple,certified-reload") || !aop->hw->sram_window) ++ return 0; ++ /* Missing the new DT is not permission to run a retained instance. */ ++ mem = of_parse_phandle(np, "memory-region", 0); ++ if (!mem) ++ return dev_err_probe(dev, -EINVAL, "warm reload requires the matching J71t DT\n"); ++ ret = of_address_to_resource(mem, 0, &res); ++ of_node_put(mem); ++ if (ret || resource_size(&res) != A9_HO_PAGE || res.start != 0x8583fc000ULL || ++ aop->akf_base != A9_SPI0_WRAPPER_PA) ++ return -EINVAL; ++ aop->shutdown_log = memremap(res.start, A9_HO_PAGE, MEMREMAP_WB); ++ if (!aop->shutdown_log) ++ return -ENOMEM; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_unmap, aop->shutdown_log); ++ if (ret) ++ return ret; ++ /* Keep release6's transcript location and validation ABI. */ ++ aop->shutdown_log = (void *)aop->shutdown_log + A9_HO_LOG_OFFSET; ++ aop->shutdown_text = (char *)(aop->shutdown_log + 1); ++ memcpy(&header, aop->shutdown_log, sizeof(header)); ++ text = devm_kmemdup(dev, aop->shutdown_text, A9_SL_CAPACITY, GFP_KERNEL); ++ if (!text) ++ return -ENOMEM; ++ if (header.magic == A9_SL_MAGIC) { ++ aop->spi0.prev.state = A9_SPI0_PREV_INVALID; ++ if (a9_sl_valid(&header, text)) { ++ aop->spi0.prev.state = A9_SPI0_PREV_VALID; ++ a9_spi0_prev_parse(&aop->spi0.prev, text, header.length); ++ for (i = 0; i < header.length;) { ++ char *next = memchr(text + i, '\n', header.length - i); ++ unsigned int n = next ? next - text - i : header.length - i; ++ ++ dev_info(dev, "AOP-PREV %.*s\n", n, text + i); ++ i += n + 1; ++ } ++ } ++ } ++ aop->warm_wrapper = devm_ioremap(dev, aop->akf_base, 0x1c000); ++ if (!aop->warm_wrapper) ++ return -ENOMEM; ++ for (i = 0; i < AOP_DOMAINS; i++) { ++ aop->domains[i] = of_parse_phandle(np, "apple,observe-domains", i); ++ if (!aop->domains[i]) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->domains[i]); ++ if (ret) ++ return ret; ++ } ++ aop->spi0_pmgr = of_parse_phandle(np, "apple,spi0-domain", 0); ++ if (!aop->spi0_pmgr) ++ return -EINVAL; ++ ret = devm_add_action_or_reset(dev, apple_aop_warm_put_node, aop->spi0_pmgr); ++ if (ret) ++ return ret; ++ /* Check all domains before reading the wrapper; no power transitions. */ ++ ret = apple_pmgr_observe(aop->domains, AOP_DOMAINS, states, a9_aop_spi0_pmgr_on, &(bool){false}); ++ if (ret) ++ return dev_err_probe(dev, ret, "AOP domain guard refused\n"); ++ auto_boot = readl(aop->control); ++ outbox = aop_rd(aop, AOP_OUTBOX_CONTROL); ++ ret = a9_aop_spi0_policy(aop, auto_boot, outbox); ++ if (ret) ++ return ret; /* Before first halt, mask, SRAM or SPI0 write. */ ++ /* Orchard's old halt leaves +0x4020 enabled. On a known kexec, ++ * that alone cannot certify the cold wrapper baseline. Release6's ++ * wrapper discriminator was tied to its own outgoing stop routine. ++ */ ++ if (of_property_read_bool(of_chosen, "linux,booted-from-kexec") && ++ a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev) != A9_SPI0_PREV_HALTED) { ++ dev_info(dev, "SPI0 policy decision=refuse reason=uncertified-kexec auto_boot=%u out4020=%08x: block not touched\n", ++ auto_boot, outbox); ++ return -EPERM; ++ } ++ aop->warm = true; ++ aop->spi0.reason = a9_spi0_decide(auto_boot, outbox & BIT(0), &aop->spi0.prev); ++ if (!a9_spi0_reason_runs(aop->spi0.reason)) { ++ /* Original cold path: no halt, drain, reset, or SPI0 access. */ ++ dev_info(dev, "SPI0 reset decision=skip reason=%s: block not touched\n", ++ a9_spi0_reason_name(aop->spi0.reason)); ++ return 0; ++ } ++ if (atomic_xchg(&aop_warm_attempted, 1)) ++ return dev_err_probe(dev, -EPERM, "refused-second-experiment-attempt\n"); ++ if (!a9_aop_ramoops_ready()) ++ return dev_err_probe(dev, -ENODEV, "warm reload requires ramoops console\n"); ++ aop->spi0.base = devm_ioremap(dev, A9_SPI0_PA, A9_SPI0_SIZE); ++ if (!aop->spi0.base) ++ return -ENOMEM; ++ /* Consume certification before the first control write. Reloading the ++ * module after a failure must not reuse it as a second attempt. ++ */ ++ aop->shutdown_log->magic = 0; ++ wmb(); ++ ret = apple_aop_warm_stop(aop); ++ if (!ret) ++ ret = a9_aop_spi0_prereset(aop, auto_boot, outbox); ++ if (ret) ++ dev_err(dev, "warm reload refused errno=%d; no copy/RUN, recover with DFU\n", ret); ++ return ret; ++} ++ ++static int apple_aop_warm_release(struct apple_aop *aop) ++{ ++ int ret; ++ ++ ret = a9_aop_spi0_recheck(aop); ++ if (!ret) ++ ret = a9_aop_spi0_trace(aop, "after-window"); ++ if (!ret) ++ ret = a9_aop_intc_ack(aop); ++ if (!ret) ++ ret = a9_aop_intc_reset(aop); ++ if (!ret) ++ ret = a9_aop_cold_checkpoint(aop, "before-run"); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "pre-run-final", "FINAL", true); ++ return ret; ++} ++ ++static void apple_aop_warm_hash(struct apple_aop *aop, u8 *digest) ++{ ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ if (aop->ext) ++ sha256_update(&ctx, aop->ext, aop->ext_size); ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) ++ if (aop->eps[i].buf) ++ sha256_update(&ctx, aop->eps[i].buf, aop->eps[i].buf_size); ++ sha256_final(&ctx, digest); ++} ++ ++static __printf(2, 3) void apple_aop_warm_log(struct apple_aop *aop, const char *fmt, ...) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ va_list args; ++ char line[512]; ++ size_t n; ++ ++ va_start(args, fmt); ++ n = vscnprintf(line, sizeof(line), fmt, args); ++ va_end(args); ++ dev_info(aop->dev, "%s", line); ++ if (n > h->capacity - h->length) { ++ h->truncated++; ++ return; ++ } ++ memcpy(aop->shutdown_text + h->length, line, n); ++ wmb(); ++ h->checksum = a9_sl_hash(h->checksum, line, n); ++ h->length += n; ++ h->lines++; ++ wmb(); ++} ++ ++static void apple_aop_warm_shutdown(struct apple_aop *aop) ++{ ++ struct a9_sl_header *h = aop->shutdown_log; ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ unsigned long deadline = jiffies + msecs_to_jiffies(10000), start; ++ unsigned int i, acked = 0; ++ int ret = -EIO; ++ ++ memset(h, 0, sizeof(*h)); ++ h->version = A9_SL_VERSION; ++ h->capacity = A9_SL_CAPACITY; ++ h->checksum = A9_SL_HASH_SEED; ++ h->mode = 1; ++ h->begin_ns = ktime_get_boottime_ns(); ++ wmb(); ++ h->magic = A9_SL_MAGIC; ++ apple_aop_warm_log(aop, "SHUTDOWN begin mode=quiesce hello=%d phase=stopping iop_crashed=%d\n", ++ aop->hello, aop->iop_crashed); ++ if (!aop->hello || aop->iop_crashed || aop->shutdown_error) ++ goto done; ++ /* Rates-zero has stopped reports. Stop one AFK at a time, max 2s ++ * per endpoint and 10s in total, including send time. Never force halt ++ * after a failed ACK. Keep every grant allocated for the entire hook. ++ */ ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ unsigned long limit; ++ ++ if (!ep->initialized) ++ continue; ++ if (time_after_eq(jiffies, deadline)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ reinit_completion(&ep->stopped); ++ start = jiffies; ++ ret = apple_aop_send(ep, AFK_OPC_SHUTDOWN, 0); ++ if (ret) ++ goto done; ++ limit = time_before(jiffies, deadline) ? min_t(unsigned long, ++ deadline - jiffies, msecs_to_jiffies(2000)) : 0; ++ if (!limit || !wait_for_completion_timeout(&ep->stopped, limit)) { ++ ret = -ETIMEDOUT; ++ goto done; ++ } ++ acked++; ++ apple_aop_warm_log(aop, "QUIESCE step=afk-ack ep=0x%x state=stopped ms=%u\n", ++ ep->index, jiffies_to_msecs(jiffies - start)); ++ } ++ /* release6: complete every AFK stop before disabling endpoint routing. */ ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; ++ } ++ for (i = 0; i < AFK_ENDPOINT_COUNT; i++) { ++ struct apple_aop_ep *ep = &aop->eps[i]; ++ ++ if (!ep->initialized) ++ continue; ++ /* MGMT STARTEP bit 0 disables routing, no separate stop ACK. */ ++ ret = apple_rtkit_send_message(aop->rtk, 0, ++ (5ULL << 52) | ((u64)ep->index << 32) | 1, NULL, false); ++ if (ret) ++ goto done; ++ } ++ if (acked != 17) { ++ ret = -ENOTCONN; ++ goto done; /* Never certify an incompletely initialized instance. */ ++ } ++ ret = apple_rtkit_shutdown_preserve(aop->rtk); ++ if (ret) ++ goto done; ++ msleep(20); ++ apple_aop_warm_hash(aop, before); ++ msleep(20); ++ apple_aop_warm_hash(aop, after); ++ ret = memcmp(before, after, sizeof(before)) ? -EBUSY : 0; ++ apple_aop_warm_log(aop, "QUIESCE step=ram-stable window_ms=20 status=%d\n", ret); ++ if (ret || aop->iop_crashed) ++ goto done; ++ ret = apple_aop_warm_stop(aop); ++done: ++ apple_aop_warm_log(aop, "QUIESCE step=complete status=%d quiesced=%d halted=%d drained=%d steps=0x%x afk_acks=%u iop_crashed=%d\n", ++ ret, !ret, aop->force_halted, aop->force_drained, ++ !ret ? 0x1bff : 0, acked, aop->iop_crashed); ++ if (ret) ++ dev_err(aop->dev, "quiesce failed; next AOP must refuse, recover with DFU\n"); ++} ++ + /* Firmware loading */ + + struct apple_aop_bootarg { +@@ -232,11 +692,17 @@ struct apple_aop_bootarg { + bool found; + }; + ++static void apple_aop_add_bootarg(struct apple_aop_bootarg *args, unsigned int *count, ++ u32 key, const void *value, u32 size) ++{ ++ args[(*count)++] = (struct apple_aop_bootarg){ .key = key, .value = value, .size = size }; ++} ++ + static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t size, +- struct apple_aop_bootarg *args, int count) ++ struct apple_aop_bootarg *args, unsigned int count) + { + size_t offset, end, pos; +- int i; ++ unsigned int i; + + offset = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_OFFSET); + end = get_unaligned_le32(image + APPLE_AOP_IMAGE_BOOTARGS_SIZE); +@@ -269,13 +735,29 @@ static int apple_aop_patch_bootargs(struct apple_aop *aop, u8 *image, size_t siz + return 0; + } + ++/* S8000: point the AOP's window at its SRAM; the firmware runs from there. */ ++static void apple_aop_map_sram(struct apple_aop *aop) ++{ ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_LO); ++ writel(0, aop->akf + APPLE_AOP_REMAP_IOVA_HI); ++ writel(aop->sram_size & ~0xfffULL, aop->akf + APPLE_AOP_REMAP_SIZE); ++ writel(0, aop->akf + APPLE_AOP_REMAP_RESERVED); ++ writel(lower_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_LO); ++ writel(upper_32_bits(aop->sram_base), aop->akf + APPLE_AOP_REMAP_PA_HI); ++ writel(1, aop->akf + APPLE_AOP_REMAP_ENABLE); ++} ++ + static int apple_aop_load(struct apple_aop *aop, u64 ecid) + { ++ struct apple_aop_bootarg args[APPLE_AOP_MAX_BOOTARGS]; + const struct firmware *main_fw, *ext_fw; + const char *main_name, *ext_name; + struct device_node *np = aop->dev->of_node; +- u8 prng[32], nonce[8]; +- __le64 ecid_le = cpu_to_le64(ecid), ext_le; ++ u8 prng[32], nonce[8], guard[4], ncal = 0; ++ __le64 ecid_le = cpu_to_le64(ecid), ext_le, wrapper_le; ++ __le32 target_le, chip_le, revision_le; ++ unsigned int count = 0; ++ size_t offset; + u8 *image; + int ret; + +@@ -316,28 +798,58 @@ static int apple_aop_load(struct apple_aop *aop, u64 ecid) + } + + get_random_bytes(prng, sizeof(prng)); +- get_random_bytes(nonce, sizeof(nonce)); +- { +- struct apple_aop_bootarg args[] = { +- { APPLE_AOP_BOOTARG_PRNG, prng, sizeof(prng) }, +- { APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le) }, +- { APPLE_AOP_BOOTARG_NONC, nonce, sizeof(nonce) }, +- { APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le) }, +- }; +- +- ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, +- ARRAY_SIZE(args)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_PRNG, &prng, sizeof(prng)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_ECID, &ecid_le, sizeof(ecid_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_EC0P, &ext_le, sizeof(ext_le)); ++ if (aop->hw->nonce) { ++ get_random_bytes(nonce, sizeof(nonce)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NONC, &nonce, sizeof(nonce)); ++ } ++ if (aop->hw->soc_bootargs) { ++ /* A random stack guard with one zero byte */ ++ get_random_bytes(guard, sizeof(guard)); ++ guard[get_random_u32_below(sizeof(guard))] = 0; ++ target_le = cpu_to_le32(aop->target); ++ chip_le = cpu_to_le32(aop->chip_id); ++ revision_le = cpu_to_le32(aop->chip_revision); ++ wrapper_le = cpu_to_le64(aop->akf_base); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_AOPT, ++ &target_le, sizeof(target_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_NCAL, &ncal, sizeof(ncal)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_WRAD, ++ &wrapper_le, sizeof(wrapper_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOC, ++ &chip_le, sizeof(chip_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_SOCR, ++ &revision_le, sizeof(revision_le)); ++ apple_aop_add_bootarg(args, &count, APPLE_AOP_BOOTARG_STKG, &guard, sizeof(guard)); + } ++ ++ ret = apple_aop_patch_bootargs(aop, image, main_fw->size, args, count); + if (ret) { + dev_err(aop->dev, "unexpected boot arguments in %s: %d\n", main_name, ret); + } else { +- memcpy_toio(aop->sram, image, main_fw->size); +- memset_io(aop->sram + main_fw->size, 0, aop->sram_size - main_fw->size); ++ /* The SRAM takes 32-bit accesses, as Apple's loader makes them. */ ++ if (aop->spi0.ran) { ++ ret = a9_aop_spi0_gate(aop, "pre-copy-final", "PRECOPY", false); ++ if (!ret) ++ ret = a9_aop_copy_verify_wake10(aop, image, main_fw->size); ++ if (!ret) ++ ret = a9_aop_spi0_gate(aop, "after-copy", "AFTERCOPY", true); ++ if (ret) ++ goto out_image; ++ } else { ++ __iowrite32_copy(aop->sram, image, main_fw->size / 4); ++ } ++ for (offset = main_fw->size; offset + 4 <= aop->sram_size; offset += 4) ++ writel_relaxed(0, aop->sram + offset); + } + ++out_image: + kfree_sensitive(image); + memzero_explicit(prng, sizeof(prng)); + memzero_explicit(nonce, sizeof(nonce)); ++ memzero_explicit(guard, sizeof(guard)); + memzero_explicit(&ecid_le, sizeof(ecid_le)); + out_ext: + release_firmware(ext_fw); +@@ -448,7 +960,8 @@ static void apple_aop_announce(struct apple_aop_ep *ep, u32 channel, const void + const struct epic_service_announce *announce = data; + struct apple_aop_register_work *rw; + +- if (len < sizeof(*announce)) ++ /* Only the name and channel are used; the S8000's announcements are shorter. */ ++ if (len < offsetofend(struct epic_service_announce, channel)) + return; + + rw = kzalloc_obj(*rw); +@@ -475,6 +988,8 @@ static void apple_aop_report(struct apple_aop_ep *ep, u32 channel, u16 subtype, + struct apple_aop_listener *listener; + bool handled = false; + ++ if (ep->index == 0x21 && subtype == 0xc2) ++ aop->accel_reports++; + mutex_lock(&aop->listeners_lock); + list_for_each_entry(listener, &aop->listeners, list) { + if (listener->endpoint == ep->index && listener->channel == channel) { +@@ -631,6 +1146,7 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + + switch (FIELD_GET(AFK_MSG_OPCODE, msg)) { + case AFK_OPC_INIT: ++ ep->initialized = true; + apple_aop_send(ep, AFK_OPC_INIT_ACK, 0); + break; + case AFK_OPC_GET_BUF: +@@ -651,7 +1167,10 @@ static void apple_aop_recv_message(void *cookie, u8 endpoint, u64 msg) + break; + case AFK_OPC_INIT_UNK: + case AFK_OPC_START_ACK: ++ break; + case AFK_OPC_SHUTDOWN_ACK: ++ case AFK_OPC_HIBERNATE_ACK: ++ complete(&ep->stopped); + break; + default: + dev_dbg(aop->dev, "endpoint 0x%02x: message 0x%llx\n", endpoint, msg); +@@ -662,7 +1181,8 @@ static void apple_aop_crashed(void *cookie, const void *crashlog, size_t crashlo + { + struct apple_aop *aop = cookie; + +- dev_err(aop->dev, "firmware crashed\n"); ++ aop->iop_crashed = true; ++ dev_err(aop->dev, "firmware crashed iop_crashed=1\n"); + } + + static const struct apple_rtkit_ops apple_aop_rtkit_ops = { +@@ -767,6 +1287,8 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + + if (!ep) + return -EINVAL; ++ if (service->aop->warm && READ_ONCE(service->aop->iop_crashed)) ++ return -EIO; + init_completion(&call.done); + + mutex_lock(&ep->lock); +@@ -797,6 +1319,58 @@ int apple_aop_call(const struct apple_aop_service *service, u16 subtype, const v + } + EXPORT_SYMBOL_GPL(apple_aop_call); + ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len) ++{ ++ u8 buf[8 + APPLE_AOP_PROPERTY_MAX]; ++ int ret; ++ ++ if (len > APPLE_AOP_PROPERTY_MAX) ++ return -EINVAL; ++ put_unaligned_le32(0, buf); ++ put_unaligned_le32(key, buf + 4); ++ memcpy(buf + 8, value, len); ++ ++ ret = apple_aop_call(service, APPLE_AOP_SET_PROPERTY, buf, 8 + len, NULL, 0); ++ return ret > 0 ? -EIO : ret; ++} ++EXPORT_SYMBOL_GPL(apple_aop_set_property); ++ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count) ++{ ++ unsigned long needed = 0, applied = 0; ++ __le32 state; ++ unsigned int i; ++ int len, ret; ++ ++ for (i = 0; i < count; i++) { ++ const void *blob; ++ ++ if (cal[i].group) ++ needed |= BIT(cal[i].group); ++ blob = of_get_property(dev->of_node, cal[i].property, &len); ++ if (!blob) ++ continue; ++ ret = apple_aop_set_property(service, cal[i].key, blob, len); ++ if (ret) { ++ dev_warn(dev, "%s not applied: %d\n", cal[i].property, ret); ++ continue; ++ } ++ if (cal[i].group) ++ applied |= BIT(cal[i].group); ++ } ++ ++ if (needed && applied == needed) { ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_COMPLETE); ++ } else { ++ dev_warn(dev, "factory calibration incomplete\n"); ++ state = cpu_to_le32(APPLE_AOP_CALIBRATION_PARTIAL); ++ } ++ return apple_aop_set_property(service, APPLE_AOP_PROP_CALIBRATION, &state, sizeof(state)); ++} ++EXPORT_SYMBOL_GPL(apple_aop_calibrate); ++ + /* Driver */ + + static void apple_aop_free_buffers(struct apple_aop *aop) +@@ -825,27 +1399,35 @@ static int apple_aop_probe(struct platform_device *pdev) + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); + mutex_init(&aop->children_lock); + INIT_LIST_HEAD(&aop->children); + +- aop->akf = devm_platform_ioremap_resource_byname(pdev, "akf"); ++ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "akf"); ++ aop->akf = devm_ioremap_resource(dev, res); + if (IS_ERR(aop->akf)) + return PTR_ERR(aop->akf); ++ aop->akf_base = res->start; + aop->control = devm_platform_ioremap_resource_byname(pdev, "control"); + if (IS_ERR(aop->control)) + return PTR_ERR(aop->control); + aop->sram = devm_platform_get_and_ioremap_resource(pdev, 2, &res); + if (IS_ERR(aop->sram)) + return PTR_ERR(aop->sram); ++ aop->sram_base = res->start; + aop->sram_size = resource_size(res); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + +- if (apple_aop_is_started(aop)) ++ ret = apple_aop_warm_prepare(aop); ++ if (ret) ++ return ret; ++ ++ if (!aop->spi0.ran && apple_aop_is_started(aop)) + return dev_err_probe(dev, -EBUSY, + "already started; the AOP starts once per boot\n"); + +@@ -853,6 +1435,17 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + return dev_err_probe(dev, ret, "no apple,ecid from the bootloader\n"); + ++ if (aop->hw->soc_bootargs) { ++ ret = of_property_read_u32(np, "apple,aop-target", &aop->target); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-id", &aop->chip_id); ++ if (!ret) ++ ret = of_property_read_u32(np, "apple,chip-revision", &aop->chip_revision); ++ if (ret) ++ return dev_err_probe(dev, ret, ++ "no AOP target or chip ID from the bootloader\n"); ++ } ++ + aop->eps = devm_kcalloc(dev, AFK_ENDPOINT_COUNT, sizeof(*aop->eps), GFP_KERNEL); + if (!aop->eps) + return -ENOMEM; +@@ -860,6 +1453,7 @@ static int apple_aop_probe(struct platform_device *pdev) + aop->eps[i].aop = aop; + aop->eps[i].index = AFK_ENDPOINT_START + i; + mutex_init(&aop->eps[i].lock); ++ init_completion(&aop->eps[i].stopped); + } + + ret = apple_aop_load(aop, ecid); +@@ -867,6 +1461,9 @@ static int apple_aop_probe(struct platform_device *pdev) + if (ret) + goto err_free; + ++ if (aop->hw->sram_window) ++ apple_aop_map_sram(aop); ++ + aop->rtk = apple_rtkit_init(dev, aop, "mbox", 0, &apple_aop_rtkit_ops); + if (IS_ERR(aop->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(aop->rtk), "failed to init RTKit\n"); +@@ -884,10 +1481,24 @@ static int apple_aop_probe(struct platform_device *pdev) + prop = of_find_property(np, "apple,ecid", NULL); + if (prop) + of_remove_property(np, prop); +- writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, +- aop->akf + APPLE_AOP_CPU_CONTROL); ++ if (aop->spi0.ran) { ++ ret = apple_aop_warm_release(aop); ++ if (ret) ++ goto err_halt; ++ } ++ if (aop->spi0.ran) { ++ aop->experiment_step = "warm-run"; ++ ret = aop_wr(aop, APPLE_AOP_CPU_CONTROL, ++ readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN); ++ if (ret) ++ goto err_halt; ++ } else { ++ writel(readl(aop->akf + APPLE_AOP_CPU_CONTROL) | APPLE_AOP_CPU_CONTROL_RUN, ++ aop->akf + APPLE_AOP_CPU_CONTROL); ++ } + writel(1, aop->control + APPLE_AOP_CONTROL_AUTO_BOOT); + ++ dev_info(dev, "AOP released warm=%d\n", aop->spi0.ran); + ret = apple_rtkit_wake(aop->rtk); + if (ret) { + dev_err(dev, "firmware did not boot: %d\n", ret); +@@ -900,6 +1511,8 @@ static int apple_aop_probe(struct platform_device *pdev) + dev_warn(dev, "failed to start endpoint 0x%02x: %d\n", i, ret); + } + ++ aop->hello = true; ++ dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); + return 0; + +@@ -929,27 +1542,81 @@ static void apple_aop_shutdown(struct platform_device *pdev) + mutex_lock(&aop->children_lock); + aop->removing = true; + list_for_each_entry_safe(child, tmp, &aop->children, list) { ++ ++ if (aop->warm) { ++ const struct apple_aop_service *service = dev_get_platdata(&child->pdev->dev); ++ static const u32 keys[] = { 0x16, 0x5c, 0 }; ++ ++ /* release6 sets the three accelerometer rates only. */ ++ if (strcmp(service->name, "accel")) ++ continue; ++ __le32 zero = 0; ++ unsigned int i; ++ int ret; ++ ++ for (i = 0; i < ARRAY_SIZE(keys); i++) { ++ ret = apple_aop_set_property(service, keys[i], &zero, sizeof(zero)); ++ if (ret) { ++ aop->shutdown_error = ret; ++ break; ++ } ++ } ++ } ++ /* Shutdown retains listeners, children and DMA grants, as release6 does. ++ * Unregister is remove-time work and can wait in IIO/driver core. ++ */ ++ if (aop->warm) ++ continue; + platform_device_unregister(child->pdev); + list_del(&child->list); + kfree(child); + } + mutex_unlock(&aop->children_lock); + ++ if (aop->warm) { ++ apple_aop_warm_shutdown(aop); ++ return; ++ } + if (!apple_aop_halt(aop)) + dev_err(aop->dev, "failed to halt\n"); + } + ++static const struct apple_aop_hw apple_aop_s8000_hw = { ++ .sram_window = true, ++ .soc_bootargs = true, ++}; ++ ++static const struct apple_aop_hw apple_aop_t8010_hw = { ++ .nonce = true, ++}; ++ + static const struct of_device_id apple_aop_of_match[] = { +- { .compatible = "apple,t8010-aop" }, ++ { .compatible = "apple,s8000-aop", .data = &apple_aop_s8000_hw }, ++ { .compatible = "apple,t8010-aop", .data = &apple_aop_t8010_hw }, + {} + }; + MODULE_DEVICE_TABLE(of, apple_aop_of_match); + ++static ssize_t warm_state_show(struct device *dev, struct device_attribute *attr, char *buf) ++{ ++ struct apple_aop *aop = dev_get_drvdata(dev); ++ ++ if (!aop) ++ return sysfs_emit(buf, "owned=0\n"); ++ return sysfs_emit(buf, "owned=%u reload=%u iop_crashed=%u accel_reports=%llu\n", ++ aop->hello, aop->spi0.ran, READ_ONCE(aop->iop_crashed), ++ READ_ONCE(aop->accel_reports)); ++} ++static DEVICE_ATTR_RO(warm_state); ++static struct attribute *apple_aop_attrs[] = { &dev_attr_warm_state.attr, NULL }; ++ATTRIBUTE_GROUPS(apple_aop); ++ + static struct platform_driver apple_aop_driver = { + .driver = { + .name = "apple-aop", + .of_match_table = apple_aop_of_match, + .suppress_bind_attrs = true, ++ .dev_groups = apple_aop_groups, + }, + .probe = apple_aop_probe, + .shutdown = apple_aop_shutdown, +@@ -957,5 +1624,5 @@ static struct platform_driver apple_aop_driver = { + module_platform_driver(apple_aop_driver); + + MODULE_AUTHOR("Dalton Alexandre "); +-MODULE_DESCRIPTION("Apple T8010 Always-On Processor driver"); ++MODULE_DESCRIPTION("Apple S8000 and T8010 Always-On Processor driver"); + MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c +index 725ab3fa7..d7928621a 100644 +--- a/drivers/soc/apple/rtkit.c ++++ b/drivers/soc/apple/rtkit.c +@@ -4,6 +4,8 @@ + * Copyright (C) The Asahi Linux Contributors + */ + ++#include ++#include + #include "rtkit-internal.h" + + enum { +@@ -867,6 +869,54 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) + } + EXPORT_SYMBOL_GPL(apple_rtkit_boot); + ++/* Unlike shutdown(), retain all grants until the caller halts the CPU. ++ * Reinit frees DMA buffers and is unsafe before that point on S8000. ++ */ ++static void apple_rtkit_grants_hash(struct apple_rtkit *rtk, u8 *digest) ++{ ++ struct apple_rtkit_shmem *buffers[] = { ++ &rtk->ioreport_buffer, &rtk->crashlog_buffer, ++ &rtk->oslog_buffer, &rtk->syslog_buffer, ++ }; ++ struct sha256_ctx ctx; ++ unsigned int i; ++ ++ sha256_init(&ctx); ++ dma_rmb(); ++ for (i = 0; i < ARRAY_SIZE(buffers); i++) ++ if (buffers[i]->buffer) ++ sha256_update(&ctx, buffers[i]->buffer, buffers[i]->size); ++ sha256_final(&ctx, digest); ++} ++ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk) ++{ ++ u8 before[SHA256_DIGEST_SIZE], after[SHA256_DIGEST_SIZE]; ++ int ret; ++ ++ if (rtk->crashed) ++ return -EIO; ++ ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_QUIESCED); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=ap-ack state=0x10\n"); ++ ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE); ++ if (ret) ++ return ret; ++ dev_info(rtk->dev, "QUIESCE step=iop-ack state=0x201\n"); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, before); ++ msleep(20); ++ apple_rtkit_grants_hash(rtk, after); ++ if (rtk->crashed || memcmp(before, after, sizeof(before))) ++ return -EBUSY; ++ /* No more callback can race the caller's halt/drain. Do not reinit. */ ++ apple_mbox_stop(rtk->mbox); ++ flush_workqueue(rtk->wq); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_rtkit_shutdown_preserve); ++ + int apple_rtkit_shutdown(struct apple_rtkit *rtk) + { + int ret; +diff --git a/include/linux/soc/apple/aop.h b/include/linux/soc/apple/aop.h +index 8c8bc71b6..b5188a0b1 100644 +--- a/include/linux/soc/apple/aop.h ++++ b/include/linux/soc/apple/aop.h +@@ -7,6 +7,8 @@ + + #include + ++struct device; ++ + struct apple_aop; + + /* +@@ -40,4 +42,30 @@ void apple_aop_unlisten(const struct apple_aop_service *service); + int apple_aop_call(const struct apple_aop_service *service, u16 subtype, + const void *data, size_t len, void *reply, size_t reply_len); + ++/* The EPIC notification subtype that sets a service property */ ++#define APPLE_AOP_SET_PROPERTY 0x4 ++ ++/* Set property @key of a service to @len (at most 64) bytes at @value. */ ++int apple_aop_set_property(const struct apple_aop_service *service, u32 key, ++ const void *value, size_t len); ++ ++/* ++ * A factory calibration blob of a sensor: the bootloader copies it from the ++ * Apple device tree into the sensor's node as @property, and the firmware ++ * takes it as property @key. The calibration is complete once a blob of each ++ * @group (1 to 31) was applied; group 0 blobs are not needed for that. ++ */ ++struct apple_aop_calibration { ++ const char *property; ++ u32 key; ++ u8 group; ++}; ++ ++/* ++ * Send the factory calibration found in @dev's node, then tell the firmware ++ * whether it is complete. Some firmware (S8000) sends no reports before. ++ */ ++int apple_aop_calibrate(const struct apple_aop_service *service, struct device *dev, ++ const struct apple_aop_calibration *cal, unsigned int count); ++ + #endif /* _LINUX_SOC_APPLE_AOP_H_ */ +diff --git a/include/linux/soc/apple/pmgr-observe.h b/include/linux/soc/apple/pmgr-observe.h +new file mode 100644 +index 000000000..477b7fa22 +--- /dev/null ++++ b/include/linux/soc/apple/pmgr-observe.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++#define _LINUX_SOC_APPLE_PMGR_OBSERVE_H ++ ++#include ++ ++struct device_node; ++ ++struct apple_pmgr_observation { ++ u32 raw; ++ int genpd_status; ++}; ++ ++/* ++ * Diagnostic only. Nodes must be in child-before-parent lock order, with no ++ * duplicates (maximum five). Takes no power/runtime-PM reference. read() must ++ * not sleep or acquire genpd locks; it runs with IRQs disabled and all domain ++ * locks held, only if every genpd and hardware actual/target state is ON and ++ * reset/disable/parent-off are clear. Hardware auto-PM remains untouched and is ++ * recorded in raw; locks exclude software transitions, not autonomous gating. ++ * -EHOSTDOWN means read() was skipped. ++ */ ++int apple_pmgr_observe(struct device_node **nodes, unsigned int count, ++ struct apple_pmgr_observation *states, ++ void (*read)(void *), void *data); ++ ++#endif +diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h +index 1f9ebb618..c8e8df255 100644 +--- a/include/linux/soc/apple/rtkit.h ++++ b/include/linux/soc/apple/rtkit.h +@@ -124,6 +124,8 @@ int apple_rtkit_wake(struct apple_rtkit *rtk); + * Shutdown the co-processor + */ + int apple_rtkit_shutdown(struct apple_rtkit *rtk); ++/* AP 0x10 / IOP 0x201, stable system RAM, stop RX; retain every grant. */ ++int apple_rtkit_shutdown_preserve(struct apple_rtkit *rtk); + + /* + * Stop the mailbox of a shut-down co-processor, so that its power domain can diff --git a/kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch b/kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch new file mode 100644 index 00000000..8a4e6613 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0016-soc-apple-aop-refuse-pre-sync-sleep-during-firmware-.patch @@ -0,0 +1,80 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] soc: apple: aop: refuse pre-sync sleep during firmware boot + +A suspend while the AOP firmware is still starting stops it part way +through its boot, and the sensors behind it do not come back. Register a +suspend precheck notifier that refuses sleep with -EBUSY until the +firmware has reported ready. + +Assisted-by: LLM +--- + drivers/soc/apple/aop.c | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -226,6 +227,8 @@ + }; + + struct apple_aop { ++ struct notifier_block sleep_precheck; ++ bool sleep_ready; + struct device *dev; + const struct apple_aop_hw *hw; + void __iomem *akf; +@@ -1385,6 +1388,22 @@ + dma_free_coherent(aop->dev, aop->ext_size, aop->ext, aop->ext_dma); + } + ++static int apple_aop_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct apple_aop *aop = container_of(nb, struct apple_aop, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ return READ_ONCE(aop->sleep_ready) ? NOTIFY_OK : notifier_from_errno(-EBUSY); ++} ++ ++static void apple_aop_sleep_unregister(void *data) ++{ ++ struct apple_aop *aop = data; ++ ++ unregister_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++} ++ + static int apple_aop_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; +@@ -1399,6 +1418,13 @@ + if (!aop) + return -ENOMEM; + aop->dev = dev; ++ aop->sleep_precheck.notifier_call = apple_aop_sleep_precheck; ++ ret = register_pm_suspend_precheck_notifier(&aop->sleep_precheck); ++ if (ret) ++ return ret; ++ ret = devm_add_action_or_reset(dev, apple_aop_sleep_unregister, aop); ++ if (ret) ++ return ret; + aop->hw = of_device_get_match_data(dev); + mutex_init(&aop->listeners_lock); + INIT_LIST_HEAD(&aop->listeners); +@@ -1514,6 +1540,7 @@ + aop->hello = true; + dev_info(dev, "AOP start complete warm=%d iop_crashed=0\n", aop->spi0.ran); + platform_set_drvdata(pdev, aop); ++ WRITE_ONCE(aop->sleep_ready, true); + return 0; + + err_halt: diff --git a/kernel/patches/drivers/soc-apple/0017-soc-apple-aop-name-the-sleep-precheck-refusal.patch b/kernel/patches/drivers/soc-apple/0017-soc-apple-aop-name-the-sleep-precheck-refusal.patch new file mode 100644 index 00000000..6e87c03e --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0017-soc-apple-aop-name-the-sleep-precheck-refusal.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:26:19 +0100 +Subject: [PATCH] soc: apple: aop: name the sleep precheck refusal + +Log that the AOP firmware has not started when the precheck refuses, +only during a suspend attempt. The decision is unchanged. + +Assisted-by: LLM +--- + drivers/soc/apple/aop.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/drivers/soc/apple/aop.c b/drivers/soc/apple/aop.c +index f0ad4c87f..84ba67c59 100644 +--- a/drivers/soc/apple/aop.c ++++ b/drivers/soc/apple/aop.c +@@ -1394,7 +1394,12 @@ static int apple_aop_sleep_precheck(struct notifier_block *nb, unsigned long act + + if (action == PM_SUSPEND_PRECHECK_COMPLETE) + return NOTIFY_DONE; +- return READ_ONCE(aop->sleep_ready) ? NOTIFY_OK : notifier_from_errno(-EBUSY); ++ if (READ_ONCE(aop->sleep_ready)) ++ return NOTIFY_OK; ++ /* Only for an attempt: the readiness query polls this. */ ++ if (data == PM_SUSPEND_PRECHECK_ATTEMPT) ++ dev_info(aop->dev, "system sleep refused: AOP firmware not started\n"); ++ return notifier_from_errno(-EBUSY); + } + + static void apple_aop_sleep_unregister(void *data) diff --git a/kernel/patches/drivers/soc-apple/0018-dt-bindings-soc-apple-pmgr-misc-Add-the-S8000.patch b/kernel/patches/drivers/soc-apple/0018-dt-bindings-soc-apple-pmgr-misc-Add-the-S8000.patch new file mode 100644 index 00000000..4263a4b1 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0018-dt-bindings-soc-apple-pmgr-misc-Add-the-S8000.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 09:49:32 +0000 +Subject: [PATCH] dt-bindings: soc: apple: pmgr-misc: Add the S8000 + +The S8000 and S8003 (A9) select their combined SoC and memory +performance state in one PMGR register, as the T8010 does: describe it +as a single "soc-ps" region. + +Assisted-by: LLM +--- + .../bindings/soc/apple/apple,t6000-pmgr-misc.yaml | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml +index e64ca2ce9..001b791dd 100644 +--- a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml ++++ b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml +@@ -12,12 +12,13 @@ maintainers: + description: | + Certain Apple SoCs include additional PMGR power states that are controlled + via a different "misc" control block. This includes the fabric and memory +- controller states. On the T8010 (A10) one register selects the combined +- SoC and memory performance state. ++ controller states. On the S8000/S8003 (A9) and T8010 (A10) one register ++ selects the combined SoC and memory performance state. + + properties: + compatible: + enum: ++ - apple,s8000-pmgr-misc + - apple,t6000-pmgr-misc + - apple,t6020-pmgr-misc + - apple,t8010-pmgr-misc +@@ -40,7 +41,9 @@ allOf: + properties: + compatible: + contains: +- const: apple,t8010-pmgr-misc ++ enum: ++ - apple,s8000-pmgr-misc ++ - apple,t8010-pmgr-misc + then: + properties: + reg: diff --git a/kernel/patches/drivers/soc-apple/0019-soc-apple-pmgr-misc-Run-the-S8000-s-SoC-and-memory-s.patch b/kernel/patches/drivers/soc-apple/0019-soc-apple-pmgr-misc-Run-the-S8000-s-SoC-and-memory-s.patch new file mode 100644 index 00000000..29c524f5 --- /dev/null +++ b/kernel/patches/drivers/soc-apple/0019-soc-apple-pmgr-misc-Run-the-S8000-s-SoC-and-memory-s.patch @@ -0,0 +1,104 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 09:49:32 +0000 +Subject: [PATCH] soc: apple: pmgr-misc: Run the S8000's SoC and memory states + high + +The S8000 and S8003 (A9) boot with SoC and memory in their low +performance state: memory at 800 of 1200 MHz. Nothing on Linux raises +it, so CPU and GPU run on two thirds of the memory bandwidth. On a J71t +the high state takes CPU read bandwidth from 8.8 to 15.1 GB/s and +glmark2 scenes 13-64 % faster at the same GPU clock. + +The register, the two states (0xd low, 0xc high) and the pending bit are +the T8010's, so add an S8000 variant that sets the high state at probe +and the low state on suspend, as the T8010 variant does. iOS's PMGR +driver waits up to 8 ms for the change; it takes 0.5-0.7 ms on the J71t, +close to the T8010's 1 ms timeout, so let a chip give its own timeout +and use iOS's for the S8000. + +Assisted-by: LLM +--- + drivers/soc/apple/Kconfig | 2 +- + drivers/soc/apple/apple-pmgr-misc.c | 30 +++++++++++++++++++++++++---- + 2 files changed, 27 insertions(+), 5 deletions(-) + +diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig +index e68a49f1b..ade72a51f 100644 +--- a/drivers/soc/apple/Kconfig ++++ b/drivers/soc/apple/Kconfig +@@ -24,7 +24,7 @@ config APPLE_PMGR_MISC + controls for SoC devices. This driver manages miscellaneous + power controls. + +- Say 'y' here if you have a Pro/Max/Ultra Apple SoC or an A10. ++ Say 'y' here if you have a Pro/Max/Ultra Apple SoC, an A9 or an A10. + + config APPLE_RTKIT + tristate "Apple RTKit co-processor IPC protocol" +diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c +index f00dcebe3..6d51295ad 100644 +--- a/drivers/soc/apple/apple-pmgr-misc.c ++++ b/drivers/soc/apple/apple-pmgr-misc.c +@@ -29,6 +29,15 @@ + #define T8010_SOC_PSTATE_LOW 0xd + #define T8010_SOC_PSTATE_HIGH 0xc + ++/* ++ * S8000/S8003 SoC/memory states: the same register and values. iOS's PMGR ++ * driver accepts only these two for its SoC domain and waits up to 8 ms ++ * for a change to complete; one takes about 0.5-0.7 ms ++ */ ++#define S8000_SOC_PSTATE_LOW 0xd ++#define S8000_SOC_PSTATE_HIGH 0xc ++#define S8000_SOC_PSTATE_TIMEOUT_US 8000 ++ + enum sys_device { + DEV_FABRIC, + DEV_DCS, +@@ -52,7 +61,7 @@ struct apple_pmgr_misc_hw { + unsigned long devices; /* BIT(enum sys_device) */ + u32 dev_min_ps[DEV_MAX]; + u32 dev_active_ps[DEV_MAX]; /* 0: keep the state the system booted with */ +- bool wait_pending; /* wait for a state change to complete */ ++ u32 pending_timeout_us; /* wait for a state change to complete; 0: don't */ + }; + + struct apple_pmgr_misc { +@@ -81,9 +90,10 @@ static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, + FIELD_MODIFY(APPLE_CLKGEN_PSTATE_DESIRED, &val, pstate); + writel_relaxed(val, misc->devices[dev].base + APPLE_CLKGEN_PSTATE); + +- if (misc->hw->wait_pending && ++ if (misc->hw->pending_timeout_us && + readl_relaxed_poll_timeout_atomic(misc->devices[dev].base + APPLE_CLKGEN_PSTATE, val, +- !(val & APPLE_CLKGEN_PSTATE_PENDING), 1, 1000)) ++ !(val & APPLE_CLKGEN_PSTATE_PENDING), 1, ++ misc->hw->pending_timeout_us)) + dev_warn(misc->dev, "%s state change to %u did not complete\n", + sys_device_names[dev], pstate); + } +@@ -194,10 +204,22 @@ static const struct apple_pmgr_misc_hw apple_pmgr_misc_hw_t8010 = { + .devices = BIT(DEV_SOC), + .dev_min_ps = { [DEV_SOC] = T8010_SOC_PSTATE_LOW }, + .dev_active_ps = { [DEV_SOC] = T8010_SOC_PSTATE_HIGH }, +- .wait_pending = true, ++ .pending_timeout_us = 1000, ++}; ++ ++/* ++ * The S8000/S8003 boot with SoC and memory in their low state too (memory ++ * at 800 of 1200 MHz): run both high while active, as on the T8010 ++ */ ++static const struct apple_pmgr_misc_hw apple_pmgr_misc_hw_s8000 = { ++ .devices = BIT(DEV_SOC), ++ .dev_min_ps = { [DEV_SOC] = S8000_SOC_PSTATE_LOW }, ++ .dev_active_ps = { [DEV_SOC] = S8000_SOC_PSTATE_HIGH }, ++ .pending_timeout_us = S8000_SOC_PSTATE_TIMEOUT_US, + }; + + static const struct of_device_id apple_pmgr_misc_of_match[] = { ++ { .compatible = "apple,s8000-pmgr-misc", .data = &apple_pmgr_misc_hw_s8000 }, + { .compatible = "apple,t6000-pmgr-misc", .data = &apple_pmgr_misc_hw_t600x }, + { .compatible = "apple,t6020-pmgr-misc", .data = &apple_pmgr_misc_hw_t602x }, + { .compatible = "apple,t8010-pmgr-misc", .data = &apple_pmgr_misc_hw_t8010 }, diff --git a/kernel/patches/drivers/sound/0010-ASoC-cs42l81-initialize-stereo-playback-mixer-source.patch b/kernel/patches/drivers/sound/0010-ASoC-cs42l81-initialize-stereo-playback-mixer-source.patch new file mode 100644 index 00000000..6f3222c6 --- /dev/null +++ b/kernel/patches/drivers/sound/0010-ASoC-cs42l81-initialize-stereo-playback-mixer-source.patch @@ -0,0 +1,47 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 16:06:48 +0100 +Subject: [PATCH] ASoC: cs42l81: initialize stereo playback mixer sources + +The ASP slot geometry is configured at power-up but the playback mixer +source selectors are left to their previous values. This can retain a +silent or duplicated channel after codec reset or firmware use. + +Follow AppleCS42L81Audio::updateRouting by assigning mb00 and mb01 to +ASP RX slots 0 and 1 before enabling the playback path. Keep the +existing stereo power and volume sequence unchanged. + +Assisted-by: LLM +--- + sound/soc/codecs/cs42l81.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/sound/soc/codecs/cs42l81.c b/sound/soc/codecs/cs42l81.c +index fc2e93b55..418f91d36 100644 +--- a/sound/soc/codecs/cs42l81.c ++++ b/sound/soc/codecs/cs42l81.c +@@ -81,6 +81,9 @@ + #define CS42L81_INT_STATUS4 0x007b + #define CS42L81_INT_STATUS5 0x007c + #define CS42L81_INT_STATUS6 0x0106 ++/* Playback mixer bus n source: 0x19 + slot selects ASP RX slot. */ ++#define CS42L81_MIX_SRC(n) (0x010b + (n)) ++#define CS42L81_MIX_SRC_ASP(n) (0x19 + (n)) + /* ASP TX slot n source: 0x18 = ADC A, 0x1c = ADC B (0x20/0x24 = DMIC2A/B) */ + #define CS42L81_ASP_TX_SRC(n) (0x0115 + (n)) /* n = 0..9 */ + /* ADC channel n source, same codes as ASP_TX_SRC */ +@@ -318,6 +321,14 @@ static void cs42l81_set_asp(struct cs42l81 *cs) + /* The codec drives BCLK/LRCK (the MCA is the clock consumer). */ + regmap_write(map, CS42L81_ASP_CLK, CS42L81_ASP_CLK_PROVIDER | c); + ++ /* ++ * iOS updateRouting maps mixer buses mb00/mb01 to ASP RX slots ++ * 0/1. Program the sources as well as the slot geometry: reset or ++ * a previous user of the codec need not leave a stereo route. ++ */ ++ regmap_write(map, CS42L81_MIX_SRC(0), CS42L81_MIX_SRC_ASP(0)); ++ regmap_write(map, CS42L81_MIX_SRC(1), CS42L81_MIX_SRC_ASP(1)); ++ + /* ASP RX slot n: byte offset n * b, width b bytes */ + for (n = 0; n < CS42L81_ASP_SLOTS; n++) + regmap_write(map, CS42L81_ASP_RX_SLOT(n), diff --git a/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch b/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch new file mode 100644 index 00000000..daa284cf --- /dev/null +++ b/kernel/patches/drivers/spi/0009-spi-apple-drain-and-idle-the-S8000-PIO-controller-on.patch @@ -0,0 +1,122 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] spi: apple: drain and idle the S8000 PIO controller on + shutdown + +On S8000 the SPI controller runs in PIO mode. A kexec left it with +queued transfers, counters and an asserted chip select, and the next +kernel probed it in that state. + +On shutdown, suspend the SPI core queue, deassert the GPIO chip selects +and write the PIO registers back to idle while the controller is +powered. Do the same at probe, so that a controller inherited in any +state starts idle. Other SoCs use different DMA and interrupt registers +and keep their current behaviour. + +Assisted-by: LLM +--- + drivers/spi/spi-apple.c | 48 +++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 48 insertions(+) + +diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c +index dc5154e1b..ed9f67a59 100644 +--- a/drivers/spi/spi-apple.c ++++ b/drivers/spi/spi-apple.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -141,6 +142,7 @@ struct apple_spi { + struct dmaengine_result dma_result; + const struct apple_spi_hw *hw; + u32 word_delay; ++ struct spi_controller *ctlr; + }; + + static inline void reg_write(struct apple_spi *spi, int offset, u32 value) +@@ -231,6 +233,19 @@ static void apple_spi_init(struct apple_spi *spi) + reg_write(spi, APPLE_SPI_DELAY_POST, 0); + } + ++/* Lab 0015: normalize only the S8000 PIO register set, while powered. */ ++static int apple_spi_s8000_idle(struct apple_spi *spi) ++{ ++ reg_write(spi, APPLE_SPI_CTRL, 0); ++ reg_write(spi, APPLE_SPI_CFG, spi->hw->cfg_base); ++ reg_write(spi, APPLE_SPI_RXCNT, 0); ++ reg_write(spi, APPLE_SPI_TXCNT, 0); ++ reg_write(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS); ++ apple_spi_t8010_clear_status(spi); ++ /* Complete posted writes before publishing children or leaving Linux. */ ++ return readl(spi->regs + APPLE_SPI_CTRL) ? -EIO : 0; ++} ++ + static int apple_spi_prepare_message(struct spi_controller *ctlr, + struct spi_message *msg) + { +@@ -1120,6 +1135,7 @@ static int apple_spi_probe(struct platform_device *pdev) + + spi = spi_controller_get_devdata(ctlr); + platform_set_drvdata(pdev, spi); ++ spi->ctlr = ctlr; + init_completion(&spi->done); + init_completion(&spi->dma_done); + spi->hw = of_device_get_match_data(&pdev->dev); +@@ -1214,6 +1230,11 @@ static int apple_spi_probe(struct platform_device *pdev) + } + + apple_spi_init(spi); ++ if (spi->hw->pio_only) { ++ ret = apple_spi_s8000_idle(spi); ++ if (ret) ++ return ret; ++ } + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret < 0) +@@ -1223,6 +1244,32 @@ static int apple_spi_probe(struct platform_device *pdev) + return 0; + } + ++static void apple_spi_shutdown(struct platform_device *pdev) ++{ ++ struct apple_spi *spi = platform_get_drvdata(pdev); ++ struct spi_controller *ctlr = spi->ctlr; ++ unsigned int cs; ++ int ret; ++ ++ /* Other SoCs use different DMA/IRQ registers and keep their policy. */ ++ if (!spi->hw->pio_only) ++ return; ++ /* Children have shut down; drain the core queue before MMIO. */ ++ ret = spi_controller_suspend(ctlr); ++ if (!ret) ++ ret = pm_runtime_resume_and_get(&pdev->dev); ++ if (ret >= 0) { ++ if (ctlr->cs_gpiods) ++ for (cs = 0; cs < ctlr->num_chipselect; cs++) ++ gpiod_set_value_cansleep(ctlr->cs_gpiods[cs], 0); ++ ret = apple_spi_s8000_idle(spi); ++ } ++ if (ret < 0 && kexec_in_progress) ++ panic("S8000 SPI shutdown failed before kexec: %d", ret); ++ /* Keep runtime power pinned until the jump. */ ++ dev_info(&pdev->dev, "S8000 SPI shutdown result %d\n", ret); ++} ++ + static const struct apple_spi_hw apple_spi_hw_s8000 = { + .legacy_status = true, + .pio_only = true, +@@ -1260,6 +1307,7 @@ MODULE_DEVICE_TABLE(of, apple_spi_of_match); + + static struct platform_driver apple_spi_driver = { + .probe = apple_spi_probe, ++ .shutdown = apple_spi_shutdown, + .driver = { + .name = "apple-spi", + .of_match_table = apple_spi_of_match, diff --git a/kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch b/kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch new file mode 100644 index 00000000..58875bd3 --- /dev/null +++ b/kernel/patches/drivers/usb/0011-usb-preserve-PMU-and-RTC-interrupt-ownership.patch @@ -0,0 +1,36 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] usb: preserve PMU and RTC interrupt ownership + +Shared wake events must have one interrupt owner. Preserve the PMU and +RTC event split so that acknowledgement by one consumer cannot steal the +other consumer's wake event. + +Assisted-by: LLM +--- + drivers/usb/gadget/composite.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 1da3effb2..14e87fbc8 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -968,7 +968,7 @@ static int composite_apple_current_limit(struct usb_composite_dev *cdev, + u16 w_value, u16 w_index) + { + cdev->apple_extra_ma = (s16)w_index; +- DBG(cdev, "Apple host current grant: extra %d mA, suspend %d mA\n", ++ dev_info(&cdev->gadget->dev, "Apple host current grant: extra %d mA, suspend %d mA\n", + (s16)w_index, (s16)w_value); + composite_vbus_draw(cdev, cdev->vbus_draw_ma); + return 0; +@@ -2221,7 +2221,7 @@ unknown: + goto check_value; + } + +- if (ctrl->bRequestType == (USB_DIR_OUT | USB_TYPE_VENDOR | ++ if ((ctrl->bRequestType & ~USB_DIR_IN) == (USB_TYPE_VENDOR | + USB_RECIP_DEVICE) && + ctrl->bRequest == USB_APPLE_REQ_CURRENT_LIMIT && !w_length && + le16_to_cpu(cdev->desc.idVendor) == USB_APPLE_VENDOR_ID) { diff --git a/kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch b/kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch new file mode 100644 index 00000000..74f69a05 --- /dev/null +++ b/kernel/patches/drivers/usb/0012-usb-gadget-composite-draw-no-extra-Apple-current-whi.patch @@ -0,0 +1,47 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 16:00:00 +0100 +Subject: [PATCH] usb: gadget: composite: draw no extra Apple current while + suspended + +The extra current an Apple host grants must not be drawn while the bus +is suspended. reset_config() could apply it during suspend, because the +suspended flag was cleared only at the end of composite_resume(). Skip +the extra current while suspended and clear the flag before the +configuration is applied again on resume. + +Assisted-by: LLM +--- + drivers/usb/gadget/composite.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c +index 1da3effb2..ea76d3918 100644 +--- a/drivers/usb/gadget/composite.c ++++ b/drivers/usb/gadget/composite.c +@@ -959,7 +959,7 @@ static void reset_config(struct usb_composite_dev *cdev) + static void composite_vbus_draw(struct usb_composite_dev *cdev, unsigned int mA) + { + cdev->vbus_draw_ma = mA; +- if (mA && cdev->apple_extra_ma) ++ if (mA && !cdev->suspended && cdev->apple_extra_ma) + mA = max(500 + cdev->apple_extra_ma, 0); + usb_gadget_vbus_draw(cdev->gadget, mA); + } +@@ -2665,6 +2665,7 @@ void composite_resume(struct usb_gadget *gadget) + DBG(cdev, "resume\n"); + if (cdev->driver->resume) + cdev->driver->resume(cdev); ++ cdev->suspended = 0; + if (cdev->config) { + list_for_each_entry(f, &cdev->config->functions, list) { + /* +@@ -2695,8 +2696,6 @@ void composite_resume(struct usb_gadget *gadget) + maxpower = min(maxpower, 100U); + composite_vbus_draw(cdev, maxpower); + } +- +- cdev->suspended = 0; + } + + /*-------------------------------------------------------------------------*/ diff --git a/kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch b/kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch new file mode 100644 index 00000000..b7291559 --- /dev/null +++ b/kernel/patches/drivers/usb/0013-dt-bindings-usb-describe-board-selected-s2idle-gadge.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] dt-bindings: usb: describe board-selected s2idle gadget + retention + +Document apple,retain-gadget-in-s2idle: the board keeps the peripheral +controller and its PHY powered during suspend to idle, so the gadget +stays connected. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/usb/dwc2.yaml | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/Documentation/devicetree/bindings/usb/dwc2.yaml b/Documentation/devicetree/bindings/usb/dwc2.yaml +index b1f66a663..81df7017a 100644 +--- a/Documentation/devicetree/bindings/usb/dwc2.yaml ++++ b/Documentation/devicetree/bindings/usb/dwc2.yaml +@@ -172,6 +172,11 @@ properties: + description: If present indicates that we need to reset the PHY when we + detect a wakeup. This is due to a hardware errata. + ++ apple,retain-gadget-in-s2idle: ++ type: boolean ++ description: ++ Keep the peripheral controller and PHY powered during suspend to idle. ++ + usb-psy-name: + $ref: /schemas/types.yaml#/definitions/string + description: diff --git a/kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch b/kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch new file mode 100644 index 00000000..8e03537a --- /dev/null +++ b/kernel/patches/drivers/usb/0014-usb-dwc2-select-retained-s2idle-gadget-from-board-da.patch @@ -0,0 +1,160 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] usb: dwc2: select retained s2idle gadget from board data + +A USB gadget network link, and anything on it such as a root over NBD, +survives a short suspend to idle when the controller, its PHY and the +gadget are left connected. The host sees NAKs while the system sleeps. + +Keep the gadget connected over suspend to idle when the board sets +apple,retain-gadget-in-s2idle and the port is in device mode and active. +The controller and the PHY are put on the wakeup path so that their +power domains stay on. A suspend precheck refuses any other sleep state, +or a port that is not a connected peripheral, with -EBUSY before sync. + +Assisted-by: LLM +--- + drivers/usb/dwc2/core.h | 5 +++ + drivers/usb/dwc2/platform.c | 63 +++++++++++++++++++++++++++++++++++++ + 2 files changed, 68 insertions(+) + +diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h +index ae563cadc..9dbddbb0d 100644 +--- a/drivers/usb/dwc2/core.h ++++ b/drivers/usb/dwc2/core.h +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include "hw.h" + +@@ -882,6 +883,8 @@ struct dwc2_hregs_backup { + * @reset_phy_on_wake: Quirk saying that we should assert PHY reset on a + * remote wakeup. + * @phy_off_for_suspend: Status of whether we turned the PHY off at suspend. ++ * @kept_for_s2idle: The gadget was left connected and running over ++ * suspend-to-idle (dwc2.s2idle_keep_gadget). + * @need_phy_for_wake: Quirk saying that we should keep the PHY on at + * suspend if we need USB to wake us up. + * @frame_number: Frame number read from the core. For both device +@@ -1088,6 +1091,8 @@ struct dwc2_hsotg { + unsigned int reset_phy_on_wake:1; + unsigned int need_phy_for_wake:1; + unsigned int phy_off_for_suspend:1; ++ unsigned int kept_for_s2idle:1; ++ struct notifier_block sleep_precheck; + u16 frame_number; + + struct phy *phy; +diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c +index ef0d73077..fabee8363 100644 +--- a/drivers/usb/dwc2/platform.c ++++ b/drivers/usb/dwc2/platform.c +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + + #include + +@@ -436,6 +437,9 @@ int dwc2_check_core_version(struct dwc2_hsotg *hsotg) + * in the device private data. This allows the driver to access the dwc2_hsotg + * structure on subsequent calls to driver methods for this device. + */ ++static int dwc2_sleep_precheck(struct notifier_block *, unsigned long, void *); ++static void dwc2_sleep_unregister(void *); ++ + static int dwc2_driver_probe(struct platform_device *dev) + { + struct dwc2_hsotg *hsotg; +@@ -625,6 +629,14 @@ static int dwc2_driver_probe(struct platform_device *dev) + } + } + #endif /* CONFIG_USB_DWC2_PERIPHERAL || CONFIG_USB_DWC2_DUAL_ROLE */ ++ hsotg->sleep_precheck.notifier_call = dwc2_sleep_precheck; ++ retval = register_pm_suspend_precheck_notifier(&hsotg->sleep_precheck); ++ if (retval) ++ goto error; ++ retval = devm_add_action_or_reset(&dev->dev, dwc2_sleep_unregister, hsotg); ++ if (retval) ++ goto error; ++ + return 0; + + #if IS_ENABLED(CONFIG_USB_DWC2_PERIPHERAL) || \ +@@ -646,6 +658,49 @@ static int dwc2_driver_probe(struct platform_device *dev) + return retval; + } + ++/* ++ * Suspend-to-idle with dwc2.s2idle_keep_gadget: leave the controller and ++ * its PHY powered and the gadget connected. Its interrupt is off while the ++ * system sleeps, so the host sees NAKs and nothing else; a gadget network ++ * link (and anything on it, e.g. an NBD root) survives a short sleep. ++ * The controller and PHY are put on the wakeup path so that their power ++ * domains stay on (GENPD_FLAG_ACTIVE_WAKEUP). ++ */ ++static bool dwc2_keep_gadget_for_s2idle(struct dwc2_hsotg *dwc2) ++{ ++ if (!device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") || !pm_suspend_no_platform() || ++ !dwc2_is_device_mode(dwc2) || dwc2->lx_state != DWC2_L0) ++ return false; ++ ++ device_set_wakeup_path(dwc2->dev); ++ if (dwc2->phy && dwc2->phy->dev.parent) ++ device_set_wakeup_path(dwc2->phy->dev.parent); ++ dwc2->kept_for_s2idle = 1; ++ dev_info(dwc2->dev, "s2idle: gadget stays connected\n"); ++ return true; ++} ++ ++static int dwc2_sleep_precheck(struct notifier_block *nb, unsigned long action, void *data) ++{ ++ struct dwc2_hsotg *dwc2 = container_of(nb, struct dwc2_hsotg, sleep_precheck); ++ ++ if (action == PM_SUSPEND_PRECHECK_COMPLETE) ++ return NOTIFY_DONE; ++ if (device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") && ++ (action != PM_SUSPEND_TO_IDLE || ++ (dwc2->op_state != OTG_STATE_B_PERIPHERAL && ++ dwc2->op_state != OTG_STATE_B_IDLE))) ++ return notifier_from_errno(-EBUSY); ++ return NOTIFY_OK; ++} ++ ++static void dwc2_sleep_unregister(void *data) ++{ ++ struct dwc2_hsotg *dwc2 = data; ++ ++ unregister_pm_suspend_precheck_notifier(&dwc2->sleep_precheck); ++} ++ + static int __maybe_unused dwc2_suspend(struct device *dev) + { + struct dwc2_hsotg *dwc2 = dev_get_drvdata(dev); +@@ -655,6 +710,9 @@ static int __maybe_unused dwc2_suspend(struct device *dev) + if (!dwc2->ll_hw_enabled) + return 0; + ++ if (dwc2_keep_gadget_for_s2idle(dwc2)) ++ return 0; ++ + is_device_mode = dwc2_is_device_mode(dwc2); + if (is_device_mode) + dwc2_hsotg_suspend(dwc2); +@@ -735,6 +793,11 @@ static int __maybe_unused dwc2_resume(struct device *dev) + if (!dwc2->ll_hw_enabled) + return 0; + ++ if (dwc2->kept_for_s2idle) { ++ dwc2->kept_for_s2idle = 0; ++ return 0; ++ } ++ + if (dwc2->phy_off_for_suspend && dwc2->ll_hw_enabled) { + ret = __dwc2_lowlevel_hw_enable(dwc2); + if (ret) diff --git a/kernel/patches/drivers/usb/0015-usb-dwc2-name-the-retained-gadget-s2idle-precheck-re.patch b/kernel/patches/drivers/usb/0015-usb-dwc2-name-the-retained-gadget-s2idle-precheck-re.patch new file mode 100644 index 00000000..d3d63dca --- /dev/null +++ b/kernel/patches/drivers/usb/0015-usb-dwc2-name-the-retained-gadget-s2idle-precheck-re.patch @@ -0,0 +1,35 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 15:26:19 +0100 +Subject: [PATCH] usb: dwc2: name the retained-gadget s2idle precheck refusal + +Log why the retained-gadget precheck refused (not-s2idle or +gadget-not-peripheral, with op_state), only during a suspend attempt. +The decision is unchanged. + +Assisted-by: LLM +--- + drivers/usb/dwc2/platform.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c +index fabee8363..0a2aa1058 100644 +--- a/drivers/usb/dwc2/platform.c ++++ b/drivers/usb/dwc2/platform.c +@@ -689,8 +689,15 @@ static int dwc2_sleep_precheck(struct notifier_block *nb, unsigned long action, + if (device_property_read_bool(dwc2->dev, "apple,retain-gadget-in-s2idle") && + (action != PM_SUSPEND_TO_IDLE || + (dwc2->op_state != OTG_STATE_B_PERIPHERAL && +- dwc2->op_state != OTG_STATE_B_IDLE))) ++ dwc2->op_state != OTG_STATE_B_IDLE))) { ++ /* Only for an attempt: the readiness query polls this. */ ++ if (data == PM_SUSPEND_PRECHECK_ATTEMPT) ++ dev_info(dwc2->dev, "system sleep refused: %s op_state=%d\n", ++ action != PM_SUSPEND_TO_IDLE ? "not-s2idle" : ++ "gadget-not-peripheral", ++ dwc2->op_state); + return notifier_from_errno(-EBUSY); ++ } + return NOTIFY_OK; + } + diff --git a/kernel/patches/drivers/usb/0016-dt-bindings-usb-Add-the-Apple-S8000-USB-role-switch.patch b/kernel/patches/drivers/usb/0016-dt-bindings-usb-Add-the-Apple-S8000-USB-role-switch.patch new file mode 100644 index 00000000..6deda9d3 --- /dev/null +++ b/kernel/patches/drivers/usb/0016-dt-bindings-usb-Add-the-Apple-S8000-USB-role-switch.patch @@ -0,0 +1,157 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Fri, 30 Oct 2026 16:50:08 +0100 +Subject: [PATCH] dt-bindings: usb: Add the Apple S8000 USB role switch + +The Apple A9 (S8000/S8003) USB complex has a device-only DWC2 and a +separate EHCI + OHCI host pair behind one AUSB PHY. A USB_CTL register +bank selects whether the PHY serves the host pair. On the iPad 5 the +Lightning port reaches the host pair only through this switch. + +Describe the switch: its USB_CTL bank, the shared PHY, the DWC2 whose +gadget it disconnects in the host role, and the EHCI and OHCI child +nodes, which the switch creates only in the host role. A Lightning board +also names the PMIC with the accessory power LDO and the Tristar port +controller, whose ACC pin routing is set up before accessory power is +turned on. + +Assisted-by: LLM +--- + .../usb/apple,s8000-usb-role-switch.yaml | 128 ++++++++++++++++++ + 1 file changed, 128 insertions(+) + create mode 100644 Documentation/devicetree/bindings/usb/apple,s8000-usb-role-switch.yaml + +diff --git a/Documentation/devicetree/bindings/usb/apple,s8000-usb-role-switch.yaml b/Documentation/devicetree/bindings/usb/apple,s8000-usb-role-switch.yaml +new file mode 100644 +index 000000000..3f9ba23b4 +--- /dev/null ++++ b/Documentation/devicetree/bindings/usb/apple,s8000-usb-role-switch.yaml +@@ -0,0 +1,128 @@ ++# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/usb/apple,s8000-usb-role-switch.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: Apple S8000 USB role switch ++ ++maintainers: ++ - Berkay Kösebay ++ ++description: | ++ The Apple A9 (S8000/S8003) USB complex pairs a device-only DWC2 controller ++ with a separate EHCI + OHCI host pair. All three share one AUSB PHY, and ++ the USB_CTL register bank selects whether the PHY serves the host pair. ++ The switch owns USB_CTL and the host controllers, which are its child ++ nodes and exist only while the port is in the host role. Device mode is ++ the default. The switch sits in the USB complex node, which owns the ++ complex's DMA remap registers. ++ ++ On a Lightning port, the host role for an accessory such as the camera ++ adapter also needs accessory power from the PMIC and the ACC pin routing ++ of the Tristar port controller. ++ ++properties: ++ compatible: ++ const: apple,s8000-usb-role-switch ++ ++ reg: ++ items: ++ - description: USB_CTL register bank ++ ++ "#address-cells": ++ const: 2 ++ ++ "#size-cells": ++ const: 2 ++ ++ ranges: true ++ ++ dma-ranges: true ++ ++ nonposted-mmio: ++ $ref: /schemas/types.yaml#/definitions/flag ++ description: The host controller registers need non-posted accesses. ++ ++ phys: ++ maxItems: 1 ++ ++ phy-names: ++ const: usb2-phy ++ ++ apple,usb-device: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: ++ The DWC2 device controller that shares the PHY. Its gadget is ++ disconnected while the port is a host. ++ ++ apple,pmic: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: ++ PMIC with the Lightning accessory power LDO and its switch. Without it ++ the host role does not power accessories. ++ ++ apple,tristar: ++ $ref: /schemas/types.yaml#/definitions/phandle ++ description: ++ Tristar (CBTL1610) Lightning port controller whose ACC pin routing is ++ set up before accessory power is turned on. Required for accessory ++ power. ++ ++patternProperties: ++ "^usb@[0-9a-f]+$": ++ type: object ++ description: The EHCI and its OHCI companion. ++ ++required: ++ - compatible ++ - reg ++ - "#address-cells" ++ - "#size-cells" ++ - ranges ++ - phys ++ - phy-names ++ - apple,usb-device ++ ++dependentRequired: ++ apple,pmic: [ "apple,tristar" ] ++ ++additionalProperties: false ++ ++examples: ++ - | ++ #include ++ #include ++ #include ++ ++ bus { ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ++ usb-role-switch@20e0d8000 { ++ compatible = "apple,s8000-usb-role-switch"; ++ reg = <0x2 0x0e0d8000 0x0 0x1000>; ++ #address-cells = <2>; ++ #size-cells = <2>; ++ ranges; ++ dma-ranges; ++ nonposted-mmio; ++ phys = <&ausbphy0 PHY_TYPE_USB2>; ++ phy-names = "usb2-phy"; ++ apple,usb-device = <&usbdev>; ++ apple,pmic = <&pmic>; ++ apple,tristar = <&tristar>; ++ ++ usb@20c200000 { ++ compatible = "apple,s8000-ehci", "generic-ehci"; ++ reg = <0x2 0x0c200000 0x0 0x10000>; ++ interrupts = ; ++ }; ++ ++ usb@20c300000 { ++ compatible = "apple,s8000-ohci", "generic-ohci"; ++ reg = <0x2 0x0c300000 0x0 0x10000>; ++ interrupts = ; ++ }; ++ }; ++ }; diff --git a/kernel/patches/drivers/usb/0017-usb-roles-Add-the-Apple-S8000-Lightning-port-role-sw.patch b/kernel/patches/drivers/usb/0017-usb-roles-Add-the-Apple-S8000-Lightning-port-role-sw.patch new file mode 100644 index 00000000..502ed45e --- /dev/null +++ b/kernel/patches/drivers/usb/0017-usb-roles-Add-the-Apple-S8000-Lightning-port-role-sw.patch @@ -0,0 +1,1128 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Fri, 30 Oct 2026 16:50:08 +0100 +Subject: [PATCH] usb: roles: Add the Apple S8000 Lightning port role switch + +The A9 USB complex pairs a device-only DWC2 with a separate EHCI + OHCI +host pair behind one AUSB PHY. On the iPad 5 the Lightning port is a USB +device by default; the camera adapter needs it to be a host. + +Add a usb_role switch for the port. Device mode stays the default and +needs no register access. Host mode follows the iOS order: soft- +disconnect the gadget (it stays bound, as iOS keeps the device +controller published), make sure the complex's DMA remap words for the +host pair are set, power the PHY, create EHCI and then OHCI from the +switch's child nodes so that their power domains are on, set USB_CTL bit +1 and then host select, and switch the PHY to its host tunables. Leaving +host mode undoes the steps in reverse and reconnects the gadget, which +comes on once the device controller's own role allows it (the Lightning +charger sets that role). + +A host role is entered on request through /sys/class/usb_role and lasts +until the device role is requested. The Lightning accessory policy in +the Tristar charger can claim a host entry for an accessory, through +. For such an entry, and before the +policy's ID-bus identification, the switch powers the accessory as iOS +does: it routes the Tristar ACC pin to the power rail and the data lanes +for USB (ACC_CTRL 0x30 and dxctrl 0xa1 for the adapter in orientation 1, +through an intermediate dxctrl byte when a lane pair changes), then +enables the PMIC's accessory LDO, its switch and its sleep enable. The +Tristar bytes it found are restored when the power goes off. The policy +also samples the Tristar through the switch, so that no role change +overlaps the sample. System sleep is refused in the host role. + +The camera adapter sequence was exercised on an iPad 5 with an earlier +kernel. This shared-driver integration still needs hardware validation. + +Assisted-by: LLM +--- + drivers/usb/roles/Kconfig | 15 + + drivers/usb/roles/Makefile | 1 + + drivers/usb/roles/apple-s8000-tristar-route.h | 87 ++ + drivers/usb/roles/apple-s8000-usb-role.c | 950 ++++++++++++++++++ + 4 files changed, 1053 insertions(+) + create mode 100644 drivers/usb/roles/apple-s8000-tristar-route.h + create mode 100644 drivers/usb/roles/apple-s8000-usb-role.c + +diff --git a/drivers/usb/roles/Kconfig b/drivers/usb/roles/Kconfig +index f8b31aa67..e4e0e23d4 100644 +--- a/drivers/usb/roles/Kconfig ++++ b/drivers/usb/roles/Kconfig +@@ -26,4 +26,19 @@ config USB_ROLES_INTEL_XHCI + To compile the driver as a module, choose M here: the module will + be called intel-xhci-usb-role-switch. + ++config USB_ROLES_APPLE_S8000 ++ tristate "Apple S8000 (A9) Lightning port role switch" ++ depends on ARCH_APPLE || COMPILE_TEST ++ depends on OF && I2C && USB_GADGET ++ select REGMAP ++ help ++ Role switch for the Apple A9 USB complex, which pairs a device-only ++ DWC2 with a separate EHCI + OHCI host pair behind one PHY. Device ++ mode is the default; host mode is entered on request through ++ /sys/class/usb_role or by the Lightning accessory policy for the ++ camera adapter. ++ ++ To compile the driver as a module, choose M here: the module will ++ be called apple-s8000-usb-role. ++ + endif # USB_ROLE_SWITCH +diff --git a/drivers/usb/roles/Makefile b/drivers/usb/roles/Makefile +index 757a7d279..0b0b0968c 100644 +--- a/drivers/usb/roles/Makefile ++++ b/drivers/usb/roles/Makefile +@@ -3,3 +3,4 @@ + obj-$(CONFIG_USB_ROLE_SWITCH) += roles.o + roles-y := class.o + obj-$(CONFIG_USB_ROLES_INTEL_XHCI) += intel-xhci-usb-role-switch.o ++obj-$(CONFIG_USB_ROLES_APPLE_S8000) += apple-s8000-usb-role.o +diff --git a/drivers/usb/roles/apple-s8000-tristar-route.h b/drivers/usb/roles/apple-s8000-tristar-route.h +new file mode 100644 +index 000000000..bb477dc1d +--- /dev/null ++++ b/drivers/usb/roles/apple-s8000-tristar-route.h +@@ -0,0 +1,87 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Tristar (CBTL1610) transport route and ACC pin configuration for the ++ * Lightning camera adapter, digital ID 11 f0 00 00 00 00 (USB 2, UART 1, ++ * power f). Pure arithmetic, no I/O. ++ * ++ * As iOS does for this ID: ++ * - The ID decodes to the transports {usb0, urt0} and the ACC roles ++ * {idbs, apwr}. Both pairs swap when the plug is in orientation 2. ++ * - Orientation (status register 0x0c): 0 unless bit 7, else 1 + bit 6. ++ * - dxctrl (register 0x01): the low nibble routes the first lane pair, the ++ * high nibble the second. usb0 is 1, urt0 is 2 (low) or 0x20 (high). ++ * Bit 3 / bit 7 mark a pair that differs from the default, which is USB ++ * on the oriented pair and the other pair open. The current byte is ++ * first normalised to that default; when a pair changes, an ++ * intermediate byte with that pair's override bit is written before ++ * the final one. ++ * - ACC_CTRL (register 0x02): ACC1 nibble idbs 0, urt1 1, jtag 2, apwr 3; ++ * ACC2 the same, shifted left by 4. ++ * ++ * For the adapter in orientation 1 this gives ACC_CTRL 0x30 and dxctrl 0xa1. ++ */ ++#ifndef APPLE_S8000_TRISTAR_ROUTE_H ++#define APPLE_S8000_TRISTAR_ROUTE_H ++ ++#include ++ ++#define TRISTAR_ROUTE_STATUS_PRESENT 0x80 ++#define TRISTAR_ROUTE_STATUS_ORIENT2 0x40 ++ ++#define TRISTAR_ROUTE_ACC_O1 0x30 /* ACC1 idbs, ACC2 apwr */ ++#define TRISTAR_ROUTE_ACC_O2 0x03 /* ACC1 apwr, ACC2 idbs */ ++#define TRISTAR_ROUTE_DX_O1 0xa1 /* usb0 | urt0 << 4 | override of pair 2 */ ++#define TRISTAR_ROUTE_DX_O2 0x1a /* urt0 | usb0 << 4 | override of pair 1 */ ++ ++/** ++ * struct tristar_route_plan - Tristar writes for the camera adapter ++ * @orientation: 0 without a plug, else 1 or 2 ++ * @acc: ACC_CTRL value (register 0x02) ++ * @dx_first: intermediate dxctrl value (register 0x01) ++ * @dx_target: final dxctrl value ++ * @dx_write: register 0x01 differs from @dx_target ++ * @dx_two_steps: @dx_first is written before @dx_target ++ */ ++struct tristar_route_plan { ++ unsigned int orientation; ++ u8 acc; ++ u8 dx_first; ++ u8 dx_target; ++ bool dx_write; ++ bool dx_two_steps; ++}; ++ ++static inline unsigned int tristar_route_orientation(u8 status) ++{ ++ if (!(status & TRISTAR_ROUTE_STATUS_PRESENT)) ++ return 0; ++ return status & TRISTAR_ROUTE_STATUS_ORIENT2 ? 2 : 1; ++} ++ ++/* @status is register 0x0c and @dx_cur register 0x01. False without a plug. */ ++static inline bool tristar_route_plan_for(u8 status, u8 dx_cur, ++ struct tristar_route_plan *plan) ++{ ++ u8 norm = dx_cur, lo, hi; ++ ++ plan->orientation = tristar_route_orientation(status); ++ if (!plan->orientation) ++ return false; ++ plan->acc = plan->orientation == 1 ? TRISTAR_ROUTE_ACC_O1 : TRISTAR_ROUTE_ACC_O2; ++ plan->dx_target = plan->orientation == 1 ? TRISTAR_ROUTE_DX_O1 : TRISTAR_ROUTE_DX_O2; ++ plan->dx_write = dx_cur != plan->dx_target; ++ /* A pair without its override bit is at the default: USB on the oriented pair. */ ++ if (plan->orientation == 1) { ++ if (!(norm & 0x08)) ++ norm = (norm & ~0x07) | 0x01; ++ } else if (!(norm & 0x80)) { ++ norm = (norm & ~0x70) | 0x10; ++ } ++ lo = (plan->dx_target ^ norm) & 0x07 ? 0x08 : plan->dx_target & 0x0f; ++ hi = (plan->dx_target ^ norm) & 0x70 ? 0x80 : plan->dx_target & 0xf0; ++ plan->dx_first = lo | hi; ++ plan->dx_two_steps = plan->dx_write && plan->dx_first != plan->dx_target; ++ return true; ++} ++ ++#endif /* APPLE_S8000_TRISTAR_ROUTE_H */ +diff --git a/drivers/usb/roles/apple-s8000-usb-role.c b/drivers/usb/roles/apple-s8000-usb-role.c +new file mode 100644 +index 000000000..a5331cf3f +--- /dev/null ++++ b/drivers/usb/roles/apple-s8000-usb-role.c +@@ -0,0 +1,950 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Apple S8000 (A9) Lightning port USB role switch. ++ * ++ * The A9 USB complex has a device-only DWC2 and a separate EHCI + OHCI host ++ * pair, all behind one AUSB PHY. For a USB device on the port, iOS keeps the ++ * device controller and publishes EHCI and OHCI next to it. The USB_CTL ++ * register bank moves the PHY to the host pair with bit 0; bit 1 is set ++ * while the PHY is powered for the host pair. ++ * ++ * Device mode is the default. Host mode is entered on request through the ++ * usb_role class and lasts until the device role is requested again. The ++ * Lightning accessory policy may claim a host entry for an accessory (the ++ * camera adapter, see ): that entry also ++ * powers the accessory through the PMIC's accessory LDO (LDO5) and its ++ * switch, after routing the Tristar ACC pin to the power rail, as iOS does. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-s8000-tristar-route.h" ++ ++/* USB_CTL */ ++#define USB_CTL_HOST 0x00 ++#define USB_CTL_HOST_EN BIT(0) ++/* Set while the PHY is powered for the host pair. */ ++#define USB_CTL_PHY_ON BIT(1) ++ ++/* ++ * USB complex DMA remap words, owned and programmed at probe by the parent ++ * usb-complex driver. ++ */ ++#define REMAP_EHCI0 0x3c ++#define REMAP_OHCI0 0x5c ++#define REMAP_EN BIT(8) ++#define REMAP_UPPER GENMASK(3, 0) ++/* DRAM is at 0x8_0000_0000; the bus dma-ranges hand out 32-bit addresses. */ ++#define REMAP_TO_DRAM (REMAP_EN | FIELD_PREP(REMAP_UPPER, 0x8)) ++ ++/* D2365 accessory LDO (LDO5) */ ++#define D2365_LDO5_ENABLE 0x0310 ++#define D2365_LDO5_EN BIT(0) ++/* Keeps the rail up while the system sleeps. */ ++#define D2365_LDO5_SLEEP_EN GENMASK(3, 1) ++#define D2365_LDO5_SWITCH 0x2603 ++ ++/* CBTL1610 (Tristar) */ ++#define TRISTAR_DX_CONTROL 0x01 ++#define TRISTAR_ACC_CTRL 0x02 ++#define TRISTAR_STATUS 0x0c ++#define TRISTAR_REVISION 0x0f ++/* The revision whose ACC and route encoding is implemented here. */ ++#define TRISTAR_CBTL1610_REVISION 0x8a ++#define TRISTAR_ROUTE_SETTLE_MS 5 ++ ++/* ++ * The port has one switch. The accessory API checks a switch handed to it ++ * against this instance, under the lock, so that an unbound switch is never ++ * dereferenced. ++ */ ++static DEFINE_MUTEX(apple_usb_role_instance_lock); ++static struct apple_usb_role *apple_usb_role_instance; ++ ++struct apple_usb_role { ++ struct device *dev; ++ void __iomem *usb_ctl; ++ /* Shared, not claimed: the usb-complex driver owns the remap words. */ ++ void __iomem *remap; ++ ++ struct phy *phy; ++ struct platform_device *udc; ++ struct device_node *ehci_np; ++ struct device_node *ohci_np; ++ struct regmap *pmic; ++ struct i2c_client *tristar; ++ ++ struct usb_role_switch *sw; ++ ++ /* Serializes role changes, accessory power and the state below. */ ++ struct mutex lock; ++ enum usb_role role; ++ struct device *gadget; ++ struct platform_device *ehci; ++ struct platform_device *ohci; ++ bool phy_held; ++ bool ctl_host; ++ bool ctl_phy_on; ++ /* LDO5 and its switch were (perhaps partly) turned on. */ ++ bool acc_power; ++ bool sleep_power_on; ++ /* The next host entry is the accessory policy's. */ ++ bool accessory_claim; ++ /* Accessory power was applied before the policy's ID-bus reads. */ ++ bool accessory_power_prepared; ++ /* Tristar 0x01/0x02 were changed for the adapter; the saved bytes restore them. */ ++ bool acc_applied; ++ u8 acc_saved; ++ u8 dx_saved; ++ /* The current host role is the accessory policy's. */ ++ bool accessory; ++ bool dying; ++ bool monitor_fault; ++}; ++ ++static void apple_usb_role_iounmap(void *addr) ++{ ++ iounmap(addr); ++} ++ ++/* Map a register range without claiming it: another driver claims it. */ ++static void __iomem *apple_usb_role_view(struct apple_usb_role *ur, ++ struct device_node *np, int index) ++{ ++ void __iomem *base = of_iomap(np, index); ++ ++ if (!base || devm_add_action_or_reset(ur->dev, apple_usb_role_iounmap, ++ (void __force *)base)) ++ return NULL; ++ return base; ++} ++ ++/* -------------------------------------------------------------------- */ ++/* Accessory power */ ++ ++static int apple_usb_role_tristar_read(struct apple_usb_role *ur, u8 reg) ++{ ++ struct i2c_client *client = ur->tristar; ++ union i2c_smbus_data data; ++ int ret; ++ ++ /* Register selector and STOP, then a separate read. */ ++ i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_WRITE, reg, I2C_SMBUS_BYTE, NULL); ++ if (!ret) ++ ret = __i2c_smbus_xfer(client->adapter, client->addr, client->flags, ++ I2C_SMBUS_READ, 0, I2C_SMBUS_BYTE, &data); ++ i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); ++ return ret ?: data.byte; ++} ++ ++static int apple_usb_role_tristar_write(struct apple_usb_role *ur, u8 reg, u8 val) ++{ ++ int ret; ++ ++ ret = i2c_smbus_write_byte_data(ur->tristar, reg, val); ++ if (ret) ++ return ret; ++ msleep(TRISTAR_ROUTE_SETTLE_MS); ++ ret = apple_usb_role_tristar_read(ur, reg); ++ if (ret < 0) ++ return ret; ++ return ret == val ? 0 : -EIO; ++} ++ ++static int apple_usb_role_ldo5(struct apple_usb_role *ur, bool on) ++{ ++ return regmap_update_bits(ur->pmic, D2365_LDO5_ENABLE, D2365_LDO5_EN, ++ on ? D2365_LDO5_EN : 0); ++} ++ ++static int apple_usb_role_ldo5_switch(struct apple_usb_role *ur, bool on) ++{ ++ /* iOS writes the whole register: 1 or 0. */ ++ return regmap_write(ur->pmic, D2365_LDO5_SWITCH, on); ++} ++ ++static int apple_usb_role_ldo5_sleep(struct apple_usb_role *ur, bool on) ++{ ++ return regmap_update_bits(ur->pmic, D2365_LDO5_ENABLE, ++ D2365_LDO5_SLEEP_EN, ++ on ? D2365_LDO5_SLEEP_EN : 0); ++} ++ ++/* ++ * iOS applies the ACC and route configuration when the accessory attaches ++ * and again before it enables accessory power, so that the ACC pin carries ++ * the power rail (apwr) before LDO5 comes on. The values and the dxctrl ++ * write order are in apple-s8000-tristar-route.h. ++ */ ++static int apple_usb_role_acc_config(struct apple_usb_role *ur) ++{ ++ struct tristar_route_plan plan; ++ int ret, status; ++ ++ if (ur->acc_applied) ++ return 0; ++ if (!ur->tristar) ++ return -ENODEV; ++ ret = apple_usb_role_tristar_read(ur, TRISTAR_REVISION); ++ if (ret < 0) ++ return ret; ++ if (ret != TRISTAR_CBTL1610_REVISION) ++ return -ENODEV; ++ status = apple_usb_role_tristar_read(ur, TRISTAR_STATUS); ++ if (status < 0) ++ return status; ++ ret = apple_usb_role_tristar_read(ur, TRISTAR_DX_CONTROL); ++ if (ret < 0) ++ return ret; ++ ur->dx_saved = ret; ++ ret = apple_usb_role_tristar_read(ur, TRISTAR_ACC_CTRL); ++ if (ret < 0) ++ return ret; ++ ur->acc_saved = ret; ++ if (!tristar_route_plan_for(status, ur->dx_saved, &plan)) { ++ dev_warn(ur->dev, "Tristar reports no plug (status %#04x)\n", status); ++ return -ENOLINK; ++ } ++ /* From the first write on, teardown restores both registers. */ ++ ur->acc_applied = true; ++ ret = 0; ++ if (plan.dx_write) { ++ if (plan.dx_two_steps) ++ ret = apple_usb_role_tristar_write(ur, TRISTAR_DX_CONTROL, plan.dx_first); ++ if (!ret) ++ ret = apple_usb_role_tristar_write(ur, TRISTAR_DX_CONTROL, plan.dx_target); ++ if (ret) ++ return ret; ++ } ++ if (ur->acc_saved != plan.acc) ++ ret = apple_usb_role_tristar_write(ur, TRISTAR_ACC_CTRL, plan.acc); ++ dev_dbg(ur->dev, "orientation %u: dxctrl %#04x -> %#04x, ACC_CTRL %#04x -> %#04x (%d)\n", ++ plan.orientation, ur->dx_saved, plan.dx_target, ur->acc_saved, ++ plan.acc, ret); ++ return ret; ++} ++ ++static void apple_usb_role_acc_restore_reg(struct apple_usb_role *ur, u8 reg, u8 saved) ++{ ++ int ret = apple_usb_role_tristar_read(ur, reg); ++ ++ if (ret == saved) ++ return; ++ if (ret >= 0) ++ ret = apple_usb_role_tristar_write(ur, reg, saved); ++ if (ret < 0) ++ dev_err(ur->dev, "Tristar register %#04x restore: %d\n", reg, ret); ++} ++ ++static void apple_usb_role_acc_restore(struct apple_usb_role *ur) ++{ ++ if (!ur->acc_applied) ++ return; ++ ur->acc_applied = false; ++ /* Reverse of the apply order: ACC_CTRL, then dxctrl. */ ++ apple_usb_role_acc_restore_reg(ur, TRISTAR_ACC_CTRL, ur->acc_saved); ++ apple_usb_role_acc_restore_reg(ur, TRISTAR_DX_CONTROL, ur->dx_saved); ++} ++ ++static void apple_usb_role_acc_power_off(struct apple_usb_role *ur) ++{ ++ int ret; ++ ++ if (ur->acc_power) { ++ if (ur->sleep_power_on) { ++ ret = apple_usb_role_ldo5_sleep(ur, false); ++ if (ret) ++ dev_err(ur->dev, "LDO5 sleep power off: %d\n", ret); ++ ur->sleep_power_on = false; ++ } ++ ret = apple_usb_role_ldo5_switch(ur, false); ++ if (ret) ++ dev_err(ur->dev, "LDO5 switch off: %d\n", ret); ++ ret = apple_usb_role_ldo5(ur, false); ++ if (ret) ++ dev_err(ur->dev, "LDO5 off: %d\n", ret); ++ ur->acc_power = false; ++ } ++ apple_usb_role_acc_restore(ur); ++} ++ ++/* ++ * The power iOS gives the camera adapter: ACC pin routing first, then the ++ * rail, its switch and its sleep enable, before the hub enumerates. Without ++ * a PMIC there is nothing to switch. ++ */ ++static int apple_usb_role_acc_power_on(struct apple_usb_role *ur) ++{ ++ int ret; ++ ++ if (!ur->pmic) ++ return 0; ++ ret = apple_usb_role_acc_config(ur); ++ if (ret) { ++ dev_err(ur->dev, "Tristar ACC configuration: %d\n", ret); ++ return ret; ++ } ++ ur->acc_power = true; ++ ret = apple_usb_role_ldo5(ur, true); ++ if (!ret) ++ ret = apple_usb_role_ldo5_switch(ur, true); ++ if (!ret) { ++ ret = apple_usb_role_ldo5_sleep(ur, true); ++ if (!ret) ++ ur->sleep_power_on = true; ++ } ++ if (ret) ++ dev_err(ur->dev, "accessory power (LDO5): %d\n", ret); ++ return ret; ++} ++ ++/* -------------------------------------------------------------------- */ ++/* Role changes */ ++ ++static int apple_usb_role_is_gadget(struct device *dev, const void *data) ++{ ++ return dev->bus && !strcmp(dev->bus->name, "gadget"); ++} ++ ++/* ++ * iOS leaves the device controller published while its host controllers ++ * are active. Keep the gadget driver bound and drop only its pull-up. ++ */ ++static int apple_usb_role_release_gadget(struct apple_usb_role *ur) ++{ ++ struct device *gadget; ++ int ret; ++ ++ gadget = device_find_child(&ur->udc->dev, NULL, apple_usb_role_is_gadget); ++ if (!gadget) ++ return 0; ++ if (!gadget->driver) { ++ /* Nothing to give back later either. */ ++ put_device(gadget); ++ return 0; ++ } ++ ret = usb_gadget_disconnect(dev_to_usb_gadget(gadget)); ++ if (ret) { ++ dev_err(ur->dev, "gadget soft disconnect: %d\n", ret); ++ put_device(gadget); ++ return ret; ++ } ++ ur->gadget = gadget; ++ return 0; ++} ++ ++static void apple_usb_role_restore_gadget(struct apple_usb_role *ur) ++{ ++ int ret; ++ ++ if (!ur->gadget) ++ return; ++ ret = usb_gadget_connect(dev_to_usb_gadget(ur->gadget)); ++ if (ret) ++ dev_err(ur->dev, "gadget soft reconnect: %d\n", ret); ++ put_device(ur->gadget); ++ ur->gadget = NULL; ++} ++ ++/* Only repairs a value lost since the usb-complex driver's probe. */ ++static void apple_usb_role_remap(struct apple_usb_role *ur, unsigned int offset) ++{ ++ if (readl(ur->remap + offset) != REMAP_TO_DRAM) ++ writel(REMAP_TO_DRAM, ur->remap + offset); ++} ++ ++static struct platform_device *apple_usb_role_add_hc(struct apple_usb_role *ur, ++ struct device_node *np) ++{ ++ struct platform_device *pdev = of_platform_device_create(np, NULL, ur->dev); ++ ++ if (!pdev) { ++ dev_err(ur->dev, "%pOFn: could not create the controller\n", np); ++ return NULL; ++ } ++ /* ehci/ohci-platform prefer asynchronous probing. */ ++ wait_for_device_probe(); ++ if (!pdev->dev.driver) { ++ dev_err(ur->dev, "%s: no driver bound\n", dev_name(&pdev->dev)); ++ of_platform_device_destroy(&pdev->dev, NULL); ++ return NULL; ++ } ++ return pdev; ++} ++ ++static int apple_usb_role_add_hcs(struct apple_usb_role *ur) ++{ ++ /* EHCI before its OHCI companion, as for any companion pair. */ ++ ur->ehci = apple_usb_role_add_hc(ur, ur->ehci_np); ++ if (!ur->ehci) ++ return -ENODEV; ++ ur->ohci = apple_usb_role_add_hc(ur, ur->ohci_np); ++ if (!ur->ohci) ++ dev_warn(ur->dev, "no OHCI: low/full-speed devices will not work\n"); ++ return 0; ++} ++ ++static void apple_usb_role_ctl_clear(struct apple_usb_role *ur) ++{ ++ /* As iOS leaves host mode: host select first, then bit 1. */ ++ if (ur->ctl_host) { ++ writel(readl(ur->usb_ctl + USB_CTL_HOST) & ~USB_CTL_HOST_EN, ++ ur->usb_ctl + USB_CTL_HOST); ++ ur->ctl_host = false; ++ } ++ if (ur->ctl_phy_on) { ++ writel(readl(ur->usb_ctl + USB_CTL_HOST) & ~USB_CTL_PHY_ON, ++ ur->usb_ctl + USB_CTL_HOST); ++ ur->ctl_phy_on = false; ++ } ++} ++ ++/* Undo whatever enter_host() got to, in reverse order. Idempotent. */ ++static void apple_usb_role_host_off(struct apple_usb_role *ur) ++{ ++ int ret; ++ ++ if (ur->ohci) { ++ of_platform_device_destroy(&ur->ohci->dev, NULL); ++ ur->ohci = NULL; ++ } ++ if (ur->ehci) { ++ of_platform_device_destroy(&ur->ehci->dev, NULL); ++ ur->ehci = NULL; ++ } ++ apple_usb_role_ctl_clear(ur); ++ if (ur->phy_held) { ++ ret = phy_set_mode(ur->phy, PHY_MODE_USB_DEVICE); ++ if (ret) ++ dev_err(ur->dev, "PHY device mode: %d\n", ret); ++ phy_power_off(ur->phy); ++ phy_exit(ur->phy); ++ ur->phy_held = false; ++ } ++ apple_usb_role_acc_power_off(ur); ++ ur->accessory_power_prepared = false; ++} ++ ++static int apple_usb_role_enter_host(struct apple_usb_role *ur, bool accessory) ++{ ++ u32 ctl; ++ int ret; ++ ++ ret = apple_usb_role_release_gadget(ur); ++ if (ret) ++ goto fail; ++ ++ if (accessory) { ++ ret = apple_usb_role_acc_power_on(ur); ++ if (ret) ++ goto fail; ++ } ++ ++ apple_usb_role_remap(ur, REMAP_EHCI0); ++ apple_usb_role_remap(ur, REMAP_OHCI0); ++ ++ ret = phy_init(ur->phy); ++ if (ret) ++ goto fail; ++ ret = phy_power_on(ur->phy); ++ if (ret) { ++ phy_exit(ur->phy); ++ goto fail; ++ } ++ ur->phy_held = true; ++ ++ /* ++ * iOS creates EHCI and OHCI before it selects host mode, so their ++ * power domains are on when USB_CTL bit 1 is written: with the domains ++ * still off the hardware discards the bit. ++ */ ++ ret = apple_usb_role_add_hcs(ur); ++ if (ret) ++ goto fail; ++ ctl = readl(ur->usb_ctl + USB_CTL_HOST); ++ if (!(ctl & USB_CTL_PHY_ON)) { ++ writel(ctl | USB_CTL_PHY_ON, ur->usb_ctl + USB_CTL_HOST); ++ ur->ctl_phy_on = true; ++ } ++ ++ /* Host select, then the PHY's host tunables and reset. */ ++ ctl = readl(ur->usb_ctl + USB_CTL_HOST); ++ writel(ctl | USB_CTL_PHY_ON | USB_CTL_HOST_EN, ur->usb_ctl + USB_CTL_HOST); ++ ur->ctl_host = true; ++ ret = phy_set_mode(ur->phy, PHY_MODE_USB_HOST); ++ if (ret) ++ goto fail; ++ ++ /* If something dropped bit 1 on the way, assert it once more. */ ++ ctl = readl(ur->usb_ctl + USB_CTL_HOST); ++ if (!(ctl & USB_CTL_PHY_ON)) ++ writel(ctl | USB_CTL_PHY_ON, ur->usb_ctl + USB_CTL_HOST); ++ ++ dev_dbg(ur->dev, "host mode%s\n", accessory ? " for the Lightning accessory" : ""); ++ return 0; ++ ++fail: ++ apple_usb_role_host_off(ur); ++ apple_usb_role_restore_gadget(ur); ++ return ret; ++} ++ ++static void apple_usb_role_enter_device(struct apple_usb_role *ur) ++{ ++ apple_usb_role_host_off(ur); ++ apple_usb_role_restore_gadget(ur); ++ dev_dbg(ur->dev, "device mode\n"); ++} ++ ++static int apple_usb_role_set(struct usb_role_switch *sw, enum usb_role role) ++{ ++ struct apple_usb_role *ur = usb_role_switch_get_drvdata(sw); ++ int ret = 0; ++ ++ /* There is no idle state: the gadget is the resting role. */ ++ if (role == USB_ROLE_NONE) ++ role = USB_ROLE_DEVICE; ++ ++ mutex_lock(&ur->lock); ++ if (ur->dying) { ++ ret = -ESHUTDOWN; ++ } else if (role == USB_ROLE_HOST && ur->monitor_fault) { ++ ret = -EUCLEAN; ++ } else if (role == ur->role) { ++ /* The accessory takes over a requested host role. */ ++ if (role == USB_ROLE_HOST && ur->accessory_claim) ++ ur->accessory = true; ++ } else if (role == USB_ROLE_HOST) { ++ bool accessory = ur->accessory_claim; ++ ++ ret = apple_usb_role_enter_host(ur, accessory); ++ if (!ret) { ++ WRITE_ONCE(ur->role, USB_ROLE_HOST); ++ ur->accessory = accessory; ++ ur->accessory_power_prepared = false; ++ } ++ } else { ++ apple_usb_role_enter_device(ur); ++ WRITE_ONCE(ur->role, USB_ROLE_DEVICE); ++ ur->accessory = false; ++ } ++ ur->accessory_claim = false; ++ mutex_unlock(&ur->lock); ++ return ret; ++} ++ ++static enum usb_role apple_usb_role_get(struct usb_role_switch *sw) ++{ ++ struct apple_usb_role *ur = usb_role_switch_get_drvdata(sw); ++ ++ return READ_ONCE(ur->role); ++} ++ ++/* Caller holds apple_usb_role_instance_lock, which keeps the instance alive. */ ++static struct apple_usb_role *apple_usb_role_from_switch(struct usb_role_switch *sw) ++{ ++ struct apple_usb_role *ur; ++ ++ lockdep_assert_held(&apple_usb_role_instance_lock); ++ if (IS_ERR_OR_NULL(sw) || !apple_usb_role_instance) ++ return NULL; ++ ur = usb_role_switch_get_drvdata(sw); ++ return ur == apple_usb_role_instance ? ur : NULL; ++} ++ ++/** ++ * apple_s8000_usb_role_claim - mark the next host entry as the accessory's ++ * @sw: the Lightning role switch ++ * @claim: true before usb_role_switch_set_role(USB_ROLE_HOST); false to drop ++ * a claim that was not used ++ * ++ * A claimed host entry powers the accessory. Any device-role request ends ++ * the host role. ++ * ++ * Return: 0, -ESHUTDOWN while the switch goes away, or -ENODEV if @sw is not ++ * this driver's switch. ++ */ ++int apple_s8000_usb_role_claim(struct usb_role_switch *sw, bool claim) ++{ ++ struct apple_usb_role *ur; ++ int ret = 0; ++ ++ mutex_lock(&apple_usb_role_instance_lock); ++ ur = apple_usb_role_from_switch(sw); ++ if (ur) { ++ mutex_lock(&ur->lock); ++ if (claim && ur->dying) ++ ret = -ESHUTDOWN; ++ ur->accessory_claim = claim && !ret; ++ mutex_unlock(&ur->lock); ++ } else { ++ ret = -ENODEV; ++ } ++ mutex_unlock(&apple_usb_role_instance_lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_s8000_usb_role_claim); ++ ++/** ++ * apple_s8000_usb_role_accessory_power - apply or release accessory power ++ * @sw: the Lightning role switch ++ * @on: apply before ID-bus identification, or unwind after a failed sequence ++ * ++ * iOS powers the accessory before it reads the accessory's information or ++ * changes the USB mode. This only sets up the local PMIC and Tristar state; ++ * it sends nothing on the ID bus. A successful host entry takes the power ++ * over, and leaving host mode turns it off. ++ * ++ * Return: 0, -EBUSY outside the device role, -ESHUTDOWN while the switch ++ * goes away, -ENODEV if @sw is not this driver's switch, or the error of a ++ * failed PMIC or Tristar access. ++ */ ++int apple_s8000_usb_role_accessory_power(struct usb_role_switch *sw, bool on) ++{ ++ struct apple_usb_role *ur; ++ int ret = -ENODEV; ++ ++ mutex_lock(&apple_usb_role_instance_lock); ++ ur = apple_usb_role_from_switch(sw); ++ if (ur) { ++ mutex_lock(&ur->lock); ++ if (ur->dying) { ++ ret = -ESHUTDOWN; ++ } else if (ur->role != USB_ROLE_DEVICE) { ++ ret = -EBUSY; ++ } else if (on && !ur->accessory_power_prepared) { ++ ret = apple_usb_role_acc_power_on(ur); ++ if (ret) ++ apple_usb_role_acc_power_off(ur); ++ else ++ ur->accessory_power_prepared = true; ++ } else if (!on && ur->accessory_power_prepared) { ++ apple_usb_role_acc_power_off(ur); ++ ur->accessory_power_prepared = false; ++ ret = 0; ++ } else { ++ ret = 0; ++ } ++ mutex_unlock(&ur->lock); ++ } ++ mutex_unlock(&apple_usb_role_instance_lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_s8000_usb_role_accessory_power); ++ ++/** ++ * apple_s8000_usb_role_accessory - whether the accessory owns the host role ++ * @sw: the Lightning role switch ++ * ++ * Return: true while the port is a host on the accessory policy's request. ++ */ ++bool apple_s8000_usb_role_accessory(struct usb_role_switch *sw) ++{ ++ struct apple_usb_role *ur; ++ bool ret = false; ++ ++ mutex_lock(&apple_usb_role_instance_lock); ++ ur = apple_usb_role_from_switch(sw); ++ if (ur) { ++ mutex_lock(&ur->lock); ++ ret = ur->role == USB_ROLE_HOST && ur->accessory; ++ mutex_unlock(&ur->lock); ++ } ++ mutex_unlock(&apple_usb_role_instance_lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_s8000_usb_role_accessory); ++ ++/** ++ * apple_s8000_usb_role_monitor - serialize a Tristar sample with role changes ++ * @sw: the Lightning role switch ++ * @sample: sleepable callback; must not query or change the USB role ++ * @data: callback argument ++ * ++ * Lock order: instance, role, then the caller's locks taken in @sample. The ++ * callback must restore DX_CONTROL before it returns. If it returns -EUCLEAN ++ * (route left broken), host entry is refused until a later callback succeeds. ++ * No role change can overlap the callback. ++ * ++ * Return: the callback's result, -EBUSY outside the device role, or -ENODEV ++ * if @sw is not this driver's switch. ++ */ ++int apple_s8000_usb_role_monitor(struct usb_role_switch *sw, ++ int (*sample)(void *data), void *data) ++{ ++ struct apple_usb_role *ur; ++ int ret = -ENODEV; ++ ++ mutex_lock(&apple_usb_role_instance_lock); ++ ur = apple_usb_role_from_switch(sw); ++ if (ur) { ++ mutex_lock(&ur->lock); ++ if (ur->dying || ur->role != USB_ROLE_DEVICE) { ++ ret = -EBUSY; ++ } else { ++ ret = sample(data); ++ ur->monitor_fault = ret == -EUCLEAN; ++ } ++ mutex_unlock(&ur->lock); ++ } ++ mutex_unlock(&apple_usb_role_instance_lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_s8000_usb_role_monitor); ++ ++/* -------------------------------------------------------------------- */ ++/* Probe */ ++ ++static void apple_usb_role_put_device(void *data) ++{ ++ put_device(data); ++} ++ ++static void apple_usb_role_put_node(void *data) ++{ ++ of_node_put(data); ++} ++ ++static void apple_usb_role_unregister(void *data) ++{ ++ usb_role_switch_unregister(data); ++} ++ ++static int apple_usb_role_get_pmic(struct apple_usb_role *ur) ++{ ++ struct device_node *np; ++ struct i2c_client *client; ++ int ret; ++ ++ np = of_parse_phandle(ur->dev->of_node, "apple,pmic", 0); ++ if (!np) ++ return 0; ++ client = of_find_i2c_device_by_node(np); ++ of_node_put(np); ++ if (!client) ++ return -EPROBE_DEFER; ++ ret = devm_add_action_or_reset(ur->dev, apple_usb_role_put_device, &client->dev); ++ if (ret) ++ return ret; ++ ur->pmic = dev_get_regmap(&client->dev, NULL); ++ if (!ur->pmic) ++ return -EPROBE_DEFER; ++ if (!device_link_add(ur->dev, &client->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) ++ return -EINVAL; ++ return 0; ++} ++ ++static int apple_usb_role_get_tristar(struct apple_usb_role *ur) ++{ ++ struct device_node *np; ++ ++ np = of_parse_phandle(ur->dev->of_node, "apple,tristar", 0); ++ if (!np) ++ return 0; ++ ur->tristar = of_find_i2c_device_by_node(np); ++ of_node_put(np); ++ if (!ur->tristar) ++ return -EPROBE_DEFER; ++ return devm_add_action_or_reset(ur->dev, apple_usb_role_put_device, ++ &ur->tristar->dev); ++} ++ ++/* The remap words belong to the parent usb-complex, whose driver claims them. */ ++static int apple_usb_role_get_complex(struct apple_usb_role *ur) ++{ ++ struct device_node *complex = of_get_parent(ur->dev->of_node); ++ ++ if (complex && of_device_is_compatible(complex, "apple,s5l8960x-usb-complex")) ++ ur->remap = apple_usb_role_view(ur, complex, 0); ++ of_node_put(complex); ++ return ur->remap ? 0 : -ENODEV; ++} ++ ++static int apple_usb_role_get_udc(struct apple_usb_role *ur) ++{ ++ struct device_node *np; ++ ++ np = of_parse_phandle(ur->dev->of_node, "apple,usb-device", 0); ++ if (!np) ++ return -EINVAL; ++ ur->udc = of_find_device_by_node(np); ++ of_node_put(np); ++ if (!ur->udc) ++ return -EPROBE_DEFER; ++ return devm_add_action_or_reset(ur->dev, apple_usb_role_put_device, &ur->udc->dev); ++} ++ ++static int apple_usb_role_get_hcs(struct apple_usb_role *ur) ++{ ++ struct device_node *np = ur->dev->of_node; ++ int ret; ++ ++ ur->ehci_np = of_get_compatible_child(np, "generic-ehci"); ++ if (!ur->ehci_np) ++ return -ENODEV; ++ ret = devm_add_action_or_reset(ur->dev, apple_usb_role_put_node, ur->ehci_np); ++ if (ret) ++ return ret; ++ ur->ohci_np = of_get_compatible_child(np, "generic-ohci"); ++ if (!ur->ohci_np) ++ return -ENODEV; ++ return devm_add_action_or_reset(ur->dev, apple_usb_role_put_node, ur->ohci_np); ++} ++ ++static void apple_usb_role_teardown(void *data) ++{ ++ struct apple_usb_role *ur = data; ++ ++ mutex_lock(&apple_usb_role_instance_lock); ++ if (apple_usb_role_instance == ur) ++ apple_usb_role_instance = NULL; ++ mutex_unlock(&apple_usb_role_instance_lock); ++ ++ mutex_lock(&ur->lock); ++ ur->dying = true; ++ if (ur->role == USB_ROLE_HOST) { ++ apple_usb_role_enter_device(ur); ++ WRITE_ONCE(ur->role, USB_ROLE_DEVICE); ++ } ++ mutex_unlock(&ur->lock); ++} ++ ++static int apple_usb_role_probe(struct platform_device *pdev) ++{ ++ struct usb_role_switch_desc desc = {}; ++ struct device *dev = &pdev->dev; ++ struct apple_usb_role *ur; ++ int ret; ++ ++ ur = devm_kzalloc(dev, sizeof(*ur), GFP_KERNEL); ++ if (!ur) ++ return -ENOMEM; ++ ur->dev = dev; ++ ur->role = USB_ROLE_DEVICE; ++ platform_set_drvdata(pdev, ur); ++ ret = devm_mutex_init(dev, &ur->lock); ++ if (ret) ++ return ret; ++ ++ ur->usb_ctl = devm_platform_ioremap_resource(pdev, 0); ++ if (IS_ERR(ur->usb_ctl)) ++ return PTR_ERR(ur->usb_ctl); ++ ret = apple_usb_role_get_complex(ur); ++ if (ret) ++ return dev_err_probe(dev, ret, "usb-complex registers\n"); ++ ++ ur->phy = devm_phy_get(dev, "usb2-phy"); ++ if (IS_ERR(ur->phy)) ++ return dev_err_probe(dev, PTR_ERR(ur->phy), "PHY\n"); ++ ++ ret = apple_usb_role_get_udc(ur); ++ if (ret) ++ return dev_err_probe(dev, ret, "device controller\n"); ++ ret = apple_usb_role_get_hcs(ur); ++ if (ret) ++ return dev_err_probe(dev, ret, "host controller nodes\n"); ++ ret = apple_usb_role_get_pmic(ur); ++ if (ret) ++ return dev_err_probe(dev, ret, "PMIC\n"); ++ ret = apple_usb_role_get_tristar(ur); ++ if (ret) ++ return dev_err_probe(dev, ret, "Tristar\n"); ++ ++ /* No register access here: device mode must work as without the switch. */ ++ desc.fwnode = dev_fwnode(dev); ++ desc.name = "lightning"; ++ desc.set = apple_usb_role_set; ++ desc.get = apple_usb_role_get; ++ desc.allow_userspace_control = true; ++ desc.driver_data = ur; ++ ur->sw = usb_role_switch_register(dev, &desc); ++ if (IS_ERR(ur->sw)) ++ return dev_err_probe(dev, PTR_ERR(ur->sw), "role switch\n"); ++ ret = devm_add_action_or_reset(dev, apple_usb_role_unregister, ur->sw); ++ if (ret) ++ return ret; ++ /* Runs before the switch goes away: no role change can race it. */ ++ ret = devm_add_action_or_reset(dev, apple_usb_role_teardown, ur); ++ if (ret) ++ return ret; ++ ++ mutex_lock(&apple_usb_role_instance_lock); ++ apple_usb_role_instance = ur; ++ mutex_unlock(&apple_usb_role_instance_lock); ++ dev_info(dev, "Lightning port in device role%s\n", ++ ur->pmic ? ", accessory power available" : ""); ++ return 0; ++} ++ ++/* ++ * The accessory rail and the Tristar mux keep their state across an SoC ++ * reset, and the boot loader brings the port up as a device again: put the ++ * shared hardware back. The host controllers, children of this device, are ++ * already shut down. ++ */ ++static void apple_usb_role_shutdown(struct platform_device *pdev) ++{ ++ struct apple_usb_role *ur = platform_get_drvdata(pdev); ++ ++ mutex_lock(&ur->lock); ++ ur->dying = true; ++ apple_usb_role_ctl_clear(ur); ++ if (ur->phy_held) ++ phy_set_mode(ur->phy, PHY_MODE_USB_DEVICE); ++ apple_usb_role_acc_power_off(ur); ++ mutex_unlock(&ur->lock); ++} ++ ++static int apple_usb_role_prepare(struct device *dev) ++{ ++ struct apple_usb_role *ur = dev_get_drvdata(dev); ++ ++ /* The role cannot change mid-suspend. */ ++ if (READ_ONCE(ur->role) == USB_ROLE_HOST) { ++ dev_warn(dev, "host mode: return to device mode before suspending\n"); ++ return -EBUSY; ++ } ++ return 0; ++} ++ ++static const struct dev_pm_ops apple_usb_role_pm_ops = { ++ .prepare = pm_sleep_ptr(apple_usb_role_prepare), ++}; ++ ++static const struct of_device_id apple_usb_role_of_match[] = { ++ { .compatible = "apple,s8000-usb-role-switch" }, ++ { } ++}; ++MODULE_DEVICE_TABLE(of, apple_usb_role_of_match); ++ ++static struct platform_driver apple_usb_role_driver = { ++ .probe = apple_usb_role_probe, ++ .shutdown = apple_usb_role_shutdown, ++ .driver = { ++ .name = "apple-s8000-usb-role", ++ .of_match_table = apple_usb_role_of_match, ++ .pm = pm_ptr(&apple_usb_role_pm_ops), ++ }, ++}; ++module_platform_driver(apple_usb_role_driver); ++ ++MODULE_DESCRIPTION("Apple S8000 Lightning port USB role switch"); ++MODULE_LICENSE("GPL"); diff --git a/kernel/patches/drivers/usb/0018-usb-roles-apple-s8000-bound-manual-host-selection.patch b/kernel/patches/drivers/usb/0018-usb-roles-apple-s8000-bound-manual-host-selection.patch new file mode 100644 index 00000000..5642353d --- /dev/null +++ b/kernel/patches/drivers/usb/0018-usb-roles-apple-s8000-bound-manual-host-selection.patch @@ -0,0 +1,112 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 02:11:49 +0100 +Subject: [PATCH] usb: roles: apple-s8000: bound manual host selection + +Restore the validated 60-second manual-host timeout. Accessory-owned +host sessions cancel the timer and keep their existing lifetime. Cancel +work outside the role lock during teardown and shutdown. + +Assisted-by: LLM +--- + drivers/usb/roles/apple-s8000-usb-role.c | 28 ++++++++++++++++++++++-- + 1 file changed, 26 insertions(+), 2 deletions(-) + +diff --git a/drivers/usb/roles/apple-s8000-usb-role.c b/drivers/usb/roles/apple-s8000-usb-role.c +index a5331cf3f..6bfad077a 100644 +--- a/drivers/usb/roles/apple-s8000-usb-role.c ++++ b/drivers/usb/roles/apple-s8000-usb-role.c +@@ -9,7 +9,7 @@ + * while the PHY is powered for the host pair. + * + * Device mode is the default. Host mode is entered on request through the +- * usb_role class and lasts until the device role is requested again. The ++ * usb_role class and reverts after 60 seconds unless claimed by an accessory. The + * Lightning accessory policy may claim a host entry for an accessory (the + * camera adapter, see ): that entry also + * powers the accessory through the PMIC's accessory LDO (LDO5) and its +@@ -33,6 +33,7 @@ + #include + #include + #include ++#include + + #include "apple-s8000-tristar-route.h" + +@@ -94,6 +95,7 @@ struct apple_usb_role { + + /* Serializes role changes, accessory power and the state below. */ + struct mutex lock; ++ struct delayed_work revert_work; + enum usb_role role; + struct device *gadget; + struct platform_device *ehci; +@@ -514,6 +516,18 @@ static void apple_usb_role_enter_device(struct apple_usb_role *ur) + dev_dbg(ur->dev, "device mode\n"); + } + ++/* Manual host entry must not strand the stable device link indefinitely. */ ++static void apple_usb_role_revert(struct work_struct *work) ++{ ++ struct apple_usb_role *ur = container_of(to_delayed_work(work), ++ struct apple_usb_role, revert_work); ++ ++ if (READ_ONCE(ur->dying) || READ_ONCE(ur->accessory)) ++ return; ++ dev_warn(ur->dev, "manual host timeout: returning to device mode\n"); ++ usb_role_switch_set_role(ur->sw, USB_ROLE_DEVICE); ++} ++ + static int apple_usb_role_set(struct usb_role_switch *sw, enum usb_role role) + { + struct apple_usb_role *ur = usb_role_switch_get_drvdata(sw); +@@ -530,8 +544,12 @@ static int apple_usb_role_set(struct usb_role_switch *sw, enum usb_role role) + ret = -EUCLEAN; + } else if (role == ur->role) { + /* The accessory takes over a requested host role. */ +- if (role == USB_ROLE_HOST && ur->accessory_claim) ++ if (role == USB_ROLE_HOST && ur->accessory_claim) { + ur->accessory = true; ++ cancel_delayed_work(&ur->revert_work); ++ } else if (role == USB_ROLE_HOST && !ur->accessory) { ++ mod_delayed_work(system_wq, &ur->revert_work, 60 * HZ); ++ } + } else if (role == USB_ROLE_HOST) { + bool accessory = ur->accessory_claim; + +@@ -540,8 +558,11 @@ static int apple_usb_role_set(struct usb_role_switch *sw, enum usb_role role) + WRITE_ONCE(ur->role, USB_ROLE_HOST); + ur->accessory = accessory; + ur->accessory_power_prepared = false; ++ if (!accessory) ++ mod_delayed_work(system_wq, &ur->revert_work, 60 * HZ); + } + } else { ++ cancel_delayed_work(&ur->revert_work); + apple_usb_role_enter_device(ur); + WRITE_ONCE(ur->role, USB_ROLE_DEVICE); + ur->accessory = false; +@@ -825,6 +846,7 @@ static void apple_usb_role_teardown(void *data) + WRITE_ONCE(ur->role, USB_ROLE_DEVICE); + } + mutex_unlock(&ur->lock); ++ cancel_delayed_work_sync(&ur->revert_work); + } + + static int apple_usb_role_probe(struct platform_device *pdev) +@@ -837,6 +859,7 @@ static int apple_usb_role_probe(struct platform_device *pdev) + ur = devm_kzalloc(dev, sizeof(*ur), GFP_KERNEL); + if (!ur) + return -ENOMEM; ++ INIT_DELAYED_WORK(&ur->revert_work, apple_usb_role_revert); + ur->dev = dev; + ur->role = USB_ROLE_DEVICE; + platform_set_drvdata(pdev, ur); +@@ -911,6 +934,7 @@ static void apple_usb_role_shutdown(struct platform_device *pdev) + phy_set_mode(ur->phy, PHY_MODE_USB_DEVICE); + apple_usb_role_acc_power_off(ur); + mutex_unlock(&ur->lock); ++ cancel_delayed_work_sync(&ur->revert_work); + } + + static int apple_usb_role_prepare(struct device *dev) diff --git a/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch b/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch new file mode 100644 index 00000000..e166e49d --- /dev/null +++ b/kernel/patches/drivers/watchdog/0001-dt-bindings-watchdog-retain-the-tethered-warm-boot-s.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Sun, 8 Nov 2026 17:15:00 +0100 +Subject: [PATCH] dt-bindings: watchdog: retain the tethered warm boot state + +A tethered kernel inherits initialized hardware rather than power-on +state. Retain the board-selected warm handoff state so probe and +shutdown do not discard resources needed by the incoming kernel. + +Assisted-by: LLM +--- + Documentation/devicetree/bindings/watchdog/apple,wdt.yaml | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +index ca4cf01db..a8179c208 100644 +--- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml ++++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml +@@ -13,6 +13,12 @@ allOf: + - $ref: watchdog.yaml# + + properties: ++ apple,tethered-boot: ++ type: boolean ++ description: | ++ A SoC reset leaves the tethered OS. The restart handler refuses warm ++ and soft reboot modes; cold reboot retains the normal SoC reset. ++ + compatible: + oneOf: + - items: diff --git a/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch b/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch new file mode 100644 index 00000000..a235291f --- /dev/null +++ b/kernel/patches/drivers/watchdog/0002-watchdog-apple-retain-the-tethered-warm-boot-state.patch @@ -0,0 +1,54 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Wed, 11 Nov 2026 23:30:21 +0100 +Subject: [PATCH] watchdog: apple: prioritize cold reset and delegate guarded + warm reset + +Tethered cold reset must beat PSCI priority 129: register at priority +255. On warm/soft modes do not program WD1, but return success so +watchdog_restart_notifier returns NOTIFY_DONE and PSCI can run; an error +would stop the chain. Other devices retain priority 128. + +Assisted-by: LLM +--- + drivers/watchdog/apple_wdt.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index f53395329..04e287fe8 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + + /* +@@ -123,6 +124,12 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + ++ if (of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && ++ (mode == REBOOT_WARM || mode == REBOOT_SOFT)) { ++ pr_emerg("apple-wdt: warm/soft reset delegated to PSCI on tethered boot\n"); ++ return 0; /* Continue the restart chain; NOTIFY_BAD would stop PSCI. */ ++ } ++ + writel_relaxed(APPLE_WDT_CTRL_RESET_EN, wdt->regs + APPLE_WDT_WD1_CTRL); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_BITE_TIME); + writel_relaxed(0, wdt->regs + APPLE_WDT_WD1_CUR_TIME); +@@ -190,7 +197,11 @@ static int apple_wdt_probe(struct platform_device *pdev) + watchdog_init_timeout(&wdt->wdd, 0, dev); + apple_wdt_set_timeout(&wdt->wdd, wdt->wdd.timeout); + watchdog_stop_on_unregister(&wdt->wdd); +- watchdog_set_restart_priority(&wdt->wdd, 128); ++ /* The tethered cold escape must beat PSCI (priority 129). Warm/soft ++ * still refuse above, allowing the PSCI handler to run next. ++ */ ++ watchdog_set_restart_priority(&wdt->wdd, ++ of_property_read_bool(dev->of_node, "apple,tethered-boot") ? 255 : 128); + + return devm_watchdog_register_device(dev, &wdt->wdd); + } diff --git a/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch b/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch new file mode 100644 index 00000000..67e6cb21 --- /dev/null +++ b/kernel/patches/drivers/watchdog/0003-watchdog-apple-initialize-the-parent-before-register.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Thu, 12 Nov 2026 00:43:06 +0100 +Subject: [PATCH] watchdog: apple: initialize the parent before registering + restart + +The tethered restart guard dereferences the parent, but probe never set +it. Initialize it before registration and tolerate a missing parent in +the restart callback so the cold watchdog writes remain reachable. + +Assisted-by: LLM +--- + drivers/watchdog/apple_wdt.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c +index 04e287fe8..6de9f863c 100644 +--- a/drivers/watchdog/apple_wdt.c ++++ b/drivers/watchdog/apple_wdt.c +@@ -124,7 +124,8 @@ static int apple_wdt_restart(struct watchdog_device *wdd, unsigned long mode, + { + struct apple_wdt *wdt = to_apple_wdt(wdd); + +- if (of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && ++ if (wdd->parent && ++ of_property_read_bool(wdd->parent->of_node, "apple,tethered-boot") && + (mode == REBOOT_WARM || mode == REBOOT_SOFT)) { + pr_emerg("apple-wdt: warm/soft reset delegated to PSCI on tethered boot\n"); + return 0; /* Continue the restart chain; NOTIFY_BAD would stop PSCI. */ +@@ -185,6 +186,7 @@ static int apple_wdt_probe(struct platform_device *pdev) + + platform_set_drvdata(pdev, wdt); + ++ wdt->wdd.parent = dev; + wdt->wdd.ops = &apple_wdt_ops; + wdt->wdd.info = &apple_wdt_info; + wdt->wdd.max_hw_heartbeat_ms = U32_MAX / wdt->clk_rate * 1000; diff --git a/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch b/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch new file mode 100644 index 00000000..91be9b20 --- /dev/null +++ b/kernel/patches/drivers/wifi/0008-wifi-brcmfmac-stay-down-after-host-sleep-link-loss.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= +Date: Tue, 10 Nov 2026 16:18:38 +0000 +Subject: [PATCH] wifi: brcmfmac: stay down after host sleep link loss + +When the PCIe link does not come back after system sleep, the host +bridge marks the device disconnected. brcmf_pcie_pm_leave_D3() then went +on to access the BARs, which can stall the CPU. Return -ENODEV and keep +the bus down instead; firmware commands and transmits are then refused +early. + +Assisted-by: LLM +--- + .../net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +index d7c6a402f..a07f7c8ad 100644 +--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c ++++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +@@ -2696,6 +2696,16 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) + devinfo = bus->bus_priv.pcie->devinfo; + brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); + ++ /* ++ * The host bridge cut the device off because its link did not come ++ * back. Any BAR access now could stall the CPU, so stay down; with the ++ * bus down, firmware commands and transmits are refused early. ++ */ ++ if (pci_dev_is_disconnected(devinfo->pdev)) { ++ brcmf_err(bus, "device lost over suspend, staying down\n"); ++ return -ENODEV; ++ } ++ + /* Check if device is still up and running, if so we are ready */ + if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { + brcmf_dbg(PCIE, "Try to wakeup device....\n"); diff --git a/pyproject.toml b/pyproject.toml index f48aa158..ad17df07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ package-dir = {"" = "src"} where = ["src"] [tool.setuptools.package-data] -orchard = ["py.typed", "usb-session"] +orchard = ["py.typed", "usb-session", "usb-identity"] [dependency-groups] dev = ["mypy==1.18.2", "jsonschema>=4.25,<5"] diff --git a/schemas/v1/device.schema.json b/schemas/v1/device.schema.json index 77db1dee..61802b1f 100644 --- a/schemas/v1/device.schema.json +++ b/schemas/v1/device.schema.json @@ -361,6 +361,30 @@ "missing", "none" ] + }, + "video_decode_hevc": { + "enum": [ + "working", + "untested", + "missing", + "none" + ] + }, + "warm_restart": { + "enum": [ + "working", + "untested", + "missing", + "none" + ] + }, + "usb_host": { + "enum": [ + "working", + "untested", + "missing", + "none" + ] } }, "required": [ @@ -388,7 +412,10 @@ "usb_network", "audio_output", "audio_input", - "camera" + "camera", + "video_decode_hevc", + "warm_restart", + "usb_host" ], "additionalProperties": false }, diff --git a/src/orchard/apple/agx_g5p.py b/src/orchard/apple/agx_g5p.py new file mode 100644 index 00000000..c46ca6a7 --- /dev/null +++ b/src/orchard/apple/agx_g5p.py @@ -0,0 +1,19 @@ +"""Extract the unmodified G5P C0 META LDR from the user's prelinked kernelcache.""" +import hashlib +from orchard.apple.prelink import Kernelcache, named + +BUNDLE = 'com.apple.AGXFirmwareKextG5P' +CONST = ('__TEXT', '__const') +SIZE = 119112 +SHA256 = '33ae20cbbe2498e4f119ba4e9a55467e4e0192d4c86d5bf1903adce63ffa5dca' + + +def extract(blob): + kernel = Kernelcache(blob) + section = named(kernel.kext_sections(BUNDLE, 'G5P firmware'), CONST)[CONST] + if section['size'] != SIZE: + raise ValueError('Unsupported G5P C0 firmware size') + data = kernel.read(section['addr'], section['size']) + if hashlib.sha256(data).hexdigest() != SHA256: + raise ValueError('Unsupported G5P C0 firmware digest') + return data diff --git a/src/orchard/apple/linux_usb.py b/src/orchard/apple/linux_usb.py index 6095440c..ca8aae49 100644 --- a/src/orchard/apple/linux_usb.py +++ b/src/orchard/apple/linux_usb.py @@ -30,6 +30,12 @@ def serial_tags(serial): 'pwned': pwned.group(1) if pwned else ('checkra1n' if 'YOLO:checkra1n' in serial else None)} +def stable_id(serial): + """Hashed per-device identity, absent on legacy gadgets.""" + match = re.search(r'\bID:([0-9a-fA-F]{12})\b', serial) + return match.group(1).lower() if match else None + + def devices(root=Path('/sys/bus/usb/devices')): found = [] for node in root.iterdir(): @@ -45,14 +51,20 @@ def devices(root=Path('/sys/bus/usb/devices')): ecid = re.search(r'\bECID:([0-9a-fA-F]+)\b', serial) chip = re.search(r'\bCPID:([0-9a-fA-F]+)\b', serial) board = re.search(r'\bBDID:([0-9a-fA-F]+)\b', serial) + usb_id = stable_id(serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) + # A compatibility PID also belongs to real Apple hardware. Only our + # existing boot/profile identity makes it an Orchard system. + mac_current = (pid == 0x1014 and (boot is not None or usb_id is not None) and + profile is not None and profile.group(1) == 'j71t') ports = {str(Path('/dev') / p.name) for p in node.glob('**/tty/ttyACM*')} if len(ports) > 1: raise ValueError('Ambiguous USB serial ports on a device') product = (node / 'product').read_text().strip() if (node / 'product').exists() else '' - found.append(dict(location=node.name, mode=MODES.get(pid, 'other'), + found.append(dict(location=node.name, mode='system' if mac_current else MODES.get(pid, 'other'), ecid=int(ecid.group(1), 16) if ecid else None, boot_id=str(uuid.UUID(boot.group(1))) if boot else None, + usb_id=usb_id, chip=int(chip.group(1), 16) if chip else None, board=int(board.group(1), 16) if board else None, profile=profile.group(1) if profile else None, product=product, diff --git a/src/orchard/apple/usb.py b/src/orchard/apple/usb.py index 1b1a4bf3..263ef546 100644 --- a/src/orchard/apple/usb.py +++ b/src/orchard/apple/usb.py @@ -5,7 +5,7 @@ import sys import uuid -from .linux_usb import MODES, M1N1, devices as linux_devices, m1n1_ports as linux_m1n1_ports, serial_tags +from .linux_usb import MODES, M1N1, devices as linux_devices, m1n1_ports as linux_m1n1_ports, serial_tags, stable_id from orchard.state import EndpointUnavailable @@ -33,7 +33,12 @@ def devices(): continue serial = node.get('USB Serial Number') or '' boot = re.search(r'\bBOOT:([0-9a-fA-F-]{36})\b', serial) + usb_id = stable_id(serial) profile = re.search(r'\bBOARD:([a-z0-9]+)\b', serial) + # A compatibility PID also belongs to real Apple hardware. Only our + # existing boot/profile identity makes it an Orchard system. + mac_current = (node.get('idProduct') == 0x1014 and (boot is not None or usb_id is not None) and + profile is not None and profile.group(1) == 'j71t') fields = [re.search(r'\b' + label + r':([0-9a-fA-F]+)\b', serial) for label in ('ECID', 'CPID', 'BDID')] descendants = list(node.get('IORegistryEntryChildren', [])) @@ -47,9 +52,10 @@ def devices(): ports.append(port) if len(ports) > 1: raise DeviceError('Ambiguous USB serial ports on the selected device') - found.append({'location': location, 'mode': MODES.get(node.get('idProduct'), 'other'), + found.append({'location': location, 'mode': 'system' if mac_current else MODES.get(node.get('idProduct'), 'other'), 'ecid': int(fields[0].group(1), 16) if fields[0] else None, 'boot_id': str(uuid.UUID(boot.group(1))) if boot else None, + 'usb_id': usb_id, 'chip': int(fields[1].group(1), 16) if fields[1] else None, 'board': int(fields[2].group(1), 16) if fields[2] else None, 'profile': profile.group(1) if profile else None, diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index b39ae7bb..9985c837 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -85,7 +85,7 @@ def tools(hardware=None, cache=None, platform='apple-16k'): return prepare_tools(hardware, cache, device=sorted(boards(platform, hardware))[0]) / 'busybox' -def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES): +def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES, pristine_dtbs=None): """orchard's initrd for installed systems: BusyBox, the storage modules and their dependencies. MODULES is {path: (data, mode)} as module_files gives it. @@ -111,10 +111,20 @@ def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_ entries = {name: (directory, b'') for name in ('bin', 'dev', 'proc', 'sys', 'etc', 'etc/orchard', 'newroot', 'lib', 'lib/modules', 'lib/modules/' + release)} entries['init'] = (stat.S_IFREG | 0o755, (hardware / 'boot/initramfs/root/init').read_bytes()) + entries['etc/orchard/usb-identity'] = (stat.S_IFREG | 0o644, (hardware / 'src/orchard/usb-identity').read_bytes()) entries['bin/busybox'] = (stat.S_IFREG | 0o755, Path(busybox).read_bytes()) entries['etc/orchard/modules'] = (stat.S_IFREG | 0o644, ('\n'.join(loaded) + '\n').encode()) if usb: entries['etc/orchard/usb-modules'] = (stat.S_IFREG | 0o644, ('\n'.join(usb) + '\n').encode()) + if pristine_dtbs: + entries['etc/orchard/dtbs'] = (directory, b'') + mapping = [] + for board, data in sorted(pristine_dtbs.items()): + if not board.replace('-', '').isalnum(): + raise ValueError('Invalid pristine DT board') + entries[f'etc/orchard/dtbs/{board}.dtb'] = (stat.S_IFREG | 0o644, data) + mapping.append(f'apple,{board} {board}.dtb\n') + entries['etc/orchard/pristine-map'] = (stat.S_IFREG | 0o644, ''.join(mapping).encode()) dependency_lines = [] for name in sorted(wanted): target = f'lib/modules/{release}/{name}' @@ -143,6 +153,19 @@ def storage_modules(hardware, platform): return tuple(modules) + +def device_features(hardware, boards): + """Public board data for runtime policy, matched against the live DT compatible.""" + from .. import hardware as catalog + records = [] + for board in sorted(boards): + described = catalog.device(board, root=hardware) + records.append({'compatible': described['device'].get('dt_compatible', []), + 'features': described.get('features', {})}) + return {'usr/lib/orchard/device-features.json': + ((json.dumps(records, sort_keys=True) + '\n').encode(), 0o644)} + + def files(bundle, platform, board_dtbs, *, busybox, hardware=None): """(kernel release, {path: (data, mode)}) of PLATFORM's layer from a staged kernel BUNDLE.""" hardware = Path(hardware or ROOT) @@ -153,14 +176,15 @@ def files(bundle, platform, board_dtbs, *, busybox, hardware=None): base = f'usr/lib/modules/{release}/' result[base + 'vmlinuz'] = ((bundle / described['image']).read_bytes(), 0o644) modules_to_load = storage_modules(hardware, platform) - result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, - storage_modules=modules_to_load), 0o644) for dtb in sorted(set(board_dtbs.values())): path = bundle / 'dtbs' / (dtb + '.dtb') if path.is_symlink() or not path.is_file(): raise ValueError(f'Kernel bundle lacks {path.name}') result[base + 'dtb/' + path.name] = (path.read_bytes(), 0o644) + pristine = {board: result[base + 'dtb/' + dtb + '.dtb'][0] for board, dtb in board_dtbs.items()} + result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, pristine_dtbs=pristine, storage_modules=modules_to_load), 0o644) result.update(regulatory_files()) + result.update(device_features(hardware, board_dtbs)) defaults = hardware / 'system/defaults' for path in sorted(defaults.rglob('*')): if path.is_symlink(): diff --git a/src/orchard/ram_session.py b/src/orchard/ram_session.py index 1d1439ba..26c58889 100644 --- a/src/orchard/ram_session.py +++ b/src/orchard/ram_session.py @@ -18,6 +18,7 @@ import subprocess import sys import time +import uuid from .serial import SerialError, SerialPort from .state import cache_root, private_directory @@ -49,7 +50,9 @@ def identity(): def payload_files(): """RAM-root files that enable the USB session in a payload.""" helper = resources.files('orchard') / 'usb-session' - return {HELPER: Path(str(helper)), AUTHORIZED: identity().with_suffix('.pub')} + return {HELPER: Path(str(helper)), + 'usr/libexec/orchard/usb-identity': Path(str(resources.files('orchard') / 'usb-identity')), + AUTHORIZED: identity().with_suffix('.pub')} def target(): @@ -69,17 +72,20 @@ def target(): def _check(device): + # Stable descriptors identify the unit, not its boot. RAM SSH is pinned to + # the per-boot key read on the open ACM endpoint; a reboot invalidates it. + fields = ('location', 'mode', 'profile', 'usb_id', 'boot_id', 'serial_port') def check(): from .apple import usb - if not any(item['location'] == device['location'] and item.get('boot_id') == device.get('boot_id') + if not any(all(item.get(key) == device.get(key) for key in fields) for item in usb.devices()): raise ValueError('The RAM payload disconnected or rebooted') return check def connect(device=None, user=None, *, host=None, ram=False): - """SSH argv for the running payload (its host key pinned over serial once - per boot) or installed system (as its user, with this computer's key).""" + """SSH argv for the running payload (its boot and host key read over ACM) + or installed system (as its user, with this computer's key).""" if ram and (not host or user): raise ValueError('--ram requires --host and connects as root, not --user') if host: @@ -106,20 +112,33 @@ def connect(device=None, user=None, *, host=None, ram=False): argv = computer_key.ssh_argv(user) wait_authenticated(argv, _check(device), seconds=20) return argv - boot = device.get('boot_id') - if not boot: + if not (device.get('boot_id') or device.get('usb_id')): raise ValueError('The RAM payload has no boot identity; rebuild it with a current orchard') check = _check(device) + # Query every connection: the same stable USB serial can name a new boot. + # ACM supplies both boot ID and trusted host key without changing any USB + # descriptor, and also works with payloads built before stable identities. + with SerialPort(device['serial_port'], check) as serial: + code, text = serial.run('cat /proc/sys/kernel/random/boot_id && ' + 'cat /run/orchard/ssh_host_ed25519_key.pub', timeout=15) + if code: + raise ValueError('No SSH in this payload; its /init must run ' + '/' + HELPER + ' (or build without transport = false)') + lines = text.strip().splitlines() + try: + boot = str(uuid.UUID(lines[0].strip())) + except (ValueError, IndexError) as error: + raise ValueError('Invalid RAM boot identity over USB serial') from error + if device.get('boot_id') and boot != device['boot_id']: + raise ValueError('The RAM payload disconnected or rebooted') + pinned = HOST + ' ' + host_key('\n'.join(lines[1:])) + '\n' hosts = _directory() / f'known_hosts-{boot}' + if hosts.is_file() and hosts.read_text() != pinned: + raise ValueError('The RAM SSH host key changed within the same boot') if not hosts.is_file(): - with SerialPort(device['serial_port'], check) as serial: - code, text = serial.run('cat /run/orchard/ssh_host_ed25519_key.pub', timeout=15) - if code: - raise ValueError('No SSH in this payload; its /init must run ' - '/' + HELPER + ' (or build without transport = false)') for old in _directory().glob('known_hosts-*'): old.unlink() - hosts.write_text(HOST + ' ' + host_key(text) + '\n') + hosts.write_text(pinned) hosts.chmod(0o600) argv = ssh_argv(identity(), hosts, HOST, 22) wait_authenticated(argv, check, seconds=20) diff --git a/src/orchard/recipe.py b/src/orchard/recipe.py index 74b73102..976c6666 100644 --- a/src/orchard/recipe.py +++ b/src/orchard/recipe.py @@ -8,7 +8,7 @@ import json import plistlib -from orchard.apple import agx, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet +from orchard.apple import agx, agx_g5p, bluetool, d5500, sgx, smartio, trx, z2fw, z2packet from orchard.apple.osserialize import apple_xml, binary @@ -43,6 +43,10 @@ def _vxd(step, data, expected): return d5500.extract(data)[0] +def _agx_g5p(step, data, expected): + return agx_g5p.extract(data) + + def _agx(step, data, expected): return agx.extract(data) @@ -100,6 +104,7 @@ def _z2fw_mtfw(step, data, expected): # step@version -> (implementation, {parameter besides 'step' and 'input': type}) STEPS = { + 'agx-g5p-ldr-from-kernelcache@1': (_agx_g5p, {}), 'copy@1': (_copy, {}), 'trx-add-nvram@1': (_trx, {'nvram': str, 'placeholder_mac': str}), 'sgx543-from-kernelcache@1': (_sgx, {'build': str}), diff --git a/src/orchard/service.py b/src/orchard/service.py index 0d5d76c7..0340f16c 100644 --- a/src/orchard/service.py +++ b/src/orchard/service.py @@ -150,7 +150,8 @@ def entry_for(record, devicetree): name = record.get('name', 'Linux') return {'id': re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') or 'linux', 'title': name, # Quiet as the recovery: no console text over the splash; the system maps its console. - 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0', + 'options': record.get('cmdline', 'rootwait') + ' fbcon=map:9 vt.global_cursor_default=0' + ( + ' panic=0' if devicetree == 's8003-j71t.dtb' else ''), 'devicetree': devicetree} diff --git a/src/orchard/usb-identity b/src/orchard/usb-identity new file mode 100644 index 00000000..b6962ec8 --- /dev/null +++ b/src/orchard/usb-identity @@ -0,0 +1,19 @@ +#!/bin/sh +# Stable USB identity, shared by installed-system and RAM gadgets. The boot +# stage copies serial-number from Apple's tree; never publish it in the clear. +set -eu +tree=${ORCHARD_DEVICE_TREE:-/proc/device-tree} +board=$(tr '\000' '\n' < "$tree/compatible" | head -1) +serial= +if [ -r "$tree/serial-number" ]; then + serial=$(tr -d '\000' < "$tree/serial-number") +fi +if [ -n "$serial" ]; then + id=$(printf 'orchard-%s' "$serial" | sha256sum | cut -c1-12) + printf 'BOARD:%s ID:%s\n' "${board#apple,}" "$id" +else + # Keep rescue serial and SSH reachable on trees without a hardware ID. + # This legacy fallback cannot retain the host's accessory approval. + echo 'orchard: no device-tree serial-number; using per-boot USB identity' >&2 + printf 'BOOT:%s BOARD:%s\n' "$(cat /proc/sys/kernel/random/boot_id)" "${board#apple,}" +fi diff --git a/src/orchard/usb-session b/src/orchard/usb-session index aadde6a1..94c3e081 100755 --- a/src/orchard/usb-session +++ b/src/orchard/usb-session @@ -17,7 +17,7 @@ echo 0x05ac > "$g/idVendor" echo 0x4144 > "$g/idProduct" mkdir -p "$g/strings/0x409" "$g/configs/c.1/strings/0x409" echo "Orchard $name" > "$g/strings/0x409/product" -echo "BOOT:$(cat /proc/sys/kernel/random/boot_id)" > "$g/strings/0x409/serialnumber" +sh /usr/libexec/orchard/usb-identity > "$g/strings/0x409/serialnumber" echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" # Descriptor only: not charging capability or PMIC policy. echo 100 > "$g/configs/c.1/MaxPower" diff --git a/system/defaults/usr/lib/orchard/device-feature b/system/defaults/usr/lib/orchard/device-feature new file mode 100755 index 00000000..809146f1 --- /dev/null +++ b/system/defaults/usr/lib/orchard/device-feature @@ -0,0 +1,19 @@ +#!/usr/bin/python3 -I +"""Exit successfully only when the running board declares FEATURE STATE.""" +import json +from pathlib import Path +import sys + +def main(): + if len(sys.argv) != 3: + return 1 + try: + live = Path('/sys/firmware/devicetree/base/compatible').read_bytes().decode().split('\0') + records = json.loads(Path('/usr/lib/orchard/device-features.json').read_text()) + matched = [r for r in records if set(r['compatible']) & set(live)] + return 0 if len(matched) == 1 and matched[0]['features'].get(sys.argv[1]) == sys.argv[2] else 1 + except (OSError, ValueError, KeyError, TypeError): + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/test_agx_g5p.py b/tests/test_agx_g5p.py new file mode 100644 index 00000000..f048ef01 --- /dev/null +++ b/tests/test_agx_g5p.py @@ -0,0 +1,29 @@ +import hashlib +import unittest +from unittest.mock import Mock, patch +from orchard.apple import agx_g5p + +class G5PFirmware(unittest.TestCase): + def test_original_bytes_and_bundle(self): + data = b'synthetic firmware, not Apple data' + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=len(data))] + kernel.read.return_value = data + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), patch.object(agx_g5p, 'SIZE', len(data)), patch.object(agx_g5p, 'SHA256', hashlib.sha256(data).hexdigest()): + self.assertIs(agx_g5p.extract(b'input'), data) + kernel.kext_sections.assert_called_once_with(agx_g5p.BUNDLE, 'G5P firmware') + kernel.read.assert_called_once_with(4096, len(data)) + + def test_wrong_size_is_rejected_before_read(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=1)] + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'size'): + agx_g5p.extract(b'input') + kernel.read.assert_not_called() + + def test_wrong_digest_is_rejected(self): + kernel = Mock() + kernel.kext_sections.return_value = [dict(segment='__TEXT', name='__const', addr=4096, size=agx_g5p.SIZE)] + kernel.read.return_value = bytes(agx_g5p.SIZE) + with patch.object(agx_g5p, 'Kernelcache', return_value=kernel), self.assertRaisesRegex(ValueError, 'digest'): + agx_g5p.extract(b'input') diff --git a/tests/test_d2365_poweroff.py b/tests/test_d2365_poweroff.py index 6e262908..52a3fadd 100644 --- a/tests/test_d2365_poweroff.py +++ b/tests/test_d2365_poweroff.py @@ -192,7 +192,7 @@ def test_both_prepare_and_atomic_final_handlers_are_registered(self): class D2365EventQuiesce(unittest.TestCase): def test_shutdown_drains_irq_and_retry_after_closing_gate(self): - post = patched_source('drivers/gpio/gpio-apple-d2365.c') + post = patched_source('drivers/gpio/gpio-apple-d2365.c', every=True) start = post.index('void apple_d2365_gpio_quiesce(') shutdown = post[start:post.index('\n}', start) + 2] self.assertNotIn('.shutdown = apple_pmic_gpio_shutdown', post) diff --git a/tests/test_identity.py b/tests/test_identity.py index 514c2d9e..5a9524cc 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -1,9 +1,12 @@ """Boot identity publication from bounded ADT fixtures; no device activation.""" import importlib.util +import hashlib import json import os from pathlib import Path +import re import stat +import subprocess import struct import sys import tempfile @@ -107,5 +110,75 @@ def test_gadget_publishes_identity_before_binding_usb(self): self.assertLess(source.index('/serialnumber'),source.index('"$g/UDC"')) +class GadgetIdentity(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.tree = self.root / 'proc/device-tree' + self.tree.mkdir(parents=True) + (self.tree / 'compatible').write_bytes(b'apple,j71t\0apple,s8003\0') + boot = self.root / 'proc/sys/kernel/random/boot_id' + boot.parent.mkdir(parents=True) + boot.write_text(BOOT) + self.script = (ROOT / 'src/orchard/usb-identity').read_text().replace( + '/proc/sys/kernel/random/boot_id', str(boot)) + + def serial(self): + return subprocess.run(['sh', '-ec', self.script], check=True, capture_output=True, text=True, + env={**os.environ, 'ORCHARD_DEVICE_TREE': str(self.tree)}) + + def test_hardware_serial_is_hashed_stable_across_boots_and_distinct_per_unit(self): + number = self.tree / 'serial-number' + number.write_bytes(b'HardwareSerialA\0') + first = self.serial() + expected = hashlib.sha256(b'orchard-HardwareSerialA').hexdigest()[:12] + self.assertEqual(first.stdout.strip(), f'BOARD:j71t ID:{expected}') + self.assertNotIn('HardwareSerialA', first.stdout + first.stderr) + (self.root / 'proc/sys/kernel/random/boot_id').write_text('22222222-2222-4222-8222-222222222222') + self.assertEqual(self.serial().stdout, first.stdout) + number.write_bytes(b'HardwareSerialB\0') + self.assertNotEqual(self.serial().stdout, first.stdout) + + def test_missing_or_empty_serial_preserves_rescue_usb_with_a_warning(self): + for value in (None, b'', b'\0'): + if value is not None: + (self.tree / 'serial-number').write_bytes(value) + result = self.serial() + self.assertEqual(result.stdout.strip(), f'BOOT:{BOOT} BOARD:j71t') + self.assertIn('no device-tree serial-number', result.stderr) + + def test_installed_and_ram_gadgets_use_the_same_stable_serial_before_binding(self): + (self.tree / 'serial-number').write_bytes(b'HardwareSerialA\0') + expected = self.serial().stdout.strip() + for relative in ('boot/initramfs/root/init', 'src/orchard/usb-session'): + with self.subTest(path=relative): + source = (ROOT / relative).read_text() + if relative.endswith('/init'): + start = source.index('usb_gadget() {') + source = source[start:source.index('\nusb_gadget ||', start)] + '\nusb_gadget' + else: + source = source[source.index('name=$(cat'):source.index('\nifconfig ')] + for directory in ('sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', + 'newroot/etc', 'newroot/usr/lib', 'etc', 'usr/libexec/orchard'): + (self.root / directory).mkdir(parents=True, exist_ok=True) + (self.root / 'newroot/etc/os-release').write_text('NAME="Linux"\n') + (self.root / 'etc/hostname').write_text('ram-test\n') + for helper in ('etc/orchard/usb-identity', 'usr/libexec/orchard/usb-identity'): + path = self.root / helper + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.script) + source = re.sub(r'/(sys|newroot|etc|usr)/', lambda m: str(self.root) + m.group(), source) + source = source.replace('mkdir "$g"', 'mkdir -p "$g/functions"') + subprocess.run(['sh', '-ec', 'mount() { :; };\n' + source], check=True, capture_output=True, + env={**os.environ, 'ORCHARD_DEVICE_TREE': str(self.tree)}) + name = 'orchard-system' if relative.endswith('/init') else 'ram-test' + gadget = self.root / 'sys/kernel/config/usb_gadget' / name + self.assertEqual((gadget / 'strings/0x409/serialnumber').read_text().strip(), expected) + self.assertEqual((gadget / 'UDC').read_text().strip(), 'usbdev') + for function in ('ncm.usb0', 'acm.usb0'): + self.assertTrue((gadget / 'configs/c.1' / function).is_symlink()) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_j71t_data.py b/tests/test_j71t_data.py new file mode 100644 index 00000000..c6d83c08 --- /dev/null +++ b/tests/test_j71t_data.py @@ -0,0 +1,31 @@ +"""J71t firmware recipe: the AOP images.""" +import json +from pathlib import Path +import tomllib +import unittest + +HARDWARE = Path(__file__).resolve().parents[1] +DEVICE = HARDWARE / 'devices/j71t/device.toml' + + +class J71tData(unittest.TestCase): + def setUp(self): + device = tomllib.loads(DEVICE.read_text()) + self.recipe = json.loads((DEVICE.parent / device['firmware']['recipe']).read_text()) + + def test_aop_images_come_from_the_board_firmware(self): + aop = self.recipe['inputs']['aop.im4p'] + self.assertEqual(aop['ipsw'], {'zip': 'Firmware/AOP/aopfw-ipad6faop.im4p'}) + self.assertEqual(aop['sha256'], '5a4d8378bb83e175fc589cc1cf978430639d7ffcef07b2d94e9d6a4801124efa') + for name, role, digest in ( + ('aop-ipad6f.bin', 'AOP', 'af5d51a368ceb89dc5c359e7c84e54cb6082e308956be742c17622eb4a6047c8'), + ('aop-ipad6f-ext.bin', 'AOP_ext', '2528d2686ed1262de9910c2d60e45b68137e50aa601a0f8d54ea7d1347b9de22')): + output, = [o for o in self.recipe['outputs'] if o['destination'] == 'lib/firmware/apple/' + name] + self.assertEqual(output['sha256'], digest) + self.assertEqual(output['steps'], [{'step': 'im4p-payload@1', 'input': 'aop.im4p'}, + {'step': 'aop-image@1', 'coprocessor': role}]) + self.assertFalse((HARDWARE / output['destination']).exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_j71t_mac_current.py b/tests/test_j71t_mac_current.py new file mode 100644 index 00000000..17b875e2 --- /dev/null +++ b/tests/test_j71t_mac_current.py @@ -0,0 +1,204 @@ +"""Host-only tests; ORCHARD_KERNEL_SOURCE selects the prepared patched kernel.""" +import os +from pathlib import Path +import plistlib +import re +import subprocess +import tempfile +import unittest +from unittest.mock import patch +from orchard.apple import usb, linux_usb + +HW = Path(__file__).resolve().parents[1] +BOOT = '12345678-1234-1234-1234-123456789abc' + + +def function(source, name): + start = re.search(r'^.*\b' + name + r'\([^;]*?\)\n\{', source, re.M).start() + brace = source.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + '\n' + + +class MacCurrent(unittest.TestCase): + def test_gadget_profile_preserves_network_serial_and_safe_default(self): + source = (HW / 'boot/initramfs/root/init').read_text() + start = source.index('usb_gadget() {') + script = source[start:source.index('\nusb_gadget ||', start)] + for board, flag, expected in [('j71t', '1', '0x1014'), ('j71t', '', '0x4145'), + ('j71t', '2400', '0x4145'), ('j71b', '1', '0x4145')]: + with self.subTest(board=board, flag=flag), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for n in ['sys/kernel/config/usb_gadget', 'sys/class/udc/usbdev', + 'newroot/etc/orchard', 'newroot/usr/lib', 'proc/device-tree', + 'proc/sys/kernel/random', 'etc/orchard']: + (root / n).mkdir(parents=True) + (root / 'proc/device-tree/compatible').write_bytes(f'apple,{board}\0apple,s8003\0'.encode()) + (root / 'proc/sys/kernel/random/boot_id').write_text(BOOT) + (root / 'newroot/etc/os-release').write_text('NAME="Omarchy"\n') + if flag: + (root / 'newroot/etc/orchard/mac-usb-current').write_text(flag + '\n') + helper = (HW / "src/orchard/usb-identity").read_text().replace("/proc/", str(root) + "/proc/") + (root / "etc/orchard/usb-identity").write_text(helper) + run = script + run = re.sub(r'/(sys|newroot|proc|etc)/', lambda m: str(root) + m.group(), run) + run = run.replace('mkdir \"$g\" &&', 'mkdir -p \"$g/functions\" &&') + subprocess.run(['sh', '-ec', 'mount() { :; };\n' + run + '\nusb_gadget'], check=True, + capture_output=True) + g = root / 'sys/kernel/config/usb_gadget/orchard-system' + for key, value in [('idProduct', expected), ('idVendor', '0x05ac'), + ('bDeviceClass', '0xef' if expected == '0x1014' else '0x00'), + ('bDeviceSubClass', '0x02' if expected == '0x1014' else '0x00'), + ('bDeviceProtocol', '0x01' if expected == '0x1014' else '0x00'), ('configs/c.1/MaxPower', '500'), + ('UDC', 'usbdev')]: + self.assertEqual((g/key).read_text().strip(), value) + for name in ['ncm.usb0', 'acm.usb0']: + self.assertTrue((g/'configs/c.1'/name).is_symlink()) + self.assertEqual((g/'strings/0x409/serialnumber').read_text().strip(), f'BOOT:{BOOT} BOARD:{board}') + + def test_compatibility_pid_requires_orchard_identity_on_both_hosts(self): + for serial, mode in [(f'BOOT:{BOOT} BOARD:j71t', 'system'), ('AppleSerial', 'other'), + (f'BOOT:{BOOT} BOARD:j71b', 'other')]: + raw = plistlib.dumps([{'IOObjectClass': 'IOUSBHostDevice', 'idVendor': 0x05ac, + 'idProduct': 0x1014, 'locationID': 123, 'USB Serial Number': serial}]) + with patch.object(usb.sys, 'platform', 'darwin'), patch.object(usb.subprocess, 'run', + return_value=subprocess.CompletedProcess([], 0, raw, b'')): + self.assertEqual(usb.devices()[0]['mode'], mode) + with tempfile.TemporaryDirectory() as tmp: + node = Path(tmp)/'1-2'; node.mkdir() + for key, value in [('idVendor', '05ac'), ('idProduct', '1014'), ('serial', serial)]: + (node/key).write_text(value) + self.assertEqual(linux_usb.devices(Path(tmp))[0]['mode'], mode) + + @unittest.skipUnless(os.environ.get('ORCHARD_KERNEL_SOURCE'), 'set ORCHARD_KERNEL_SOURCE for production C tests') + def test_production_grant_and_charger_policy(self): + source = Path(os.environ['ORCHARD_KERNEL_SOURCE']) + composite = (source/'drivers/usb/gadget/composite.c').read_text() + charger = (source/'drivers/power/supply/apple_tristar_charger.c').read_text() + start = composite.index('if ((ctrl->bRequestType & ~USB_DIR_IN)') + match = composite[start+4:composite.index(' {', start)-1] + match = match.replace('ctrl->bRequestType', 'type').replace('ctrl->bRequest', 'request').replace('le16_to_cpu(cdev->desc.idVendor)', 'vendor') + c = r''' +#include +#include +#include +#include +typedef uint16_t u16; typedef int16_t s16; +#define max(a,b) ((a)>(b)?(a):(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define READ_ONCE(a) (a) +#define USB_DIR_IN 0x80 +#define USB_TYPE_VENDOR 0x40 +#define USB_RECIP_DEVICE 0 +#define USB_APPLE_REQ_CURRENT_LIMIT 0x40 +#define USB_APPLE_VENDOR_ID 0x05ac +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_info_ratelimited(...) ((void)0) +#define EINVAL 22 +#define TRISTAR_CHG_FALLBACK_CODE 34 +#define TRISTAR_RAMPING 2 +#define TRISTAR_CHARGING 3 +struct usb_gadget {int dummy;}; +struct usb_composite_dev {struct usb_gadget *gadget; unsigned vbus_draw_ma; int apple_extra_ma; bool suspended;}; +static unsigned draw; +static void usb_gadget_vbus_draw(struct usb_gadget *g, unsigned ma){draw=ma;} +struct tristar_charger {unsigned usb_epoch, epoch, usb_ma, source_ma, max_ma, target_code, code, phase, step_ns; bool present,usb_on,stopping; void *dev,*wq; int lock,poll_work;}; +struct power_supply {struct tristar_charger *chg;}; +union power_supply_propval {int intval;}; +enum power_supply_property {POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT}; +static unsigned notifications; +static struct tristar_charger *power_supply_get_drvdata(struct power_supply *p){return p->chg;} +static void mutex_lock(int *p){assert(!*p);*p=1;} +static void mutex_unlock(int *p){assert(*p);*p=0;} +static void mod_delayed_work(void *wq,int *work,unsigned d){} +static void power_supply_changed(struct power_supply *p){notifications++;} +static void tristar_withdraw(struct tristar_charger *c,unsigned code){c->code=code;} +''' + for n in ['composite_vbus_draw', 'composite_apple_current_limit']: + c += function(composite, n) + for n in ['tristar_code_ma', 'tristar_ma_code', 'tristar_usb_ma', 'tristar_target_code', 'tristar_update_target', 'tristar_set_property']: + c += function(charger, n) + c += f'int match(unsigned type,unsigned request,unsigned w_length,unsigned vendor){{return {match};}}\n' + c += r''' +static void report(struct power_supply *p,int ma){union power_supply_propval v={.intval=ma*1000};assert(!tristar_set_property(p,POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,&v));} +int main(void){ + for(unsigned t=0;t<256;t++)for(unsigned r=0;r<256;r++)for(unsigned l=0;l<2;l++)for(unsigned v=0;v<2;v++) + assert(match(t,r,l,v?0x05ac:0x1234)==((t==0x40||t==0xc0)&&r==0x40&&!l&&v)); + struct usb_gadget g={0};struct usb_composite_dev d={.gadget=&g}; + composite_vbus_draw(&d,500);assert(draw==500); + composite_apple_current_limit(&d,500,1900);assert(draw==2400); + composite_vbus_draw(&d,500);assert(draw==2400); /* configuration preserves grant */ + for(unsigned i=0;i<65536;i++){ + composite_apple_current_limit(&d,0,i);assert(draw==(unsigned)max(500+(s16)i,0)); + } + d.suspended=true;composite_vbus_draw(&d,2); + composite_apple_current_limit(&d,500,1900);assert(draw==2); + d.suspended=false;composite_vbus_draw(&d,500);assert(draw==2400); + d.apple_extra_ma=0;composite_vbus_draw(&d,100);assert(draw==100); + composite_vbus_draw(&d,0);composite_apple_current_limit(&d,0,1900);assert(draw==0); + struct tristar_charger ch={.epoch=7,.usb_epoch=7,.max_ma=2400,.present=true,.usb_on=true,.phase=TRISTAR_CHARGING,.code=34}; + struct power_supply p={.chg=&ch}; + report(&p,2400);assert(ch.usb_ma==2400&&ch.target_code==186&&ch.phase==TRISTAR_RAMPING&¬ifications==1); + ch.code=186;ch.phase=TRISTAR_CHARGING;report(&p,500);assert(ch.code==34&&ch.target_code==34); + for(int ma=501;ma<=33267;ma++){ + report(&p,ma);assert(ch.usb_ma<=(unsigned)ma&&ch.usb_ma<=2400); + assert(tristar_code_ma(ch.target_code)<=min((unsigned)ma,2400)); + } + report(&p,-1);assert(!ch.usb_ma&&ch.target_code==34); + ch.usb_on=false;report(&p,2400);assert(!ch.usb_ma); + ch.usb_on=true;ch.present=false;report(&p,2400);assert(!ch.usb_ma); + ch.present=true;report(&p,2400);ch.epoch++;assert(!tristar_usb_ma(&ch)&&tristar_target_code(&ch)==34); + puts("PASS 262144 request tuples, all signed grants, suspend, board cap, ramp/revoke, attachment and epoch gates"); +} +''' + # Lifecycle invalidations remain in the actual callbacks. + for name in ['composite_reset', 'composite_disconnect', 'composite_suspend']: + self.assertIn('cdev->apple_extra_ma = 0;', function(composite, name)) + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp); (p/'test.c').write_text(c) + subprocess.run(['cc','-std=gnu11','-Wall','-Werror','-fsanitize=address,undefined', str(p/'test.c'),'-o',str(p/'test')],check=True) + subprocess.run([str(p/'test')], check=True) + +if __name__ == '__main__': + unittest.main() + + +"""A compatibility PID must never classify a real Apple device as Orchard.""" +from pathlib import Path +import plistlib +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from orchard.apple import usb, linux_usb + +BOOT = '12345678-1234-1234-1234-123456789abc' +STABLE = 'f09dfe4c604a' + + +class StableMacCurrent(unittest.TestCase): + def test_compatibility_pid_requires_orchard_identity_on_both_hosts(self): + for serial, mode in [(f'BOOT:{BOOT} BOARD:j71t', 'system'), + (f'BOARD:j71t ID:{STABLE}', 'system'), + ('AppleSerial', 'other'), ('BOARD:j71t', 'other'), + (f'ID:{STABLE}', 'other'), ('BOARD:j71t ID:bad', 'other'), + (f'BOARD:j71b ID:{STABLE}', 'other'), + (f'BOOT:{BOOT} BOARD:j71b', 'other')]: + with self.subTest(serial=serial): + raw = plistlib.dumps([{'IOObjectClass': 'IOUSBHostDevice', 'idVendor': 0x05ac, + 'idProduct': 0x1014, 'locationID': 123, 'USB Serial Number': serial}]) + with patch.object(usb.sys, 'platform', 'darwin'), patch.object(usb.subprocess, 'run', + return_value=subprocess.CompletedProcess([], 0, raw, b'')): + self.assertEqual(usb.devices()[0]['mode'], mode) + with tempfile.TemporaryDirectory() as tmp: + node = Path(tmp) / '1-2' + node.mkdir() + for key, value in [('idVendor', '05ac'), ('idProduct', '1014'), ('serial', serial)]: + (node / key).write_text(value) + self.assertEqual(linux_usb.devices(Path(tmp))[0]['mode'], mode) diff --git a/tests/test_m1n1_psci.py b/tests/test_m1n1_psci.py new file mode 100644 index 00000000..85091696 --- /dev/null +++ b/tests/test_m1n1_psci.py @@ -0,0 +1,81 @@ +"""Host-only framing checks for the J71t PSCI loader variables.""" +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +TOOL = ROOT / 'boot/m1n1/psci-loader' + + +class PSCILoader(unittest.TestCase): + def test_payload_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as tmp: + loader, output = Path(tmp) / 'm1n1.bin', Path(tmp) / 'psci.bin' + original = b'bare loader\0' + loader.write_bytes(original) + cmd = [sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + self.assertEqual(loader.read_bytes(), original) + self.assertNotEqual(subprocess.run(cmd, capture_output=True).returncode, 0) + self.assertEqual(output.read_bytes(), original + b'psci=1\n') + + def test_reset_and_off_variables(self): + cases = {('--system-reset', 'proxy'): b'psci=1\npsci_reset=proxy\n', + ('--system-reset', 'off'): b'psci=1\npsci_reset=0\n', + ('--system-reset', 'boot'): b'psci=1\n', + ('--no-psci',): b'psci=0\n'} + with tempfile.TemporaryDirectory() as tmp: + loader = Path(tmp) / 'm1n1.bin' + loader.write_bytes(b'bare') + for n, (extra, records) in enumerate(cases.items()): + output = Path(tmp) / f'out{n}.bin' + result = subprocess.run([sys.executable, str(TOOL), '--device', 'j71t', '--loader', + str(loader), '--output', str(output), *extra], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_bytes(), b'bare' + records) + both = subprocess.run([sys.executable, str(TOOL), '--device', 'j71t', '--loader', str(loader), + '--output', str(Path(tmp) / 'x.bin'), '--no-psci', '--system-reset', + 'proxy'], capture_output=True) + self.assertNotEqual(both.returncode, 0) + + def test_series_has_reset_patches(self): + names = sorted(p.name for p in (ROOT / 'boot/m1n1/patches/orchard').glob('*.patch')) + for prefix in ('0015-psci-', '0016-psci-Turn-the-resident-EL3-PSCI-on-by-default', + '0017-psci-Trap-EL1-WFI-WFE', '0018-psci-Implement-SYSTEM_RESET', + '0019-bootmgr-Boot-the-default-entry'): + self.assertTrue(any(n.startswith(prefix) for n in names), prefix) + + def test_manifest_psci_record_keeps_logo_first(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, data in {'loader': b'M1N1', 'dtb': b'DTB', 'kernel': b'KERNEL', + 'initrd': b'INITRD'}.items(): + (root / name).write_bytes(data) + logo = b'm1n1_logo_256128' + bytes(4 * (256 * 256 + 128 * 128)) + (root / 'logo').write_bytes(logo) + records = b'chosen.bootargs=console=tty0\npsci=1\n' + (root / 'profile.json').write_text(json.dumps({'bootargs': records.decode()})) + result = subprocess.run([sys.executable, '-I', str(ROOT / 'src/orchard/image/assemble.py'), + '--manifest', str(root / 'profile.json'), '--loader', str(root / 'loader'), + '--logo', str(root / 'logo'), '--dtb', str(root / 'dtb'), + '--kernel', str(root / 'kernel'), '--initramfs', str(root / 'initrd'), + '--output', str(root / 'payload')], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((root / 'payload').read_bytes().startswith(b'M1N1' + logo + records + b'DTB')) + + def test_board_requires_explicit_opt_in(self): + for device in ([], ['--device', 'j71b']): + result = subprocess.run([sys.executable, str(TOOL), *device, '--loader', 'unused', + '--output', 'unused'], capture_output=True) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_platform_layer.py b/tests/test_platform_layer.py index 9acba731..b7029826 100644 --- a/tests/test_platform_layer.py +++ b/tests/test_platform_layer.py @@ -82,7 +82,10 @@ def test_the_initrd_carries_busybox_and_the_storage_modules_only(self): base = f'lib/modules/{RELEASE}/' self.assertEqual(names['bin/busybox'], b'static busybox') self.assertEqual(names['init'], (ROOT / 'boot/initramfs/root/init').read_bytes()) + self.assertEqual(names['etc/orchard/usb-identity'], (ROOT / 'src/orchard/usb-identity').read_bytes()) self.assertEqual(names['etc/orchard/modules'], b'nvme\n') + self.assertEqual(names['etc/orchard/dtbs/j71t.dtb'], b's8003-j71t') + self.assertIn(b'apple,j71t j71t.dtb\n', names['etc/orchard/pristine-map']) self.assertEqual(sorted(name for name in names if name.endswith('.ko')), [base + 'kernel/drivers/nvme/host/nvme-core.ko', base + 'kernel/drivers/nvme/host/nvme.ko']) self.assertEqual(names[base + 'modules.dep'], diff --git a/tests/test_ram.py b/tests/test_ram.py index 89c83aaa..6e430d29 100644 --- a/tests/test_ram.py +++ b/tests/test_ram.py @@ -475,6 +475,7 @@ def test_usb_session_is_added_unless_the_project_opts_out(self): ram.build_directory(example, hardware=self.hardware) archive = gzip.decompress((example / 'out/apple-16k/initramfs.cpio.gz').read_bytes()) self.assertIn(b'usr/libexec/orchard/usb-session\0', archive) + self.assertIn(b'usr/libexec/orchard/usb-identity\0', archive) self.assertIn(b'etc/orchard/authorized_keys\0', archive) public = (self.base / 'home/private/ram-ssh/id_ed25519.pub').read_bytes() self.assertIn(public, archive) @@ -484,6 +485,7 @@ def test_usb_session_is_added_unless_the_project_opts_out(self): ram.build_directory(example, hardware=self.hardware) archive = gzip.decompress((example / 'out/apple-16k/initramfs.cpio.gz').read_bytes()) self.assertNotIn(b'usr/libexec/orchard/usb-session\0', archive) + self.assertNotIn(b'usr/libexec/orchard/usb-identity\0', archive) self.assertNotIn(b'etc/orchard/authorized_keys\0', archive) def test_arbitrary_directory_layout_uses_active_hardware_package(self): diff --git a/tests/test_system_session.py b/tests/test_system_session.py index 197c0907..8f89b1d6 100644 --- a/tests/test_system_session.py +++ b/tests/test_system_session.py @@ -9,6 +9,9 @@ SYSTEM = {'location': 1, 'mode': 'system', 'product': 'Omarchy A', 'serial_port': '/dev/cu.x', 'boot_id': '00000000-0000-4000-8000-000000000001', 'profile': 'j71b'} PAYLOAD = {**SYSTEM, 'mode': 'ram', 'product': 'Orchard gpu_001'} +STABLE_PAYLOAD = {**PAYLOAD, 'boot_id': None, 'usb_id': 'f09dfe4c604a'} +BOOT_A = '11111111-1111-4111-8111-111111111111' +BOOT_B = '22222222-2222-4222-8222-222222222222' class SystemSession(unittest.TestCase): @@ -62,6 +65,60 @@ def test_without_a_user_it_says_how_to_get_one(self): with self.assertRaisesRegex(ValueError, '--user'): ram_session.connect(SYSTEM) + def ram_connect(self, reply, device=STABLE_PAYLOAD): + with mock.patch.object(ram_session, '_directory', return_value=Path(self.temp.name)), \ + mock.patch.object(ram_session, 'identity', return_value=Path('/ram-key')), \ + mock.patch.object(ram_session, 'SerialPort') as port, \ + mock.patch.object(ram_session, 'wait_authenticated') as wait: + port.return_value.__enter__.return_value.run.return_value = (0, reply) + argv = ram_session.connect(device) + command = port.return_value.__enter__.return_value.run.call_args.args[0] + self.assertIn('/proc/sys/kernel/random/boot_id', command) + self.assertIn('/run/orchard/ssh_host_ed25519_key.pub', command) + self.assertIn('StrictHostKeyChecking=yes', argv) + wait.assert_called_once() + return argv + + def test_stable_usb_identity_gets_fresh_boot_and_key_from_acm_on_each_connection(self): + # A second connect must not reuse a pin just because USB is unchanged. + first = self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n') + pin_a = Path(self.temp.name) / f'known_hosts-{BOOT_A}' + self.assertEqual(pin_a.read_text(), '172.16.42.1 ssh-ed25519 AAAA\n') + self.assertEqual(pin_a.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n'), first) + second = self.ram_connect(BOOT_B + '\nssh-ed25519 BBBB\n') + self.assertNotEqual(first, second) + self.assertFalse(pin_a.exists()) + self.assertEqual((Path(self.temp.name) / f'known_hosts-{BOOT_B}').read_text(), + '172.16.42.1 ssh-ed25519 BBBB\n') + + def test_legacy_ram_boot_tag_is_still_verified_against_acm(self): + self.ram_connect(PAYLOAD['boot_id'] + '\nssh-ed25519 AAAA\n', device=PAYLOAD) + with self.assertRaisesRegex(ValueError, 'disconnected or rebooted'): + self.ram_connect(BOOT_B + '\nssh-ed25519 AAAA\n', device=PAYLOAD) + + def test_invalid_acm_identity_or_changed_key_does_not_reuse_a_pin(self): + self.ram_connect(BOOT_A + '\nssh-ed25519 AAAA\n') + for reply, error in [('garbage\nssh-ed25519 BBBB\n', 'Invalid RAM boot'), + ('../../escape\nssh-ed25519 BBBB\n', 'Invalid RAM boot'), + (BOOT_A + '\n', 'exactly one SSH host key'), + (BOOT_A + '\nssh-ed25519 BBBB\n', 'changed within the same boot')]: + with self.subTest(reply=reply), self.assertRaisesRegex(ValueError, error): + self.ram_connect(reply) + self.assertEqual((Path(self.temp.name) / f'known_hosts-{BOOT_A}').read_text(), + '172.16.42.1 ssh-ed25519 AAAA\n') + + def test_stable_usb_check_rejects_another_unit_or_mode_at_the_same_port(self): + check = ram_session._check(STABLE_PAYLOAD) + with mock.patch('orchard.apple.usb.devices', return_value=[STABLE_PAYLOAD]): + check() + for change in ({'usb_id': 'fedcba987654'}, {'mode': 'system'}, + {'profile': 'j71t'}, {'serial_port': '/dev/cu.other'}): + with self.subTest(change=change), \ + mock.patch('orchard.apple.usb.devices', return_value=[{**STABLE_PAYLOAD, **change}]), \ + self.assertRaisesRegex(ValueError, 'disconnected or rebooted'): + check() + if __name__ == '__main__': unittest.main() diff --git a/tools/catalog/drivers.json b/tools/catalog/drivers.json index 5b000d53..39e45a4f 100644 --- a/tools/catalog/drivers.json +++ b/tools/catalog/drivers.json @@ -193,6 +193,7 @@ "touch", "usb_network", "usb_serial", + "video_decode_hevc", "wifi" ], "j71s": [ @@ -205,8 +206,10 @@ "charging", "light_sensor", "touch", + "usb_host", "usb_network", - "usb_serial" + "usb_serial", + "video_decode_hevc" ], "j71t": [ "audio_output", @@ -220,8 +223,10 @@ "storage_read", "storage_write", "touch", + "usb_host", "usb_network", "usb_serial", + "video_decode_hevc", "wifi" ], "j72b": [ @@ -238,6 +243,7 @@ "touch", "usb_network", "usb_serial", + "video_decode_hevc", "wifi" ], "j72s": [ @@ -250,8 +256,10 @@ "charging", "light_sensor", "touch", + "usb_host", "usb_network", - "usb_serial" + "usb_serial", + "video_decode_hevc" ], "j72t": [ "audio_output", @@ -265,8 +273,10 @@ "storage_read", "storage_write", "touch", + "usb_host", "usb_network", "usb_serial", + "video_decode_hevc", "wifi" ], "j73": [ diff --git a/tools/catalog/feature-drivers.toml b/tools/catalog/feature-drivers.toml index 9616d549..8988ef4b 100644 --- a/tools/catalog/feature-drivers.toml +++ b/tools/catalog/feature-drivers.toml @@ -16,4 +16,6 @@ battery = ["bq27xxx_battery_hdq", "macsmc_hwmon"] charging = ["apple_d2365_charger", "apple_tristar_charger"] usb_serial = ["dwc2"] usb_network = ["dwc2"] +usb_host = ["apple_s8000_usb_role"] audio_output = ["snd_soc_apple_s8000_mca"] +video_decode_hevc = ["apple_vxd"] diff --git a/tools/testing/test_kernel_safety.py b/tools/testing/test_kernel_safety.py new file mode 100644 index 00000000..292dc553 --- /dev/null +++ b/tools/testing/test_kernel_safety.py @@ -0,0 +1,268 @@ +"""Execute kernel entry points with host MMIO/OS mocks (no device access).""" +from pathlib import Path +import os +import subprocess +import tempfile +import unittest +def function(text, name): + start = text.rfind('\n', 0, text.index(name + '(')) + 1 + opening = text.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + +def execute(code): + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp)/'test.c';p.write_text(code) + subprocess.run(['cc','-std=gnu11','-Wall','-Werror','-Wno-unused-parameter', + str(p),'-o',str(p.with_suffix(''))],check=True) + subprocess.run([str(p.with_suffix(''))],check=True) + +# Point at the fully applied release kernel tree, never a hand-made struct fixture. +S=Path(os.environ['APPLIED_KERNEL_SOURCE']) +def source(n): return (S/n).read_text() +class KernelSafety(unittest.TestCase): + def test_N1_probe_initializes_real_watchdog_then_restart(self): + s=source('drivers/watchdog/apple_wdt.c') + defines='\n'.join(x for x in s.splitlines() if x.startswith('#define APPLE_WDT_')) + execute(r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U<<(n)) +#define U32_MAX UINT32_MAX +#define GFP_KERNEL 0 +#define WDOG_HW_RUNNING 1 +#define REBOOT_WARM 1 +#define REBOOT_SOFT 2 +#define REBOOT_COLD 0 +#define pr_emerg(...) ((void)0) +#define IS_ERR(p) false +#define PTR_ERR(p) (-1) +#define min(a,b) ((a)<(b)?(a):(b)) +struct device { void *of_node; }; +struct platform_device { struct device dev; void *data; }; +struct watchdog_device {struct device *parent;void *ops,*info;unsigned max_hw_heartbeat_ms,timeout;unsigned long status;}; +struct apple_wdt {struct watchdog_device wdd;unsigned char *regs;unsigned long clk_rate;}; +struct clk {int x;}; +static int apple_wdt_ops,apple_wdt_info,tethered,priority,writes,flushes,delay; +static struct clk clock_ref; +static struct watchdog_device *registered; +static unsigned char regs[64]; +static int offsets[8],values[8]; +static struct apple_wdt *to_apple_wdt(struct watchdog_device *w) {return (void *)((char*)w - offsetof(struct apple_wdt,wdd));} +static void *devm_kzalloc(struct device *d,size_t n,int f){return calloc(1,n);} +static void *devm_platform_ioremap_resource(struct platform_device *p,int i){return regs;} +static struct clk *devm_clk_get_enabled(struct device *d,void *n){return &clock_ref;} +static unsigned long clk_get_rate(struct clk *c){return 1000;} +static void platform_set_drvdata(struct platform_device *p,void *d){p->data=d;} +static int readl_relaxed(unsigned char *p){return 0;} +static int readl(unsigned char *p){flushes++;return 0;} +static void writel_relaxed(unsigned v,unsigned char *p){offsets[writes]=p-regs;values[writes++]=v;} +static void set_bit(int b,unsigned long *p){*p|=1UL<parent==&p.dev && priority==255); + writes=0;assert(!apple_wdt_restart(registered,REBOOT_WARM,0));assert(!writes); + assert(!apple_wdt_restart(registered,REBOOT_SOFT,0));assert(!writes); + assert(!apple_wdt_restart(registered,REBOOT_COLD,0)); + assert(writes==3 && offsets[0]==APPLE_WDT_WD1_CTRL && values[0]==APPLE_WDT_CTRL_RESET_EN); + assert(offsets[1]==APPLE_WDT_WD1_BITE_TIME && !values[1]); + assert(offsets[2]==APPLE_WDT_WD1_CUR_TIME && !values[2] && flushes==1 && delay==150); + free(p.data);tethered=0;assert(!apple_wdt_probe(&p));assert(priority==128); + writes=0;apple_wdt_restart(registered,REBOOT_WARM,0);assert(writes==3); + registered->parent=0;writes=0;apple_wdt_restart(registered,REBOOT_COLD,0);assert(writes==3); + free(p.data); +} +''') + + def test_N2_each_pcie_failure_cold_resets_without_panic(self): + body=function(source('drivers/pci/controller/pcie-apple-h9p.c'),'apple_h9p_s8003_shutdown') + self.assertNotIn('panic(',body) + self.assertNotIn('return apple_cold_reset(',body) + execute(r''' +#include +#include +#include +#include +typedef uint32_t u32; +#define __iomem +#define BIT(n) (1U<<(n)) +#define GENMASK(a,b) 0 +#define READ_ONCE(x) (x) +#define H9P_NUM_PORTS 2 +#define H9P_CFG_PORT_STRIDE 0 +#define PCI_COMMAND 0 +#define PCI_PRIMARY_BUS 0 +#define S8003_RC_PORT_PWR(n) 0 +#define S8003_RC_PORT_PWR_REFCLK_NOGATE 0 +#define S8003_PORT_LINKSTS 0 +#define S8003_PORT_LINKSTS_UP 1 +#define S8003_PERST_TO_REFCLK_US 1 +#define H9P_NVMMU_SART_CTRL 0 +#define readl_poll_timeout(a,b,c,d,e) do {b=0;(void)b;}while(0) +#define dev_info(...) ((void)0) +struct device {void *of_node;};struct platform_device {struct device dev;}; +struct apple_h9p_pcie {struct device *dev;bool ans_handoff_stopped;int late_work;unsigned enabled_ports;unsigned char *base_config,*base_rc,*base_port[2];void *perst[2];struct {bool ready;unsigned char *base;} nvmmu;}; +static struct device dev;static struct apple_h9p_pcie pcie={.dev=&dev}; +enum system_states { SYSTEM_RUNNING, SYSTEM_POWER_OFF, SYSTEM_RESTART }; +static int system_state=SYSTEM_RUNNING; +static int kexec_in_progress=1,site,resets,powered_calls; +static unsigned offs; +static const char *reason; +static struct apple_h9p_pcie *apple_h9p_pcie_lookup(struct device *d){return &pcie;} +static bool of_device_is_compatible(void *n,const char *s){return true;} +static bool of_property_read_bool(void *n,const char *s){return !strcmp(s,"apple,ans-kexec-handoff");} +static void cancel_delayed_work_sync(int *w){} +static bool apple_s8003_port_powered(struct apple_h9p_pcie *p){return !(site==4 && ++powered_calls==2);} +static void writew(int v,unsigned char *p){} +static void h9p_rmw(unsigned char *p,int a,int b){} +static int gpiod_direction_output(void *p,int v){return site==2;} +static int apple_s8003_port_stop(struct apple_h9p_pcie *p,unsigned port){return site==3;} +static void udelay(int us){} +static void h9p_writel_flush(int v,unsigned char *p){} +static void apple_s8003_port_off(struct apple_h9p_pcie *p,unsigned port){offs|=BIT(port);} +static int gpiod_get_value_cansleep(void *p){return 1;} +static void apple_cold_reset(const char *s){resets++;reason=s;} +''' + body + r''' +int main(void){struct platform_device p;unsigned char regs[8]; + pcie.base_config=regs;pcie.base_rc=regs;pcie.base_port[0]=pcie.base_port[1]=regs; + for(site=1;site<=4;site++){ + pcie.ans_handoff_stopped=site!=1;pcie.enabled_ports=1;powered_calls=0;resets=0; + apple_h9p_s8003_shutdown(&p);assert(resets==1 && reason); + assert(strstr(reason,site==1?"uncertified":site==2?"PERST":site==3?"LTSSM":"certificate")); + assert(pcie.enabled_ports==1); + } + site=0;pcie.ans_handoff_stopped=true;pcie.enabled_ports=1;resets=0; + apple_h9p_s8003_shutdown(&p);assert(!resets && !pcie.enabled_ports); + kexec_in_progress=0;site=1;apple_h9p_s8003_shutdown(&p);assert(!resets); + /* kexec keeps every port powered, the Wi-Fi port included. */ + kexec_in_progress=1;site=0;pcie.ans_handoff_stopped=true;pcie.enabled_ports=3;offs=0;resets=0; + apple_h9p_s8003_shutdown(&p);assert(!resets && !pcie.enabled_ports && !offs); + /* Restart: the same handoff, and the other ports powered off. */ + kexec_in_progress=0;system_state=SYSTEM_RESTART;pcie.enabled_ports=3;offs=0; + apple_h9p_s8003_shutdown(&p);assert(!resets && !pcie.enabled_ports && offs==BIT(1)); + for(site=1;site<=4;site++){ + pcie.ans_handoff_stopped=site!=1;pcie.enabled_ports=3;powered_calls=0;resets=0; + apple_h9p_s8003_shutdown(&p);assert(resets==1 && reason); + } + /* Power-off and halt leave the ports alone, as before. */ + site=0;system_state=SYSTEM_POWER_OFF;pcie.ans_handoff_stopped=true;pcie.enabled_ports=3;offs=0;resets=0; + apple_h9p_s8003_shutdown(&p);assert(!resets && pcie.enabled_ports==3 && !offs); +} +''') + + def test_S1_second_sysfs_handover_succeeds_but_errors_survive(self): + body=function(source('drivers/gpu/apple-g5p/teardown.inc'),'g5p_td_command') + execute(r''' +#include +#include +#include +struct g5p_diag {int unused;}; +static int result; +#define sysfs_streq(a,b) (!strcmp(a,b)) +static int g5p_td_capture(struct g5p_diag *g,const char *s){return result;} +static int g5p_td_final(struct g5p_diag *g,const char *s){return result;} +static int g5p_kq_run(struct g5p_diag *g,const char *s){return result;} +'''+body+r''' +int main(void){struct g5p_diag g; + result=-EALREADY;assert(!g5p_td_command(&g,"td-kexec")); + assert(g5p_td_command(&g,"td-final")==-EALREADY); + result=-EIO;assert(g5p_td_command(&g,"td-kexec")==-EIO); + result=0;assert(!g5p_td_command(&g,"td-kexec"));} +''') + + def test_S2_notifier_only_escalates_restart_not_halt_poweroff_or_retired(self): + body=function(source('drivers/gpu/apple-g5p/teardown.inc'),'g5p_td_reboot_notify') + execute(r''' +#include +#include +struct notifier_block {int x;};struct g5p_diag {void *dev;}; +static struct g5p_diag gpu,*g5p_td_dev=&gpu; +static int result,resets; +#define READ_ONCE(x) (x) +#define SYS_RESTART 1 +#define SYS_HALT 2 +#define SYS_POWER_OFF 3 +#define NOTIFY_DONE 0 +#define dev_emerg(...) ((void)0) +static int g5p_kq_run(struct g5p_diag *g,const char *s){return result;} +static void apple_cold_reset(const char *s){resets++;} +'''+body+r''' +int main(void){result=-EBUSY; + g5p_td_reboot_notify(0,SYS_HALT,0);g5p_td_reboot_notify(0,SYS_POWER_OFF,0);assert(!resets); + g5p_td_reboot_notify(0,SYS_RESTART,0);assert(resets==1); + result=-EALREADY;g5p_td_reboot_notify(0,SYS_RESTART,0);assert(resets==1); + result=0;g5p_td_reboot_notify(0,SYS_RESTART,0);assert(resets==1);} +''') + run=function(source('drivers/gpu/apple-g5p/teardown.inc'),'g5p_kq_run') + retired=run[run.index('if (g5p_r8_is_retired())'):run.index('if (READ_ONCE(g->sleep_failed))')] + self.assertIn('return -EALREADY;',retired) + + def test_S6_flush_precedes_dump_and_cold_restart(self): + body=function(source('include/linux/soc/apple/cold-reset.h'),'apple_cold_reset') + execute(r''' +#include +#include +static int step,reboot_mode; +#define pr_emerg(...) assert(step++==0) +#define KMSG_DUMP_PANIC 1 +#define REBOOT_COLD 2 +#define WRITE_ONCE(x,v) do {assert(step++==3);x=v;}while(0) +static void pr_flush(int ms,bool reset){assert(step++==1 && ms==1000 && reset);} +static void kmsg_dump_desc(int reason,const char *s){assert(step++==2);} +static void emergency_restart(void){assert(step++==4 && reboot_mode==REBOOT_COLD);} +'''+body+r''' +int main(void){apple_cold_reset("test");assert(step==5);} +''') + + def test_S4_no_incoming_probe_terminate_or_failed_put_on_the_incoming_kernel(self): + s=source('drivers/media/platform/apple/vxd/apple-vxd-core.c') + self.assertNotIn('apple_vxd_fw_probe_stop',s) + self.assertNotIn('ProbeTerminate',source('drivers/media/platform/apple/vxd/apple-vxd-fw.c')) + + def test_S5_failed_shutdown_does_not_put_or_cross_kexec(self): + body=function(source('drivers/media/platform/apple/vxd/apple-vxd-core.c'),'apple_vxd_shutdown') + execute(r''' +#include +#include +struct variant {bool sleep_stop;};struct device {int x;}; +struct platform_device {struct device dev;}; +struct apple_vxd_dev {struct variant *variant;int watchdog_work,job_done_work;}; +static struct variant variant={true};static struct apple_vxd_dev vxd={.variant=&variant}; +static int result,kexec_in_progress,resets,puts; +static void *platform_get_drvdata(struct platform_device *p){return &vxd;} +static void cancel_delayed_work_sync(int *w){} +static void flush_work(int *w){} +static int apple_vxd_fw_shutdown_stop(struct apple_vxd_dev *v){return result;} +static int pm_runtime_force_suspend(struct device *d){puts++;return 0;} +static void apple_cold_reset(const char *s){resets++;} +#define dev_warn(...) ((void)0) +'''+body+r''' +int main(void){struct platform_device p; + result=-1;kexec_in_progress=1;apple_vxd_shutdown(&p);assert(resets==1 && !puts); + kexec_in_progress=0;apple_vxd_shutdown(&p);assert(resets==1 && !puts); + result=0;kexec_in_progress=1;apple_vxd_shutdown(&p);assert(puts==1); + variant.sleep_stop=false;apple_vxd_shutdown(&p);assert(puts==1);} +''') + stop=function(source('drivers/media/platform/apple/vxd/apple-vxd-fw.c'),'apple_vxd_fw_shutdown_stop') + failure=stop[stop.index('if (ret) {'):stop.index('apple_vxd_cmdbufs_fini')] + self.assertIn('return ret;',failure) + self.assertNotIn('pm_runtime_put',failure) + self.assertNotIn('dma_free',failure) + +if __name__=='__main__':unittest.main() diff --git a/tools/testing/test_nvme_shutdown_safety.py b/tools/testing/test_nvme_shutdown_safety.py new file mode 100644 index 00000000..122577fb --- /dev/null +++ b/tools/testing/test_nvme_shutdown_safety.py @@ -0,0 +1,95 @@ +"""Execute applied ANS shutdown paths with injected failures and no hardware.""" +from pathlib import Path +import os +import unittest +from test_kernel_safety import execute, function +SOURCE=Path(os.environ['APPLIED_KERNEL_SOURCE']) +class NVMeShutdownSafety(unittest.TestCase): + def test_shutdown_certificates_cold_reset_without_teardown_after_failure(self): + text=(SOURCE/'drivers/nvme/host/pci.c').read_text() + disable=function(text,'nvme_dev_disable_paired') + shutdown=function(text,'nvme_shutdown') + self.assertNotIn('panic(',disable+shutdown) + execute(r''' +#include +#include +#include +#include +#define dev_info(...) ((void)0) +#define dev_err(...) ((void)0) +enum nvme_ctrl_state { NVME_CTRL_LIVE, NVME_CTRL_RESETTING }; +struct nvme_ctrl { int queue_count; void *admin_q; }; +struct pci_dev {int x;}; +struct nvme_dev {struct nvme_ctrl ctrl; void *dev,*dma_ops; int shutdown_lock,queues[1];}; +enum system_states { SYSTEM_RUNNING, SYSTEM_POWER_OFF, SYSTEM_RESTART }; +static int system_state=SYSTEM_RUNNING; +static int kexec_in_progress=1,nvme_pci_apple_ans_ops,site,resets,teardown,locked; +static const char *reason; +static struct pci_dev pci; +static struct nvme_dev dev={.ctrl={.queue_count=1},.dma_ops=&nvme_pci_apple_ans_ops}; +static enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *c){return NVME_CTRL_LIVE;} +static struct pci_dev *to_pci_dev(void *d){return &pci;} +static void mutex_lock(int *p){assert(!locked);locked=1;} +static void mutex_unlock(int *p){assert(locked);locked=0;} +static bool nvme_pci_ctrl_is_dead(struct nvme_dev *d){return site==2;} +static bool pci_is_enabled(struct pci_dev *p){return true;} +static int nvme_wait_freeze_timeout(struct nvme_ctrl *c){return site==1?0:1;} +static void apple_cold_reset(const char *why){resets++;reason=why;} +static int apple_ans_nvmmu_drain(void *d){return site==3?-EIO:0;} +static int nvme_disable_ctrl(struct nvme_ctrl *c,bool shutdown){return 0;} +static int nvme_apple_ans_handoff_disable(struct nvme_dev *d,int ret){return ret;} +static int apple_ans_handoff_stopped(void *d){return 0;} +static bool blk_queue_dying(void *q){return false;} +#define NOOP(name,type) static void name(type *p){} +NOOP(nvme_start_freeze,struct nvme_ctrl) +NOOP(nvme_unfreeze,struct nvme_ctrl) +NOOP(nvme_quiesce_io_queues,struct nvme_ctrl) +NOOP(nvme_sync_io_queues,struct nvme_ctrl) +NOOP(nvme_cancel_tagset,struct nvme_ctrl) +NOOP(nvme_cancel_admin_tagset,struct nvme_ctrl) +NOOP(nvme_unquiesce_io_queues,struct nvme_ctrl) +NOOP(nvme_unquiesce_admin_queue,struct nvme_ctrl) +static void nvme_delete_io_queues(struct nvme_dev *d){teardown++;} +static void nvme_suspend_io_queues(struct nvme_dev *d){teardown++;} +static void nvme_reap_pending_cqes(struct nvme_dev *d){teardown++;} +static void nvme_suspend_queue(struct nvme_dev *d,int n){teardown++;} +static void pci_free_irq_vectors(struct pci_dev *p){teardown++;} +static void pci_disable_device(struct pci_dev *p){teardown++;} +static void nvme_poll_irqdisable(int *q){} +static struct nvme_dev *pci_get_drvdata(struct pci_dev *p){return &dev;} +static bool apple_ans_is_s8003(void *d){return true;} +static int apple_ans_handoff_begin(void *d){return 0;} +static bool nvme_wait_reset(struct nvme_ctrl *c){return site!=4;} +static void nvme_disable_prepare_reset(struct nvme_dev *d,bool shutdown){teardown++;} +'''+disable+'\n'+shutdown+r''' +int main(void){ + for(site=1;site<=3;site++){ + resets=teardown=locked=0; + assert(nvme_dev_disable_paired(&dev,true,true)==-EIO); + assert(resets==1 && reason && !teardown && !locked); + } + site=4;resets=teardown=0;nvme_shutdown(&pci); + assert(resets==1 && !teardown); + site=0;resets=teardown=0;nvme_shutdown(&pci); + assert(!resets && teardown && !locked); + kexec_in_progress=0;site=1;resets=teardown=0; + assert(nvme_dev_disable_paired(&dev,true,true)==-EBUSY); + assert(!resets && !locked && !teardown); + site=0;resets=teardown=0;nvme_shutdown(&pci); + assert(!resets && teardown); + /* A restart that re-enters the boot loader takes the kexec certificate. */ + system_state=SYSTEM_RESTART; + for(site=1;site<=3;site++){ + resets=teardown=locked=0; + assert(nvme_dev_disable_paired(&dev,true,true)==-EIO); + assert(resets==1 && reason && !teardown && !locked); + } + site=4;resets=teardown=0;nvme_shutdown(&pci); + assert(resets==1 && strstr(reason,"restart") && !teardown); + site=0;resets=teardown=0;nvme_shutdown(&pci); + assert(!resets && teardown && !locked); + /* Power-off keeps the plain shutdown. */ + system_state=SYSTEM_POWER_OFF;site=4;resets=teardown=0;nvme_shutdown(&pci); + assert(!resets && teardown==1); +} +''') diff --git a/tools/testing/test_usb_release_safety.py b/tools/testing/test_usb_release_safety.py new file mode 100644 index 00000000..ea1d3547 --- /dev/null +++ b/tools/testing/test_usb_release_safety.py @@ -0,0 +1,91 @@ +"""Execute applied charger and role-switch policy with host-only mocks.""" +from pathlib import Path +import os +import unittest +from test_kernel_safety import execute, function +S=Path(os.environ['APPLIED_KERNEL_SOURCE']) +class USBReleaseSafety(unittest.TestCase): + def test_reconnect_after_long_unplug_keeps_data_off_for_identification(self): + text=(S/'drivers/power/supply/apple_tristar_charger.c').read_text() + poll=function(text,'tristar_poll') + start=poll.index('\tif (!status) {');end=start;depth=0 + for end in range(poll.index('{',start),len(poll)): + depth+=(poll[end]=='{')-(poll[end]=='}') + if not depth:break + empty=poll[start:end+1].replace('goto again;','return;') + execute(r''' +#include +#include +#include +typedef uint64_t u64;typedef int64_t s64; +#define TRISTAR_HOLD_MS 1000 +#define NSEC_PER_MSEC 1000000ULL +#define TRISTAR_EMPTY 0 +#define POWER_SUPPLY_USB_TYPE_UNKNOWN 0 +#define TRISTAR_CHG_HANDOFF_CODE 0 +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_crit(...) ((void)0) +struct tristar_charger {bool usb_on,present;u64 usb_off_ns;int phase,source_ma,usb_ma,usb_type;char source[4];void *dev;}; +static int enables; +static void tristar_set_usb(struct tristar_charger *c,bool on){c->usb_on=on;enables+=on;} +static void tristar_no_source(struct tristar_charger *c,unsigned status,u64 now){} +static int tristar_set_code(struct tristar_charger *c,int code){return 0;} +'''+function(text,'tristar_release_usb')+'\nstatic void poll_empty(struct tristar_charger *chg,unsigned status,u64 now){\n'+empty+'\n}\n'+r''' +int main(void){ + struct tristar_charger c={.usb_on=false,.usb_off_ns=0}; + poll_empty(&c,0,5000*NSEC_PER_MSEC); + assert(c.usb_off_ns==5000*NSEC_PER_MSEC); + tristar_release_usb(&c,5010*NSEC_PER_MSEC,0x40);assert(!c.usb_on&&!enables); + tristar_release_usb(&c,5999*NSEC_PER_MSEC,0x40);assert(!c.usb_on&&!enables); + tristar_release_usb(&c,6000*NSEC_PER_MSEC,0x40);assert(c.usb_on&&enables==1); + c.usb_on=false;poll_empty(&c,0,9000*NSEC_PER_MSEC); + tristar_release_usb(&c,9001*NSEC_PER_MSEC,0x40);assert(!c.usb_on&&enables==1); +} +''') + + def test_manual_host_expires_but_accessory_host_does_not(self): + text=(S/'drivers/usb/roles/apple-s8000-usb-role.c').read_text() + execute(r''' +#include +#include +#include +#include +#ifndef EUCLEAN +#define EUCLEAN 117 +#endif +#define HZ 100 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define dev_warn(...) ((void)0) +struct work_struct {int x;};struct delayed_work {struct work_struct work;}; +enum usb_role {USB_ROLE_NONE,USB_ROLE_HOST,USB_ROLE_DEVICE}; +struct apple_usb_role {bool dying,monitor_fault,accessory_claim,accessory,accessory_power_prepared;enum usb_role role;int lock;struct delayed_work revert_work;void *dev;struct usb_role_switch *sw;}; +struct usb_role_switch {struct apple_usb_role *ur;}; +static struct apple_usb_role ur; +static struct usb_role_switch sw={.ur=&ur}; +static int system_wq,timer,delay,entered,entry_error; +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_delayed_work(w) ((struct delayed_work *)(w)) +static void mutex_lock(int *p){assert(!*p);*p=1;} +static void mutex_unlock(int *p){assert(*p);*p=0;} +static struct apple_usb_role *usb_role_switch_get_drvdata(struct usb_role_switch *s){return s->ur;} +static void cancel_delayed_work(struct delayed_work *w){timer=0;} +static void mod_delayed_work(int q,struct delayed_work *w,int ticks){timer=1;delay=ticks;} +static int apple_usb_role_enter_host(struct apple_usb_role *u,bool accessory){return entry_error;} +static void apple_usb_role_enter_device(struct apple_usb_role *u){entered++;} +static int usb_role_switch_set_role(struct usb_role_switch *s,enum usb_role r); +'''+function(text,'apple_usb_role_revert')+'\n'+function(text,'apple_usb_role_set')+r''' +static int usb_role_switch_set_role(struct usb_role_switch *s,enum usb_role r){return apple_usb_role_set(s,r);} +int main(void){ + ur.role=USB_ROLE_DEVICE;ur.sw=&sw; + assert(!apple_usb_role_set(&sw,USB_ROLE_HOST));assert(timer&&delay==60*HZ); + apple_usb_role_revert(&ur.revert_work.work);assert(ur.role==USB_ROLE_DEVICE&&!timer&&entered==1); + ur.accessory_claim=true;assert(!apple_usb_role_set(&sw,USB_ROLE_HOST));assert(ur.accessory&&!timer); + apple_usb_role_revert(&ur.revert_work.work);assert(ur.role==USB_ROLE_HOST&&entered==1); + apple_usb_role_set(&sw,USB_ROLE_DEVICE);apple_usb_role_set(&sw,USB_ROLE_HOST);assert(timer); + ur.accessory_claim=true;apple_usb_role_set(&sw,USB_ROLE_HOST);assert(!timer&&ur.accessory); + ur.dying=true;apple_usb_role_revert(&ur.revert_work.work);assert(ur.role==USB_ROLE_HOST); + assert(apple_usb_role_set(&sw,USB_ROLE_DEVICE)==-ESHUTDOWN); +} +''')