From 8d859a307786845137366569bfd192a87a18244a Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:57:28 -0600 Subject: [PATCH 01/16] build: produce static ARMv7 recovery tools Select ARM/v7 or ARM64 tool recipes explicitly and keep their outputs in separate caches. Pin Debian's ARMHF BusyBox package and verify each finished tool's ELF architecture and static linkage. Native builds require a host whose architecture matches the selected target. Assisted-by: LLM --- boot/initramfs/recovery/build-tools | 29 ++++-- .../recovery/build-tools-in-container | 13 ++- boot/initramfs/recovery/check-tools.py | 77 +++++++++++++++ boot/initramfs/recovery/install-toolchain | 2 +- boot/initramfs/recovery/upstream.lock | 4 + src/orchard/buildenv.py | 31 ++++--- tests/test_recovery_tools_arch.py | 93 +++++++++++++++++++ 7 files changed, 224 insertions(+), 25 deletions(-) create mode 100644 boot/initramfs/recovery/check-tools.py create mode 100644 tests/test_recovery_tools_arch.py diff --git a/boot/initramfs/recovery/build-tools b/boot/initramfs/recovery/build-tools index a5ea574e..dd8a7fa5 100755 --- a/boot/initramfs/recovery/build-tools +++ b/boot/initramfs/recovery/build-tools @@ -1,12 +1,20 @@ #!/usr/bin/env bash -# Acquire pinned public inputs and build static AArch64 recovery tools. +# Acquire pinned public inputs and build static recovery tools for the selected ARM architecture. set -euo pipefail # Recipes need only Python 3.11+ and its standard library; Orchard passes its own. python=${ORCHARD_PYTHON:-python3} +arch=arm64 +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) [[ $# -ge 2 ]] || { echo '--arch needs arm or arm64' >&2; exit 1; }; arch=$2; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done +case "$arch" in arm|arm64) ;; *) echo '--arch needs arm or arm64' >&2; exit 1 ;; esac here=$(cd "$(dirname "$0")" && pwd) -work="${OMA_RECOVERY_TOOLS_WORK:-$here/.work/tools}" +work="${OMA_RECOVERY_TOOLS_WORK:-$here/.work/tools-$arch}" mkdir -p "$work" -"$python" - "$here/upstream.lock" "$work" <<'PY' +"$python" - "$here/upstream.lock" "$work" "$arch" <<'PY' import hashlib import json import pathlib @@ -23,7 +31,8 @@ for component, filename in (('busybox', 'busybox.deb'), ('iw', 'iw.tar.xz'), ('zstd', 'zstd.tar.gz'), ('e2fsprogs', 'e2fsprogs.tar.gz')): - item = lock[component] + key = 'busybox_arm' if component == 'busybox' and sys.argv[3] == 'arm' else component + item = lock[key] target = work / filename if target.exists(): source = target.open('rb') @@ -49,9 +58,11 @@ for component, filename in (('busybox', 'busybox.deb'), pathlib.Path(temporary.name).rename(target) print(f'Verified {component}: {target}') PY -"$python" "$here/../../../src/orchard/buildenv.py" build --arch arm64 --file "$here/Containerfile" \ - --image localhost/orchard-hardware-recovery-tools:bookworm --context "$here" -"$python" "$here/../../../src/orchard/buildenv.py" run --arch arm64 \ - --bind "$work:/work:rw" --bind "$here:/recipe:ro" \ - --image localhost/orchard-hardware-recovery-tools:bookworm -- bash /recipe/build-tools-in-container +image=localhost/orchard-hardware-recovery-tools:bookworm +[[ "$arch" == arm64 ]] || image=localhost/orchard-hardware-recovery-tools-arm:bookworm +"$python" "$here/../../../src/orchard/buildenv.py" build --arch "$arch" --file "$here/Containerfile" \ + --image "$image" --context "$here" +"$python" "$here/../../../src/orchard/buildenv.py" run --arch "$arch" \ + --bind "$work:/work:rw" --bind "$here:/recipe:ro" --env "ORCHARD_RECOVERY_ARCH=$arch" \ + --image "$image" -- bash /recipe/build-tools-in-container echo "Recovery tools: $work/bin" diff --git a/boot/initramfs/recovery/build-tools-in-container b/boot/initramfs/recovery/build-tools-in-container index d6b04b48..8186df88 100755 --- a/boot/initramfs/recovery/build-tools-in-container +++ b/boot/initramfs/recovery/build-tools-in-container @@ -1,6 +1,9 @@ #!/bin/bash set -euo pipefail work=${ORCHARD_WORK:-/work} +recipe=${ORCHARD_RECIPE:-/recipe} +arch=${ORCHARD_RECOVERY_ARCH:-arm64} +case "$arch" in arm) deb_arch=armhf ;; arm64) deb_arch=arm64 ;; *) echo 'Unsupported recovery architecture' >&2; exit 1 ;; esac [[ ! -e "$work/bin" ]] || { echo 'Tool output exists; choose a fresh work directory' >&2; exit 1; } build=$(mktemp -d "$work/build.XXXXXX") mkdir "$build/bin" @@ -67,7 +70,7 @@ tar -xJf "$work/iw.tar.xz" -C "$build" tar -xzf "$work/zstd.tar.gz" -C "$build" make -C "$build/zstd-1.5.7/lib" -j2 libzstd.a CC=musl-gcc ZSTD_LEGACY_SUPPORT=0 $CC $CFLAGS -static -Wall -Wextra -pthread -I"$build/zstd-1.5.7/lib" -o "$build/bin/orchard-write" \ - "${ORCHARD_RECIPE:-/recipe}/orchard-write.c" "$build/zstd-1.5.7/lib/libzstd.a" + "$recipe/orchard-write.c" "$build/zstd-1.5.7/lib/libzstd.a" tar -xzf "$work/e2fsprogs.tar.gz" -C "$build" ( cd "$build/e2fsprogs-1.47.4" @@ -78,11 +81,13 @@ tar -xzf "$work/e2fsprogs.tar.gz" -C "$build" cp resize/resize2fs misc/tune2fs "$build/bin/" ) +[[ $(dpkg-deb -f "$work/busybox.deb" Architecture) == "$deb_arch" ]] || { + echo 'BusyBox package architecture differs from the target' >&2; exit 1; +} + dpkg-deb -x "$work/busybox.deb" "$build/package" cp "$build/package/bin/busybox" "$build/bin/busybox" chmod 755 "$build/bin/"* -for tool in "$build/bin/"*; do - file "$tool" | grep -q 'statically linked' || { echo "$tool is not static" >&2; exit 1; } -done +python3 "$recipe/check-tools.py" --arch "$arch" "$build/bin/"* mv "$build/bin" "$work/bin" echo "Static recovery tools copied to $work/bin" diff --git a/boot/initramfs/recovery/check-tools.py b/boot/initramfs/recovery/check-tools.py new file mode 100644 index 00000000..5ff6eb61 --- /dev/null +++ b/boot/initramfs/recovery/check-tools.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Check the recovery tools' target ISA and static ELF linkage without running them.""" +import argparse +from pathlib import Path +import stat +import struct + + +TARGETS = {'arm': (1, 40), 'arm64': (2, 183)} + + +def validate(data, arch): + if arch not in TARGETS: + raise ValueError('Unsupported recovery architecture: ' + arch) + elf_class, machine = TARGETS[arch] + if len(data) < 16 or data[:7] != b'\x7fELF' + bytes((elf_class, 1, 1)): + raise ValueError('ELF class or byte order differs from target ' + arch) + header = '<16sHHIIIIIHHHHHH' if elf_class == 1 else '<16sHHIQQQIHHHHHH' + if len(data) < struct.calcsize(header): + raise ValueError('Incomplete ELF header') + fields = struct.unpack_from(header, data) + if fields[1] not in (2, 3) or fields[2] != machine or fields[3] != 1: + raise ValueError('ELF machine or executable type differs from target ' + arch) + if fields[8] != struct.calcsize(header): + raise ValueError('Invalid ELF header size') + if arch == 'arm' and (fields[7] & 0xff000000 != 0x05000000 or + fields[7] & 0x600 != 0x400): + raise ValueError('ARM recovery requires EABI5 hard-float executables') + segment = ' len(data): + raise ValueError('Invalid ELF program header table') + dynamic = [] + for index in range(count): + values = struct.unpack_from(segment, data, offset + index * stride) + kind = values[0] + start, size = (values[1], values[4]) if elf_class == 1 else (values[2], values[5]) + if start + size > len(data): + raise ValueError('ELF segment exceeds the file') + if kind == 3: + raise ValueError('Recovery tools must be static: PT_INTERP is present') + if kind == 2: + dynamic.append((start, size)) + entry = ' 64 * 1024 * 1024: + raise ValueError('Expected a bounded regular executable: ' + str(path)) + validate(path.read_bytes(), arch) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--arch', choices=TARGETS, required=True) + parser.add_argument('tools', nargs='+', type=Path) + args = parser.parse_args() + for path in args.tools: + try: + check(path, args.arch) + except (OSError, ValueError, struct.error) as error: + parser.exit(1, str(path) + ': ' + str(error) + '\n') + + +if __name__ == '__main__': + main() diff --git a/boot/initramfs/recovery/install-toolchain b/boot/initramfs/recovery/install-toolchain index e0375165..6c33aaad 100644 --- a/boot/initramfs/recovery/install-toolchain +++ b/boot/initramfs/recovery/install-toolchain @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -packages=(bison build-essential bzip2 ca-certificates dpkg file flex linux-libc-dev musl-tools pkg-config xz-utils) +packages=(bison build-essential bzip2 ca-certificates dpkg file flex linux-libc-dev musl-tools pkg-config python3 xz-utils) if [[ ${1:-} == --packages ]]; then printf '%s\n' "${packages[@]}"; exit 0; fi apt-get update apt-get install -y --no-install-recommends "${packages[@]}" diff --git a/boot/initramfs/recovery/upstream.lock b/boot/initramfs/recovery/upstream.lock index c0cd9e62..0a8e32a1 100644 --- a/boot/initramfs/recovery/upstream.lock +++ b/boot/initramfs/recovery/upstream.lock @@ -26,5 +26,9 @@ "e2fsprogs": { "url": "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.47.4/e2fsprogs-1.47.4.tar.gz", "sha256": "da274408bebbfd13a5a2fc3cfc66e3ffff17c48534673aa67f88d49b99123b96" + }, + "busybox_arm": { + "url": "https://deb.debian.org/debian/pool/main/b/busybox/busybox-static_1.35.0-4+deb12u1+b1_armhf.deb", + "sha256": "2f68dbeaebeda49f3cde34ac421d5add85fb1c837ff7c4a4971945340d9d66ff" } } diff --git a/src/orchard/buildenv.py b/src/orchard/buildenv.py index 332c7e04..a4c1d763 100644 --- a/src/orchard/buildenv.py +++ b/src/orchard/buildenv.py @@ -2,7 +2,7 @@ Recipes address inputs through ORCHARD_* path variables. Native mode provides real paths; local mode provides the corresponding container mount paths. Native -mode is not a sandbox and is only for trusted, disposable Linux ARM64 workers. +mode is not a sandbox and is only for trusted, disposable Linux ARM workers. """ import argparse import os @@ -17,8 +17,8 @@ def native(): if value not in ('0', '1'): raise ValueError('ORCHARD_BUILD_NATIVE must be 0 or 1') if value == '1': - if platform.system() != 'Linux' or platform.machine() not in ('aarch64', 'arm64'): - raise ValueError('Native builds require Linux ARM64') + if platform.system() != 'Linux' or platform.machine().lower() not in ('aarch64', 'arm64', 'armv7l', 'armv7', 'armhf'): + raise ValueError('Native builds require Linux ARM64 or ARMv7') if not os.environ.get('ORCHARD_BUILD_ENV_ID'): raise ValueError('Native builds require ORCHARD_BUILD_ENV_ID') return True @@ -26,20 +26,31 @@ def native(): # Podman's names for host machines it can build for. -HOST_ARCHES = {'x86_64': 'amd64', 'amd64': 'amd64', 'aarch64': 'arm64', 'arm64': 'arm64'} +HOST_ARCHES = {'x86_64': 'amd64', 'amd64': 'amd64', 'aarch64': 'arm64', 'arm64': 'arm64', + 'armv7l': 'arm', 'armv7': 'arm', 'armhf': 'arm'} + + +def architecture_arguments(arch): + # OCI's ARM architecture needs a variant; plain arm can resolve to ARMv5. + return ['--platform', 'linux/arm/v7'] if arch == 'arm' else (['--arch', arch] if arch else []) + + +def check_native_architecture(arch): + if arch is not None and HOST_ARCHES.get(platform.machine().lower()) != arch: + raise ValueError(f'Native target {arch} differs from host architecture ' + f'{HOST_ARCHES.get(platform.machine().lower(), platform.machine())}') def build(image, containerfile, context, arch=None): if native(): - if arch not in (None, 'arm64'): - raise ValueError('Native builds require ARM64 inputs') + check_native_architecture(arch) print('Using provisioned native environment: ' + identity(image), flush=True) else: # Always name the architecture: recipes share base images such as # debian:bookworm, and an arm64 recipe re-points that local tag, which # a host-architecture build would otherwise silently inherit. arch = arch or HOST_ARCHES.get(platform.machine().lower()) - argv = ['podman', 'build'] + (['--arch', arch] if arch else []) + argv = ['podman', 'build', *architecture_arguments(arch)] subprocess.run([*argv, '-f', str(containerfile), '-t', image, str(context)], check=True) @@ -63,8 +74,7 @@ def translate(value, mounts): def run(image, command, mounts=(), *, env=None, cwd=None, root=False, network=False, arch=None, memory=None): """MEMORY: keep ORCHARD_WORK in memory of this size where a container runs it (e.g. '12g').""" if native(): - if arch not in (None, 'arm64'): - raise ValueError('Native builds require ARM64 inputs') + check_native_architecture(arch) with tempfile.TemporaryDirectory(prefix='orchard-recipe-') as temporary: work, tmp = Path(temporary) / 'work', Path(temporary) / 'tmp' work.mkdir() @@ -77,8 +87,7 @@ def run(image, command, mounts=(), *, env=None, cwd=None, root=False, network=Fa env={**os.environ, **paths, **extra}, check=True) else: argv = ['podman', 'run', '--rm', '--userns', 'keep-id:uid=0,gid=0' if root else 'keep-id'] - if arch: - argv += ['--arch', arch] + argv += architecture_arguments(arch) if not network: argv += ['--network', 'none'] if memory: diff --git a/tests/test_recovery_tools_arch.py b/tests/test_recovery_tools_arch.py new file mode 100644 index 00000000..8277841f --- /dev/null +++ b/tests/test_recovery_tools_arch.py @@ -0,0 +1,93 @@ +"""Recovery tools must match their target ISA before entering an image.""" +import importlib.util +import json +import os +from pathlib import Path +import struct +import unittest +from unittest import mock + +from orchard import buildenv + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('check_recovery_tools', + ROOT / 'boot/initramfs/recovery/check-tools.py') +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +def elf(arch, *, kind=1, dynamic_tag=None, flags=0x05000400): + arm = arch == 'arm' + header_size, ph_size = (52, 32) if arm else (64, 56) + ident = b'\x7fELF' + bytes((1 if arm else 2, 1, 1)) + bytes(9) + dynamic = struct.pack(' Date: Fri, 6 Nov 2026 11:57:46 -0600 Subject: [PATCH 02/16] boot: build Cascadia payloads from source Use board-owned handoff data to compile the ARM32 staging loader. Preserve Cascadia's cache/MMU handoff and require executable code at the upload address. A checked configuration places the supplied external initramfs in reserved RAM before entering Linux through its edited DTB. Bound the zImage decompressor workspace and preserve the kernel bytes. Add synthetic layout tests and a Linux fixture that exercises the actual loader C copy and rejection paths. Source provenance is recorded in NOTICE. The external-initramfs handoff still requires physical qualification. Assisted-by: LLM --- boot/cascadia/NOTICE | 14 ++ boot/cascadia/README.md | 37 +++ boot/cascadia/build | 50 ++++ boot/cascadia/compile | 14 ++ boot/cascadia/linux-boot.ld.in | 22 ++ boot/cascadia/linux_boot.c | 87 +++++++ boot/cascadia/linux_boot_start.S | 135 ++++++++++ chips/s5l8942x.toml | 1 + devices/p105/README.md | 11 + devices/p105/cascadia.toml | 12 + devices/p105/device.toml | 1 + src/orchard/hardware.py | 10 + src/orchard/image/cascadia.py | 393 ++++++++++++++++++++++++++++++ tests/fixtures/cascadia-runtime.c | 101 ++++++++ tests/test_cascadia_loader.py | 239 ++++++++++++++++++ 15 files changed, 1127 insertions(+) create mode 100644 boot/cascadia/NOTICE create mode 100644 boot/cascadia/README.md create mode 100755 boot/cascadia/build create mode 100755 boot/cascadia/compile create mode 100644 boot/cascadia/linux-boot.ld.in create mode 100644 boot/cascadia/linux_boot.c create mode 100644 boot/cascadia/linux_boot_start.S create mode 100644 devices/p105/cascadia.toml create mode 100644 src/orchard/image/cascadia.py create mode 100644 tests/fixtures/cascadia-runtime.c create mode 100644 tests/test_cascadia_loader.py diff --git a/boot/cascadia/NOTICE b/boot/cascadia/NOTICE new file mode 100644 index 00000000..a20b5572 --- /dev/null +++ b/boot/cascadia/NOTICE @@ -0,0 +1,14 @@ +The ARMv7 cache-maintenance and Linux handoff sequence is derived from the +Project Cascadia A5 staging loader: + + https://github.com/rzxvx/Project-Cascadia + reference revision: 091d441f19e101699f7e1835a80ccb48d6e54eec + license: GPL-2.0 + +The local P105 work fixed the executable entry by discarding ARM unwind +metadata and asserting the upload address in the linker script. + +This producer keeps those entry safeguards. Hardware addresses come from +the device's profile. Orchard's configuration record and DTB editor add +bounded external-initramfs placement; that extension requires a fresh +hardware boot before it is described as working. diff --git a/boot/cascadia/README.md b/boot/cascadia/README.md new file mode 100644 index 00000000..2a5eccd6 --- /dev/null +++ b/boot/cascadia/README.md @@ -0,0 +1,37 @@ +# ARM32 iBEC staging payload + +`build --profile devices/p105/cascadia.toml --output DIR` builds a loader +with Orchard's kernel toolchain through `buildenv`. The output directory +contains `loader.bin`, its ELF and a source/profile/toolchain receipt. +The ELF check requires executable ARM code at the upload address and checks +every allocated section against the raw bytes. ARM unwind tables are +discarded so they cannot displace the entry instruction. + +`python -m orchard.image.cascadia --profile devices/p105/cascadia.toml +--loader DIR/loader.bin --loader-elf DIR/loader.elf --kernel zImage +--dtb s5l8942x-p105.dtb --initramfs recovery.cpio.gz --bootargs 'rdinit=/init' +--output linux.bin` assembles one device's payload. Kernel bytes remain +unchanged. The DTB editor preserves properties while updating `/chosen` +boot arguments and its initrd addresses, then reserves that destination. + +The layout is a checked loader followed by padding to 32 KiB, the zImage, +the edited DTB and the supplied initramfs, with each data address aligned to +eight bytes. The loader validates its configuration checksum and bounds +before copying the initramfs to the profile's reserved high-memory range. +It then performs the Cascadia cache/MMU handoff with the DTB in `r2`. +The fixed loader stack uses the gap below offset `0x7000`; code must finish +below `0x6000`, and the configuration starts at `0x7f00`. + +The zImage must carry the v6.12 ARM size table and end on an eight-byte +boundary. Its DTB follows immediately so the appended-DTB decompressor can +relocate it. Assembly bounds the inflated kernel together with the relocated +decompressor's workspace below the initramfs destination; unknown table +formats and overlapping layouts are refused. + +The receipt describes every byte range and its hash. It records +`architecture = "arm"` and `initramfs_encoding = "cascadia-dtb"`, allowing a +distribution to reject an image assembled with another architecture or +payload format. This external-initramfs extension awaits hardware testing; +the earlier P105 boot embedded its initramfs in the kernel. + +See `NOTICE` for source provenance. This producer contacts no device. diff --git a/boot/cascadia/build b/boot/cascadia/build new file mode 100755 index 00000000..10e647c6 --- /dev/null +++ b/boot/cascadia/build @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Build the ARM32 staging loader from source and check its executable entry.""" +import argparse +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'src')) +from orchard import buildenv +from orchard.image.cascadia import digest, profile, verify_loader + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--profile', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + described = profile(args.profile) + out = args.output.resolve() + if out.is_symlink() or (out.exists() and (not out.is_dir() or any(out.iterdir()))): + parser.error('The loader output must be a new or empty directory') + out.mkdir(parents=True, exist_ok=True) + (out / 'cascadia-generated.h').write_text(''.join( + f'#define {name.upper()} {hex(value)}\n' for name, value in sorted(described.items()) if name != 'schema')) + template = (ROOT / 'boot/cascadia/linux-boot.ld.in').read_text() + (out / 'linux-boot.ld').write_text(template.replace('@LOAD_ADDRESS@', hex(described['load_address']))) + image = 'localhost/orchard-hardware-kernel-toolchain:bookworm' + buildenv.build(image, ROOT / 'kernel/Containerfile', ROOT / 'kernel') + buildenv.run(image, ['sh', '/src/boot/cascadia/compile', '/out'], + [(ROOT, '/src', True), (out, '/out', False)]) + binary, elf = (out / 'loader.bin').read_bytes(), (out / 'loader.elf').read_bytes() + verify_loader(binary, described, elf) + source = ROOT / 'boot/cascadia' + receipt = {'schema': 1, 'kind': 'cascadia-loader', 'architecture': 'arm', + 'loader_format': 'cascadia', 'profile_sha256': digest(args.profile.read_bytes()), + 'toolchain': buildenv.identity(image), + 'sources': {path.name: digest(path.read_bytes()) for path in sorted(source.iterdir()) if path.is_file()}, + 'loader': {'bytes': len(binary), 'sha256': digest(binary)}, + 'elf': {'bytes': len(elf), 'sha256': digest(elf)}, + 'entry': described['load_address'], 'hardware_boot_verified': False} + (out / 'loader.json').write_text(json.dumps(receipt, indent=2, sort_keys=True) + '\n') + for name in ('linux_boot.o', 'linux_boot_start.o', 'cascadia-generated.h', 'linux-boot.ld'): + (out / name).unlink() + print(f'Loader: {out / "loader.bin"}') + + +if __name__ == '__main__': + main() diff --git a/boot/cascadia/compile b/boot/cascadia/compile new file mode 100755 index 00000000..c824c114 --- /dev/null +++ b/boot/cascadia/compile @@ -0,0 +1,14 @@ +#!/usr/bin/env sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +out=$1 +src=${ORCHARD_SRC:-$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)} +clang-19 --target=armv7a-none-eabi -march=armv7-a -marm -mfloat-abi=soft \ + -ffreestanding -fno-builtin -fno-stack-protector -fno-unwind-tables \ + -fno-asynchronous-unwind-tables -Os -Wall -Wextra -Werror \ + -I "$out" -c "$src/boot/cascadia/linux_boot.c" -o "$out/linux_boot.o" +clang-19 --target=armv7a-none-eabi -march=armv7-a -marm -I "$out" \ + -c "$src/boot/cascadia/linux_boot_start.S" -o "$out/linux_boot_start.o" +ld.lld-19 --build-id=none -T "$out/linux-boot.ld" \ + "$out/linux_boot_start.o" "$out/linux_boot.o" -o "$out/loader.elf" +llvm-objcopy-19 -O binary "$out/loader.elf" "$out/loader.bin" diff --git a/boot/cascadia/linux-boot.ld.in b/boot/cascadia/linux-boot.ld.in new file mode 100644 index 00000000..aa0ddcbc --- /dev/null +++ b/boot/cascadia/linux-boot.ld.in @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +ENTRY(_start) +SECTIONS +{ + . = @LOAD_ADDRESS@; + .text : { + KEEP(*(.text.start)) + *(.text .text.*) + *(.rodata .rodata.*) + *(.data .data.*) + } + ASSERT(. <= @LOAD_ADDRESS@ + 0x6000, "loader overlaps its stack") + . = @LOAD_ADDRESS@ + 0x7f00; + .loader_config : { KEEP(*(.loader_config)) } + ASSERT(SIZEOF(.loader_config) == 68, "loader configuration changed") + .ARM.attributes 0 : { *(.ARM.attributes) } + /DISCARD/ : { + *(.comment) *(.note*) *(.eh_frame*) + *(.ARM.exidx*) *(.ARM.extab*) + } +} +ASSERT(_start == @LOAD_ADDRESS@, "loader entry differs from the upload address") diff --git a/boot/cascadia/linux_boot.c b/boot/cascadia/linux_boot.c new file mode 100644 index 00000000..16526a73 --- /dev/null +++ b/boot/cascadia/linux_boot.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* ARM32 iBEC handoff. See NOTICE for the Cascadia source and local changes. */ +typedef unsigned int u32; +typedef unsigned char u8; + +#include "cascadia-generated.h" + +struct config { + u8 magic[4]; + u32 version, header_size, load_address, kernel_offset; + u32 dtb_offset, dtb_size, initrd_offset, initrd_size; + u32 initrd_address, image_size, watchdog_address, l2_address; + u32 ram_start, ram_end, initrd_limit, crc32; +}; + +/* The assembler fills the dynamic fields and checksum before upload. */ +__attribute__((section(".loader_config"), used)) +volatile const struct config cascadia_config = { + { 'C', 'B', '3', '2' }, 1, sizeof(struct config), LOAD_ADDRESS, + KERNEL_OFFSET, 0, 0, 0, 0, INITRD_ADDRESS, 0, WATCHDOG_ADDRESS, + L2_ADDRESS, RAM_START, RAM_END, INITRD_LIMIT, 0, +}; + +static int range(u32 start, u32 size, u32 first, u32 end) +{ + return start >= first && start < end && size && size <= end - start; +} + +static u32 checksum(void) +{ + volatile const u8 *p = (volatile const u8 *)&cascadia_config; + u32 crc = ~0u, i, bit; + for (i = 0; i < sizeof(struct config) - sizeof(u32); i++) { + crc ^= p[i]; + for (bit = 0; bit < 8; bit++) + crc = (crc >> 1) ^ (0xedb88320u & (0u - (crc & 1))); + } + return ~crc; +} + +static u32 big_endian(volatile const u8 *p) +{ + return ((u32)p[0] << 24) | ((u32)p[1] << 16) | + ((u32)p[2] << 8) | p[3]; +} + +u32 cascadia_main(void) +{ + volatile const struct config *c = &cascadia_config; + volatile const u8 *dtb, *from; + volatile u8 *to; + u32 i; + + if (c->magic[0] != 'C' || c->magic[1] != 'B' || + c->magic[2] != '3' || c->magic[3] != '2' || + c->version != 1 || c->header_size != sizeof(struct config) || + c->load_address != LOAD_ADDRESS || c->kernel_offset != KERNEL_OFFSET || + c->watchdog_address != WATCHDOG_ADDRESS || c->l2_address != L2_ADDRESS || + c->ram_start != RAM_START || c->ram_end != RAM_END || + c->initrd_limit != INITRD_LIMIT || + checksum() != c->crc32) + return 0; + if (!range(LOAD_ADDRESS, c->image_size, RAM_START, RAM_END) || + c->dtb_offset < KERNEL_OFFSET || c->dtb_size < 40 || + c->dtb_offset >= c->image_size || + c->dtb_size > c->image_size - c->dtb_offset || + c->initrd_offset < c->dtb_offset + c->dtb_size || + c->initrd_offset >= c->image_size || + c->initrd_size > c->image_size - c->initrd_offset || + c->initrd_address != INITRD_ADDRESS || + !range(INITRD_ADDRESS, c->initrd_size, RAM_START, INITRD_LIMIT) || + INITRD_ADDRESS < LOAD_ADDRESS + c->image_size) + return 0; + dtb = (volatile const u8 *)(LOAD_ADDRESS + c->dtb_offset); + if (big_endian(dtb) != 0xd00dfeedu || big_endian(dtb + 4) != c->dtb_size) + return 0; + if (WATCHDOG_ADDRESS) { + volatile u32 *wdt = (volatile u32 *)WATCHDOG_ADDRESS; + for (i = 0; i < 4; i++) + wdt[i] = 0; + } + from = (volatile const u8 *)(LOAD_ADDRESS + c->initrd_offset); + to = (volatile u8 *)INITRD_ADDRESS; + for (i = 0; i < c->initrd_size; i++) + to[i] = from[i]; + return LOAD_ADDRESS + c->dtb_offset; +} diff --git a/boot/cascadia/linux_boot_start.S b/boot/cascadia/linux_boot_start.S new file mode 100644 index 00000000..1052e304 --- /dev/null +++ b/boot/cascadia/linux_boot_start.S @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Derived from Project Cascadia; see NOTICE. */ +#include "cascadia-generated.h" + + .arch armv7-a + .syntax unified + .arm + + .equ KERNEL_ENTRY, LOAD_ADDRESS + KERNEL_OFFSET + + .section .text.start, "ax" + .global _start + .global cascadia_main + .global linux_boot_jump +_start: + cpsid if + msr cpsr_c, #0xd3 + + ldr sp, =LOAD_ADDRESS + 0x7000 + bl cascadia_main + /* main validates the image and returns DTB physical address. */ + cmp r0, #0 + beq 1f + b linux_boot_jump +1: wfi + b 1b + +/* + * r0 = DTB physical address + * Tear down MMU/caches, then bx KERNEL_ENTRY with + * r0=0, r1=~0, r2=DTB + */ +linux_boot_jump: + mov r4, r0 /* save DTB */ + + bl v7_flush_dcache_all + bl l2x0_flush_all + + mrc p15, 0, r0, c1, c0, 0 + bic r0, r0, #(1 << 0) /* MMU */ + bic r0, r0, #(1 << 2) /* D-cache */ + bic r0, r0, #(1 << 12) /* I-cache */ + bic r0, r0, #(1 << 11) /* Z */ + mcr p15, 0, r0, c1, c0, 0 + isb + + mov r0, #0 + mcr p15, 0, r0, c7, c5, 0 + mcr p15, 0, r0, c7, c5, 6 + mcr p15, 0, r0, c8, c7, 0 + dsb + isb + + mov r0, #0 + mvn r1, #0 + mov r2, r4 + ldr r3, =KERNEL_ENTRY + bx r3 + +1: wfi + b 1b + +v7_flush_dcache_all: + push {r4-r11, lr} + dmb + mrc p15, 1, r0, c0, c0, 1 + ands r3, r0, #0x7000000 + mov r3, r3, lsr #23 + beq .Lfinished + mov r10, #0 +.Lloop_level: + add r2, r10, r10, lsr #1 + mov r1, r0, lsr r2 + and r1, r1, #7 + cmp r1, #2 + blt .Lskip_level + mcr p15, 2, r10, c0, c0, 0 + isb + mrc p15, 1, r1, c0, c0, 0 + and r2, r1, #7 + add r2, r2, #4 + movw r4, #0x3ff + ands r4, r4, r1, lsr #3 + clz r5, r4 + movw r7, #0x7fff + ands r7, r7, r1, lsr #13 +.Lloop_way: + mov r9, r7 +.Lloop_set: + orr r11, r10, r4, lsl r5 + orr r11, r11, r9, lsl r2 + mcr p15, 0, r11, c7, c14, 2 + subs r9, r9, #1 + bge .Lloop_set + subs r4, r4, #1 + bge .Lloop_way +.Lskip_level: + add r10, r10, #2 + cmp r3, r10 + bgt .Lloop_level +.Lfinished: + mov r10, #0 + mcr p15, 2, r10, c0, c0, 0 + dsb + isb + pop {r4-r11, pc} + +l2x0_flush_all: + push {r0, r1, r2, lr} + ldr r0, =L2_ADDRESS + cmp r0, #0 + beq 2f + ldr r1, [r0] + lsr r2, r1, #6 + and r2, r2, #0x3f + cmp r2, #3 + cmpne r2, #2 + cmpne r2, #1 + bne 2f + ldr r1, [r0, #0x100] + tst r1, #1 + beq 2f + mov r1, #0xff00 + orr r1, r1, #0xff + str r1, [r0, #0x7fc] + ldr r2, =1000000 +1: ldr r1, [r0, #0x7fc] + tst r1, #1 + beq 2f + subs r2, r2, #1 + bne 1b +3: wfi + b 3b +2: dsb + pop {r0, r1, r2, pc} diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index 50ba0c2d..1df5e145 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -14,3 +14,4 @@ stages = ["iBSS", "iBEC"] platform = "P105 on a Cascadia-based build" stages = "Project Cascadia P105 boot chain" reference_version = "Project Cascadia builds its A5 boot chain from 8.4.1 / 12H321" +cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff awaits hardware qualification" diff --git a/devices/p105/README.md b/devices/p105/README.md index a39986c9..f66904bd 100644 --- a/devices/p105/README.md +++ b/devices/p105/README.md @@ -26,3 +26,14 @@ the IPSW members `058-24036-023.dmg`, `kernelcache.release.p105`, `Firmware/dfu/ The touch calibration is per unit and private: `lib/firmware/mtcal.bin`, the unit's syscfg `MtCl`, captured from its jailbroken iOS. It is never part of a public recipe or release. + +## ARMv7 recovery assembly + +The platform's `cascadia` payload producer builds the staging loader from +source with `cascadia.toml`. Select this board explicitly when resolving a +release's inputs: `python -m orchard.inputs --device p105 --output inputs.json`. +The profile identifies the memory window and handoff registers for this board. +The generated loader checks its configuration before copying the recovery +initramfs into the reserved high-memory range. The DTB names that initramfs. +This extension needs a hardware boot of the final producer revision; the +earlier installed P105 boot used an initramfs compiled into its kernel. diff --git a/devices/p105/cascadia.toml b/devices/p105/cascadia.toml new file mode 100644 index 00000000..95f23a4a --- /dev/null +++ b/devices/p105/cascadia.toml @@ -0,0 +1,12 @@ +# P105 handoff addresses from the Cascadia-derived staging loader used for the +# owner's successful iBEC boot. The external-initramfs placement is new and +# requires a fresh hardware test with this producer. +schema = 1 +load_address = 0x80000000 +kernel_offset = 0x8000 +watchdog_address = 0x0f103020 +l2_address = 0x3e000000 +ram_start = 0x80000000 +ram_end = 0xa0000000 +initrd_address = 0x98000000 +initrd_limit = 0x9e000000 diff --git a/devices/p105/device.toml b/devices/p105/device.toml index f87d1027..bd06785c 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -13,6 +13,7 @@ chip = 0x8942 board = 0x0a [boot] +# Source-built Cascadia payload profile: cascadia.toml. External-initramfs handoff awaits testing. methods = ["jailbreak-kdfu", "recovery-autogo"] entries = ["pico-checkm8", "kdfu"] # The CLI consumes an owner-private boot manifest. Public launch assets await release integration. diff --git a/src/orchard/hardware.py b/src/orchard/hardware.py index e4d5b010..9eea2e88 100644 --- a/src/orchard/hardware.py +++ b/src/orchard/hardware.py @@ -59,6 +59,16 @@ def device_path(profile, root=None): return path +def payload_profile(profile, kind, root=None): + """The board-owned profile for a source-built payload loader.""" + if kind not in ('cascadia',): + raise ValueError('Unsupported payload profile: ' + str(kind)) + path = device_path(profile, root).with_name(kind + '.toml') + if not path.is_file() or path.is_symlink(): + raise ValueError(f'{profile} has no {kind} handoff profile') + return path + + def profiles(root=None): """Every device this hardware data describes.""" devices = Path(root or data_root()) / 'devices' diff --git a/src/orchard/image/cascadia.py b/src/orchard/image/cascadia.py new file mode 100644 index 00000000..6b02caff --- /dev/null +++ b/src/orchard/image/cascadia.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Assemble an ARM32 iBEC payload with an external initramfs described by its DTB. + +The loader occupies the first 32 KiB. Its checked configuration tells the +loader where to copy the initramfs before entering the unchanged zImage. +This module uses only the standard library and also runs by file path. +""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +import tomllib +import zlib + + +CONFIG_OFFSET = 0x7f00 +CONFIG = struct.Struct('<4s16I') +CONFIG_FIELDS = ('magic', 'version', 'header_size', 'load_address', 'kernel_offset', + 'dtb_offset', 'dtb_size', 'initrd_offset', 'initrd_size', + 'initrd_address', 'image_size', 'watchdog_address', 'l2_address', + 'ram_start', 'ram_end', 'initrd_limit', 'crc32') +PROFILE_FIELDS = frozenset(('schema', 'load_address', 'kernel_offset', 'watchdog_address', + 'l2_address', 'ram_start', 'ram_end', 'initrd_address', 'initrd_limit')) +FDT_HEADER = struct.Struct('>10I') + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def aligned(value, size=8): + return (value + size - 1) & -size + + +def profile(path): + return validate_profile(tomllib.loads(Path(path).read_text())) + + +def validate_profile(result): + if set(result) != PROFILE_FIELDS or result['schema'] != 1: + raise ValueError('Expected the schema-1 Cascadia handoff profile') + if any(type(value) is not int or not 0 <= value <= 0xffffffff for value in result.values()): + raise ValueError('Handoff addresses must be unsigned 32-bit integers') + first, end = result['ram_start'], result['ram_end'] + base, target, limit = result['load_address'], result['initrd_address'], result['initrd_limit'] + if (result['kernel_offset'] != 0x8000 or base != first or base % 0x8000000 or + not first <= base < base + 0x8000 < target < limit <= end or target % 4096): + raise ValueError('Handoff memory ranges overlap or have invalid alignment') + if any(result[key] % 4 for key in ('watchdog_address', 'l2_address')): + raise ValueError('Handoff register addresses must be word aligned') + return result + + +def configuration(raw): + if len(raw) < CONFIG_OFFSET + CONFIG.size: + raise ValueError('Loader lacks its configuration record') + result = dict(zip(CONFIG_FIELDS, CONFIG.unpack_from(raw, CONFIG_OFFSET))) + if (result['magic'], result['version'], result['header_size']) != (b'CB32', 1, CONFIG.size): + raise ValueError('Unknown loader configuration format') + return result + + +def verify_loader(binary, described, elf=None): + """Reject metadata before ARM code and a loader built for another profile.""" + if (not CONFIG_OFFSET + CONFIG.size <= len(binary) <= 0x8000 or + binary[:4] != bytes.fromhex('c0000cf1')): + raise ValueError('Expected the ARM CPSID instruction at loader byte zero') + config = configuration(binary) + for field in PROFILE_FIELDS - {'schema'}: + if config[field] != described[field]: + raise ValueError('Loader configuration differs from the handoff profile: ' + field) + if any(config[field] for field in ('dtb_offset', 'dtb_size', 'initrd_offset', 'initrd_size', + 'image_size', 'crc32')): + raise ValueError('Expected a pristine loader before payload assembly') + if elf is None: + return + if len(elf) < 52 or elf[:7] != b'\x7fELF\x01\x01\x01': + raise ValueError('Expected little-endian ELF32') + if struct.unpack_from(' len(elf): + raise ValueError('Invalid loader section table') + allocated = [] + for index in range(count): + _, kind, flags, address, source, length, *_ = struct.unpack_from('<10I', elf, offset + index * size) + if not flags & 2 or not length: + continue + relative = address - described['load_address'] + if (kind == 8 or relative < 0 or source + length > len(elf) or + relative + length > len(binary) or binary[relative:relative + length] != elf[source:source + length]): + raise ValueError('ELF allocated section differs from the raw loader') + if relative != CONFIG_OFFSET and relative + length > 0x6000: + raise ValueError('ELF section overlaps the loader stack or configuration') + allocated.append((relative, flags, length)) + if not allocated or min(allocated)[0] != 0 or not min(allocated)[1] & 4: + raise ValueError('Loader starts with non-executable metadata') + if not any(start == CONFIG_OFFSET and length == CONFIG.size for start, _, length in allocated): + raise ValueError('ELF lacks the configuration section') + + +def _cstring(data, offset, end): + if not 0 <= offset < end: + raise ValueError('FDT string offset exceeds its block') + stop = data.find(b'\0', offset, end) + if stop < 0: + raise ValueError('Unterminated FDT string') + try: + return data[offset:stop].decode('ascii'), stop + 1 + except UnicodeDecodeError as error: + raise ValueError('FDT names must be ASCII') from error + + +def read_fdt(data): + """Return a checked tree and reservation list, preserving every property.""" + if len(data) < FDT_HEADER.size: + raise ValueError('Truncated FDT header') + magic, total, tree_at, strings_at, reserve_at, version, last, cpu, strings_size, tree_size = FDT_HEADER.unpack_from(data) + if magic != 0xd00dfeed or total != len(data) or version < 17 or last > 17: + raise ValueError('Unsupported FDT header') + if (tree_at % 4 or reserve_at % 8 or + tree_size <= 0 or any(start < 40 or size < 0 or start + size > total + for start, size in ((tree_at, tree_size), (strings_at, strings_size))) or + not 40 <= reserve_at <= total - 16): + raise ValueError('Invalid FDT block bounds') + intervals = [(tree_at, tree_at + tree_size), (strings_at, strings_at + strings_size)] + if max(intervals[0][0], intervals[1][0]) < min(intervals[0][1], intervals[1][1]): + raise ValueError('FDT blocks overlap') + reservations, at = [], reserve_at + while True: + if at + 16 > total or any(max(at, start) < min(at + 16, end) for start, end in intervals): + raise ValueError('FDT reservation map is truncated or overlaps another block') + address, length = struct.unpack_from('>QQ', data, at) + at += 16 + if address == length == 0: + break + if not length or address + length > 1 << 64: + raise ValueError('Invalid FDT reservation') + reservations.append((address, length)) + names = data[strings_at:strings_at + strings_size] + at, end, stack, root, finished = tree_at, tree_at + tree_size, [], None, False + while at < end: + if at + 4 > end: + raise ValueError('Truncated FDT token') + token = struct.unpack_from('>I', data, at)[0] + at += 4 + if token == 1: + name, at = _cstring(data, at, end) + at = aligned(at, 4) + if at > end or len(stack) >= 128 or '/' in name: + raise ValueError('Invalid FDT node') + node = {'name': name, 'properties': {}, 'children': []} + if stack: + if not name or any(child['name'] == name for child in stack[-1]['children']): + raise ValueError('Duplicate or empty FDT child node') + stack[-1]['children'].append(node) + elif root is not None or name: + raise ValueError('Expected exactly one unnamed FDT root') + else: + root = node + stack.append(node) + elif token == 2: + if not stack: + raise ValueError('Unbalanced FDT end-node') + stack.pop() + elif token == 3: + if not stack or at + 8 > end or stack[-1]['children']: + raise ValueError('Invalid FDT property position') + length, name_at = struct.unpack_from('>II', data, at) + at += 8 + name, _ = _cstring(names, name_at, len(names)) + if not name or name in stack[-1]['properties'] or at + length > end: + raise ValueError('Invalid or repeated FDT property') + stack[-1]['properties'][name] = data[at:at + length] + at = aligned(at + length, 4) + elif token == 4: + continue + elif token == 9: + if stack or root is None or at != end: + raise ValueError('FDT ends before its nodes or has trailing tokens') + finished = True + break + else: + raise ValueError('Unknown FDT token') + if not finished: + raise ValueError('FDT lacks its end token') + return root, reservations, cpu + + +def check_compatible(data, expected): + """Bind a board profile to the supplied DTB before payload assembly.""" + tree, _, _ = read_fdt(data) + value = tree['properties'].get('compatible', b'') + if not value or not value.endswith(b'\0'): + raise ValueError('Cascadia DTB has no complete compatible list') + try: + observed = set(value[:-1].decode('ascii').split('\0')) + except UnicodeDecodeError as error: + raise ValueError('Cascadia DTB compatible is invalid') from error + if not observed.intersection(expected): + raise ValueError('Cascadia DTB compatible differs from the selected board') + + +def write_fdt(root, reservations=(), cpu=0): + names, offsets, tree = bytearray(), {}, bytearray() + def integer(value): + tree.extend(struct.pack('>I', value)) + def node(item): + integer(1) + tree.extend(item['name'].encode('ascii') + b'\0') + tree.extend(b'\0' * (-len(tree) % 4)) + for name, value in item['properties'].items(): + if name not in offsets: + offsets[name] = len(names) + names.extend(name.encode('ascii') + b'\0') + integer(3) + tree.extend(struct.pack('>II', len(value), offsets[name])) + tree.extend(value) + tree.extend(b'\0' * (-len(tree) % 4)) + for child in item['children']: + node(child) + integer(2) + node(root) + integer(9) + # A valid tree may have no properties; retain a one-byte strings block. + if not names: + names.append(0) + reserve = b''.join(struct.pack('>QQ', *row) for row in reservations) + b'\0' * 16 + tree_at = 40 + len(reserve) + strings_at = tree_at + len(tree) + return (FDT_HEADER.pack(0xd00dfeed, strings_at + len(names), tree_at, strings_at, + 40, 17, 16, cpu, len(names), len(tree)) + reserve + tree + names) + + +def chosen(data, bootargs, start, size): + if not isinstance(bootargs, str) or any(char in bootargs for char in ('\0', '\n', '\r')): + raise ValueError('Kernel arguments must be one line') + encoded = bootargs.encode('utf-8') + if len(encoded) > 2047: + raise ValueError('Kernel arguments exceed the ARM command-line bound') + root, reservations, cpu = read_fdt(data) + cell_bytes = root['properties'].get('#address-cells', struct.pack('>I', 2)) + if len(cell_bytes) != 4 or (cells := struct.unpack('>I', cell_bytes)[0]) not in (1, 2): + raise ValueError('Expected one or two root address cells') + selected = next((item for item in root['children'] if item['name'] == 'chosen'), None) + if selected is None: + selected = {'name': 'chosen', 'properties': {}, 'children': []} + root['children'].append(selected) + properties = selected['properties'] + width = cells * 4 + previous_start = properties.get('linux,initrd-start', b'') + previous_end = properties.get('linux,initrd-end', b'') + if len(previous_start) == len(previous_end) == width: + old_start, old_end = int.from_bytes(previous_start, 'big'), int.from_bytes(previous_end, 'big') + reservations = [row for row in reservations if row != (old_start, old_end - old_start)] + if any(max(start, address) < min(start + size, address + length) for address, length in reservations): + raise ValueError('Initramfs destination overlaps reserved memory') + properties['bootargs'] = encoded + b'\0' + properties['linux,initrd-start'] = start.to_bytes(width, 'big') + properties['linux,initrd-end'] = (start + size).to_bytes(width, 'big') + reservations.append((start, size)) + return write_fdt(root, reservations, cpu) + + +def decompression_bound(kernel, dtb_size, described): + """Conservatively bound the v6.12 ARM decompressor's relocated workspace. + + The zImage table describes its inflated image and BSS. The low upload + address overlaps the output, so reserve room for a relocated compressed + image as well as DTB growth, stack and malloc space above that output. + Unknown table formats are refused before a payload is written. + """ + if len(kernel) < 0x3c or len(kernel) % 8: + raise ValueError('zImage must end on an eight-byte boundary for its appended DTB') + if dtb_size > 0x100000: + raise ValueError('DTB exceeds the ARM decompressor workspace bound') + if struct.unpack_from(' len(kernel): + raise ValueError('zImage size table exceeds the kernel') + count, magic, size_at, bss_size, text_offset, malloc_size, end = struct.unpack_from('<7I', kernel, table_at) + if count != 6 or magic != 0x5a534c4b or end or size_at < 0x3c or size_at + 4 > len(kernel): + raise ValueError('Unsupported zImage size table') + inflated = struct.unpack_from(' 0x100000: + raise ValueError('Invalid zImage decompression parameters') + # head.S allows a DTB to grow by 50%, bounded at 1 MiB. Include the + # decompressor relocation/code/stack alignment with an extra 128 KiB. + dtb_workspace = min(max(aligned(dtb_size + dtb_size // 2), 0x8000), 0x100000) + bound = (described['ram_start'] + text_offset + inflated + bss_size + + aligned(len(kernel) + dtb_workspace + malloc_size + 0x20000, 4096)) + if bound > described['initrd_address']: + raise ValueError('Kernel decompression workspace overlaps the initramfs destination') + return bound + + +def assemble(loader, kernel, dtb, initramfs, described, bootargs, *, elf=None): + validate_profile(described) + verify_loader(loader, described, elf) + if (len(kernel) < 0x30 or struct.unpack_from(' described['initrd_limit'] - described['initrd_address']: + raise ValueError('Initramfs exceeds its reserved destination') + tree = chosen(dtb, bootargs, described['initrd_address'], len(initramfs)) + expanded_end = decompression_bound(kernel, len(tree), described) + kernel_at = described['kernel_offset'] + dtb_at = kernel_at + len(kernel) + initrd_at = aligned(dtb_at + len(tree)) + image_size = initrd_at + len(initramfs) + if described['load_address'] + image_size > described['initrd_address']: + raise ValueError('Uploaded payload overlaps the initramfs destination') + values = configuration(loader) + values.update(dtb_offset=dtb_at, dtb_size=len(tree), initrd_offset=initrd_at, + initrd_size=len(initramfs), image_size=image_size) + record = CONFIG.pack(*(values[field] for field in CONFIG_FIELDS)) + values['crc32'] = zlib.crc32(record[:-4]) + framed_loader = bytearray(loader) + framed_loader[CONFIG_OFFSET:CONFIG_OFFSET + CONFIG.size] = CONFIG.pack(*(values[field] for field in CONFIG_FIELDS)) + components, image = [], bytearray() + def append(name, payload): + if payload: + components.append({'name': name, 'offset': len(image), 'bytes': len(payload), 'sha256': digest(payload)}) + image.extend(payload) + append('loader', framed_loader) + append('loader-padding', b'\0' * (kernel_at - len(image))) + append('kernel', kernel) + append('kernel-padding', b'\0' * (dtb_at - len(image))) + append('dtb', tree) + append('dtb-padding', b'\0' * (initrd_at - len(image))) + append('initramfs', initramfs) + return bytes(image), {'schema': 1, 'architecture': 'arm', 'loader_format': 'cascadia', + 'image_sha256': digest(image), 'initramfs_encoding': 'cascadia-dtb', + 'components': components, 'load_address': described['load_address'], + 'kernel_entry': described['load_address'] + kernel_at, + 'dtb_address': described['load_address'] + dtb_at, + 'kernel_decompression_bound': expanded_end, + 'initramfs_address': described['initrd_address'], + 'input_loader_sha256': digest(loader), 'input_dtb_sha256': digest(dtb)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + given = parser.add_mutually_exclusive_group(required=True) + given.add_argument('--bootargs') + given.add_argument('--manifest', type=Path) + for name in ('loader', 'kernel', 'initramfs', 'profile', 'output'): + parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--loader-elf', type=Path) + parser.add_argument('--dtb', action='append', type=Path, required=True) + parser.add_argument('--logo', type=Path) + args = parser.parse_args() + if len(args.dtb) != 1: + parser.error('A Cascadia payload takes exactly one device tree') + if args.logo: + parser.error('Cascadia payloads do not implement the m1n1 logo format') + receipt = args.output.with_suffix(args.output.suffix + '.json') + if args.output.exists() or receipt.exists(): + parser.error('Choose new image and receipt paths') + try: + bootargs = args.bootargs + if args.manifest: + bootargs = json.loads(args.manifest.read_text())['bootargs'] + prefix = 'chosen.bootargs=' + if not bootargs.startswith(prefix) or not bootargs.endswith('\n'): + raise ValueError('Invalid boot-argument manifest') + bootargs = bootargs[len(prefix):-1] + data, document = assemble(args.loader.read_bytes(), args.kernel.read_bytes(), args.dtb[0].read_bytes(), + args.initramfs.read_bytes(), profile(args.profile), bootargs, + elf=args.loader_elf.read_bytes() if args.loader_elf else None) + except (ValueError, KeyError) as error: + parser.error(str(error)) + document['profile_sha256'] = digest(args.profile.read_bytes()) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open('xb') as output: + output.write(data) + with receipt.open('x') as output: + json.dump(document, output, indent=2, sort_keys=True) + output.write('\n') + print(f'Built {args.output}; component receipt: {receipt}') + + +if __name__ == '__main__': + main() diff --git a/tests/fixtures/cascadia-runtime.c b/tests/fixtures/cascadia-runtime.c new file mode 100644 index 00000000..b3f276e9 --- /dev/null +++ b/tests/fixtures/cascadia-runtime.c @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Exercise the producer's actual C entry against anonymous mapped RAM/MMIO. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +/* The assembler patches this record in writable upload RAM on the device. */ +#define const +#include "linux_boot.c" +#undef const + +static unsigned int checks; + +#define CHECK(test) do { \ + checks++; \ + if (!(test)) { fprintf(stderr, "line %d: %s\n", __LINE__, #test); exit(1); } \ +} while (0) + +static void *memory(uintptr_t address, size_t bytes) +{ + void *value = mmap((void *)address, bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(value == (void *)address); + return value; +} + +static void prepare(void) +{ + struct config value = { + { 'C', 'B', '3', '2' }, 1, sizeof(struct config), LOAD_ADDRESS, + KERNEL_OFFSET, 0x9000, 40, 0xa000, 33, INITRD_ADDRESS, + 0xa000 + 33, WATCHDOG_ADDRESS, L2_ADDRESS, + RAM_START, RAM_END, INITRD_LIMIT, 0, + }; + memcpy((void *)&cascadia_config, &value, sizeof(value)); + unsigned char *ram = (void *)(uintptr_t)LOAD_ADDRESS; + memset(ram, 0, 0x20000); + ram[0x9000] = 0xd0; + ram[0x9001] = 0x0d; + ram[0x9002] = 0xfe; + ram[0x9003] = 0xed; + ram[0x9007] = 40; + for (unsigned int i = 0; i < 33; i++) + ram[0xa000 + i] = (unsigned char)(i ^ 0x5a); + memset((void *)(uintptr_t)INITRD_ADDRESS, 0x77, 0x10000); + memset((void *)(uintptr_t)WATCHDOG_ADDRESS, 0xa5, 16); + cascadia_config.crc32 = checksum(); +} + +static void rejects(void) +{ + CHECK(cascadia_main() == 0); + for (unsigned int i = 0; i < 33; i++) + CHECK(((unsigned char *)(uintptr_t)INITRD_ADDRESS)[i] == 0x77); + for (unsigned int i = 0; i < 16; i++) + CHECK(((unsigned char *)(uintptr_t)WATCHDOG_ADDRESS)[i] == 0xa5); +} + +int main(void) +{ + CHECK(sizeof(u32) == 4 && sizeof(struct config) == 68); + memory(LOAD_ADDRESS, 0x20000); + memory(INITRD_ADDRESS, 0x10000); + memory(WATCHDOG_ADDRESS & ~4095u, 4096); + prepare(); + CHECK(cascadia_main() == LOAD_ADDRESS + 0x9000); + for (unsigned int i = 0; i < 33; i++) + CHECK(((unsigned char *)(uintptr_t)INITRD_ADDRESS)[i] == (i ^ 0x5a)); + for (unsigned int i = 0; i < 16; i++) + CHECK(((unsigned char *)(uintptr_t)WATCHDOG_ADDRESS)[i] == 0); + CHECK(((unsigned char *)(uintptr_t)INITRD_ADDRESS)[33] == 0x77); + + /* Every field is either profile-fixed or has a checked dynamic bound. */ + volatile u32 *fields = (volatile u32 *)&cascadia_config; + for (unsigned int i = 0; i < sizeof(struct config) / sizeof(u32); i++) { + prepare(); + fields[i] = 0xffffffffu; + if (i != 16) + cascadia_config.crc32 = checksum(); + rejects(); + } + prepare(); + cascadia_config.dtb_offset = KERNEL_OFFSET - 4; + cascadia_config.crc32 = checksum(); + rejects(); + prepare(); + cascadia_config.initrd_offset = 0x9000; + cascadia_config.crc32 = checksum(); + rejects(); + prepare(); + ((unsigned char *)(uintptr_t)(LOAD_ADDRESS + 0x9000))[3] ^= 1; + rejects(); + prepare(); + ((unsigned char *)(uintptr_t)(LOAD_ADDRESS + 0x9000))[7] ^= 1; + rejects(); + printf("%u runtime checks passed\n", checks); + return 0; +} diff --git a/tests/test_cascadia_loader.py b/tests/test_cascadia_loader.py new file mode 100644 index 00000000..7f8bae26 --- /dev/null +++ b/tests/test_cascadia_loader.py @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: GPL-2.0-only +import json +from pathlib import Path +import struct +import shutil +import subprocess +import sys +import tempfile +import unittest +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'src')) +from orchard.image import cascadia + + +PROFILE = dict(schema=1, load_address=0x80000000, kernel_offset=0x8000, + watchdog_address=0x0f103020, l2_address=0x3e000000, + ram_start=0x80000000, ram_end=0xa0000000, + initrd_address=0x98000000, initrd_limit=0x9e000000) + + +def loader(described=PROFILE): + record = dict(magic=b'CB32', version=1, header_size=cascadia.CONFIG.size, + dtb_offset=0, dtb_size=0, initrd_offset=0, initrd_size=0, + image_size=0, crc32=0) + record.update({key: value for key, value in described.items() if key != 'schema'}) + raw = bytearray(cascadia.CONFIG_OFFSET + cascadia.CONFIG.size) + raw[:4] = bytes.fromhex('c0000cf1') + raw[cascadia.CONFIG_OFFSET:] = cascadia.CONFIG.pack(*(record[key] for key in cascadia.CONFIG_FIELDS)) + return bytes(raw) + + +def executable(raw, described=PROFILE): + data = bytearray(0x300 + 3 * 40) + data[:7] = b'\x7fELF\x01\x01\x01' + struct.pack_into('I', cells), + 'compatible': b'example,test\0'}, + 'children': [{'name': 'chosen', 'properties': {'bootargs': b'old\0', 'stdout-path': b'serial0\0'}, + 'children': []}, + {'name': 'memory@80000000', 'properties': {'device_type': b'memory\0', + 'reg': b'\x80\0\0\0\x20\0\0\0'}, + 'children': []}]} + + +class CascadiaLoaderTests(unittest.TestCase): + @unittest.skipUnless(sys.platform == 'linux' and shutil.which('cc'), 'The C runtime fixture needs Linux mmap and a C compiler') + def test_actual_runtime_c_copy_and_rejection_paths(self): + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + (folder / 'cascadia-generated.h').write_text(''.join( + f'#define {name.upper()} {hex(value)}\n' for name, value in PROFILE.items() if name != 'schema')) + executable = folder / 'runtime-check' + command = ['cc', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', '-Wno-int-to-pointer-cast', + '-I', str(folder), '-I', str(ROOT / 'boot/cascadia'), + str(ROOT / 'tests/fixtures/cascadia-runtime.c'), '-o', str(executable)] + compiled = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + result = subprocess.run([str(executable)], capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('runtime checks passed', result.stdout) + + def test_elf_entry_and_raw_code_must_match(self): + binary = loader() + elf = executable(binary) + cascadia.verify_loader(binary, PROFILE, elf) + shifted = bytearray(elf) + struct.pack_into('I', 0x98000000), + 'linux,initrd-end': struct.pack('>I', 0x98000010)}) + old = cascadia.write_fdt(value, [(0x98000000, 16), (0x9f000000, 4096)]) + new = cascadia.chosen(old, 'new', 0x98000000, 32) + self.assertEqual(cascadia.read_fdt(new)[1], [(0x9f000000, 4096), (0x98000000, 32)]) + + def test_reserved_destination_refused(self): + old = cascadia.write_fdt(tree(), [(PROFILE['initrd_address'] + 4, 4096)]) + with self.assertRaisesRegex(ValueError, 'reserved memory'): + self.assemble(dtb=old) + + def test_fdt_corruption_is_rejected(self): + original = cascadia.write_fdt(tree()) + for offset, value in ((4, len(original) + 1), (8, 0xffffffff), (12, 0), (16, 41), (20, 16)): + with self.subTest(offset=offset): + bad = bytearray(original) + struct.pack_into('>I', bad, offset, value) + with self.assertRaises(ValueError): + cascadia.read_fdt(bad) + with self.assertRaisesRegex(ValueError, 'overlap'): + bad = bytearray(original) + struct.pack_into('>I', bad, 12, struct.unpack_from('>I', bad, 8)[0]) + cascadia.read_fdt(bad) + + def test_wrong_kernel_and_out_of_bounds_initramfs_refused(self): + with self.assertRaisesRegex(ValueError, 'ARM zImage'): + self.assemble(kernel=b'arm64 Image') + with self.assertRaisesRegex(ValueError, 'length'): + self.assemble(kernel=kernel() + b'appended old dtb') + smaller = dict(PROFILE, initrd_limit=PROFILE['initrd_address'] + 8) + with self.assertRaisesRegex(ValueError, 'destination'): + self.assemble(loader=loader(smaller), described=smaller, elf=None) + with self.assertRaisesRegex(ValueError, 'one line'): + self.assemble(bootargs='a\nb') + + def test_decompressor_workspace_cannot_reach_high_initramfs(self): + large = bytearray(kernel()) + struct.pack_into(' Date: Fri, 6 Nov 2026 11:57:46 -0600 Subject: [PATCH 03/16] build: resolve ARMv7 recovery and distribution inputs Resolve one root architecture's per-platform loader and target tools through Orchard's producer contract. Preserve explicit board selection for Cascadia and stage ARM EABI5 hard-float Python dependency closures. Platform layers use the declared kernel image and validate the requested root ISA. Route RAM and recovery assembly through the declared payload format, bind Cascadia DTBs to their selected board, and keep Pongo inputs on their own boot path. Existing ARM64 default arguments remain supported. Document the producer contract and exercise cache separation and architecture refusal. Assisted-by: LLM --- docs/distribution-contract.md | 42 ++++- platforms/apple-armv7.toml | 5 + src/orchard/build.py | 59 ++++-- src/orchard/command.py | 8 +- src/orchard/distro.py | 18 +- src/orchard/image/elf.py | 105 +++++++++-- src/orchard/image/platform.py | 63 +++++-- src/orchard/image/prepare_runtime.py | 24 ++- src/orchard/image/recovery.py | 30 ++- src/orchard/inputs.py | 109 ++++++++--- src/orchard/project.py | 1 + src/orchard/ram.py | 76 ++++++-- tests/hardware_layout.py | 2 +- tests/test_armv7_producers.py | 263 +++++++++++++++++++++++++++ tests/test_distribution_contract.py | 8 +- tests/test_platform_layer.py | 3 +- tests/test_recovery_tools_arch.py | 56 +++++- 17 files changed, 764 insertions(+), 108 deletions(-) create mode 100644 tests/test_armv7_producers.py diff --git a/docs/distribution-contract.md b/docs/distribution-contract.md index 082f604f..9ac4f435 100644 --- a/docs/distribution-contract.md +++ b/docs/distribution-contract.md @@ -21,11 +21,11 @@ without orchard takes it as an argument. | Command | Does | Files | |---|---|---| -| `orchard.inputs --output FILE [--platform P …] [--kernel-dir P=DIR …] [--m1n1 FILE] [--pongo FILE] [--tools DIR] [--kernel-tree DIR]` (`ORCHARD_KERNEL_TREE` too) | resolves what the build takes finished | `FILE`: per platform its kernel bundle and boards (board → DTB), the shared m1n1, Pongo and static tools, orchard's revision, and under `produced` how each was got (built: its commands; reused from the cache; downloaded from a nightly: tag, asset, sha256); paths into orchard's cache | -| `orchard.image.platform P --output DIR --kernel-dir BUNDLE --busybox FILE [--receipt FILE]` | the platform layer of installed systems | `DIR/platform-P.tar.gz`, its receipt | +| `orchard.inputs --output FILE [--platform P …] [--device BOARD] [--platform-loader P=FILE] [--platform-tools P=DIR] [--kernel-dir P=DIR …] [--m1n1 FILE] [--pongo FILE] [--tools DIR] [--kernel-tree DIR]` (`ORCHARD_KERNEL_TREE` too) | resolves what the build takes finished | `FILE`: per platform its kernel bundle and boards (board → DTB), the shared m1n1, Pongo and static tools, orchard's revision, and under `produced` how each was got (built: its commands; reused from the cache; downloaded from a nightly: tag, asset, sha256); paths into orchard's cache | +| `orchard.image.platform P --output DIR --kernel-dir BUNDLE --busybox FILE [--receipt FILE] [--root-architecture arm64|arm] [--device BOARD]` | the platform layer of installed systems | `DIR/platform-P.tar.gz`, its receipt | | `orchard.image.platforms_index DIR (--release TAG [--platform P …] \| --inputs FILE \| --kernels DIR --tools DIR --commit SHA)` | what a build takes from orchard: every arm64 platform layer, Pongo, m1n1, `install-tools.tar.gz`, and `platforms.toml` listing them; from a published orchard build (`--release`, which `orchard.image.publish` then links instead of copying), or built here | `DIR/*` | | `orchard.image.root_image ROOT.tar.* --output DIR` | a root as the block image the installer writes; needs Podman, or root on a native build | `DIR/root.blocks.zst`, `DIR/root.map.json`, `DIR/root-image.json` (the manifest's `[root]`) | -| `orchard.image.recovery --dtb FILE … --loader FILE [--logo FILE] --kernel FILE --modules-dir DIR --busybox --dropbear --dropbearkey --wpa-supplicant --wpa-cli --iw FILE --base FILE --base-receipt FILE --root-receipt FILE --output FILE [--verbose]` | a platform's RAM recovery | the image and `IMAGE.json` | +| `orchard.image.recovery --dtb FILE … --loader FILE [--logo FILE] --kernel FILE --modules-dir DIR --busybox --dropbear --dropbearkey --wpa-supplicant --wpa-cli --iw FILE --base FILE --base-receipt FILE --root-receipt FILE --output FILE [--verbose] [--platform P] [--device BOARD]` | a platform's RAM recovery | the image and `IMAGE.json` | | `orchard.installer release DIR [--installer-url URL] [--pongo FILE] [--m1n1 FILE]` | the installer of a release | `DIR/install.sh`, `DIR/orchard-install.pyz` | | `orchard.manifests FILE … [--verify-local]` | checks release manifests | exit status | | `orchard.image.publish --distro NAME --tag TAG --release-dir DIR --output DIR` | the hosted release | `DIR` | @@ -34,3 +34,39 @@ without orchard takes it as an argument. Changing one of these is a contract change: say so in the changelog, and update both distributions in the same push set. + +## Architecture and board selection + +The default input document still selects every ARM64 platform and preserves +its `shared` m1n1/Pongo/tool record. An explicit ARMv7 build selects a board +with `orchard.inputs --device p105`. Each selected platform record includes +`architecture` (`arm64` or `arm`), `image`, `loader` (`kind` and `file`) and +its own `tools` paths. `boards` contains the selected board when `--device` +is given. A document has one root architecture; mixing ARM32 and ARM64 +platforms is refused before producing components. The top-level +`architecture` repeats that target for the distribution's root builder. + +`--platform-loader PLATFORM=FILE` and `--platform-tools PLATFORM=DIR` supply +finished components for offline builds. ARMv7 uses these per-platform inputs +and has an empty `shared` record. A Cascadia payload uses one board's handoff +profile; a platform with several boards requires explicit board selection. + +`orchard.image.platform --root-architecture arm --device p105` checks the +root ISA and writes a layer for that board. Its kernel image comes from the +platform descriptor. Optional `[root] storage_modules` in that descriptor +names the early-root modules; an empty list requires built-in storage. + +`orchard.image.recovery --platform apple-armv7 --device p105` stages an +ARM EABI5 hard-float runtime and invokes the source-built Cascadia loader. +The ELF dependency reader follows ELF32 libraries as well as ELF64, and +recovery refuses libraries or executables from another ISA. Cascadia puts +the external recovery initramfs in a reserved RAM range and passes its +addresses through the DTB. Its image receipt has +`initramfs_encoding = "cascadia-dtb"` and `architecture = "arm"`; the component +records describe contiguous `loader`, `kernel`, `dtb`, and `initramfs` ranges +with padding records where needed. This new handoff awaits physical testing. + +A distribution that accepts a board also declares `selects-device = true` +under `[release]` in `orchard.toml`, alongside `selects-platform = true`. +Orchard then forwards its selected `--device` to that distribution's build +entry. Existing distribution entries retain their previous arguments. diff --git a/platforms/apple-armv7.toml b/platforms/apple-armv7.toml index 61510ff2..05dba21e 100644 --- a/platforms/apple-armv7.toml +++ b/platforms/apple-armv7.toml @@ -15,3 +15,8 @@ config = "kernel/configs/apple-armv7/config" defconfig = "orchard_applearmv7_defconfig" # the same config in OrchardLinux/linux branch = "armv7" # its branch in OrchardLinux/linux loader = "cascadia" # the payload format that boots this kernel + +[root] +# H2FMI is built in. The installed NAND backend supplies the early root +# setup; the arm64 platforms' NVMe modules do not apply to this platform. +storage_modules = [] diff --git a/src/orchard/build.py b/src/orchard/build.py index b57711c5..a11eeecc 100644 --- a/src/orchard/build.py +++ b/src/orchard/build.py @@ -491,7 +491,7 @@ def has_loader_patches(hardware): M1N1_INPUTS = ['boot/m1n1', 'boot/quiet/m1n1.patch', 'src/orchard/buildenv.py'] TOOL_INPUTS = ['boot/initramfs/recovery/' + name for name in ('build-tools', 'build-tools-in-container', 'Containerfile', - 'install-toolchain', 'upstream.lock', 'orchard-write.c')] + ['src/orchard/buildenv.py'] + 'install-toolchain', 'upstream.lock', 'check-tools.py', 'orchard-write.c')] + ['src/orchard/buildenv.py'] def recipe_work(components, recipe, key): @@ -508,30 +508,62 @@ def recipe_work(components, recipe, key): return work -def _prepare_tools(hardware, components): +def _prepare_tools(hardware, components, architecture='arm64'): """The hardware's static recovery tools (BusyBox, Dropbear, Wi-Fi tools). Caller holds the lock.""" + if architecture not in ('arm', 'arm64'): + raise ValueError('Unsupported recovery architecture: ' + architecture) + name = 'recovery-tools' if architecture == 'arm64' else 'recovery-tools-' + architecture + key = recipe_key(hardware, TOOL_INPUTS) + if architecture != 'arm64': + key = hashlib.sha256((key + ':' + architecture).encode()).hexdigest() def produce_tools(stage): - work = recipe_work(components, 'recovery-tools', recipe_key(hardware, TOOL_INPUTS)) + work = recipe_work(components, name, key) if not (work / 'bin/busybox').exists(): - command(hardware / 'boot/initramfs/recovery/build-tools', + command(hardware / 'boot/initramfs/recovery/build-tools', '--arch', architecture, env={**os.environ, 'OMA_RECOVERY_TOOLS_WORK': str(work)}) + command(sys.executable, hardware / 'boot/initramfs/recovery/check-tools.py', + '--arch', architecture, *(work / 'bin' / name for name in + ('busybox', 'dropbear', 'dropbearkey', 'wpa_supplicant', 'wpa_cli', 'iw', + 'orchard-write', 'resize2fs', 'tune2fs'))) for tool in (work / 'bin').iterdir(): if tool.is_file() and not tool.is_symlink(): shutil.copyfile(tool, stage / tool.name) (stage / tool.name).chmod(0o755) - return cached_component(components, 'recovery-tools', - lambda: recipe_key(hardware, TOOL_INPUTS), produce_tools) + return cached_component(components, name, lambda: key, produce_tools) def prepare_tools(hardware=None, cache=None, *, device): """The directory of static recovery tools for RAM images, built or reused.""" from . import published hardware, cache = _build_context(hardware, cache, device) - tools = published.tools(hardware, cache) + architecture = kernels.descriptor(hardware, device)['arch'] + tools = published.tools(hardware, cache) if architecture == 'arm64' else None if tools is not None: return tools with component_lock(cache): - return _prepare_tools(hardware, cache / 'components-v2') + if architecture == 'arm64': + return _prepare_tools(hardware, cache / 'components-v2') + return _prepare_tools(hardware, cache / 'components-v2', architecture) + + +def prepare_payload_loader(hardware=None, cache=None, *, device): + """Build the loader format named by the device's platform descriptor.""" + hardware, cache = _build_context(hardware, cache, device) + kind = kernels.descriptor(hardware, device)['loader'] + components = cache / 'components-v2' + with component_lock(cache): + if kind == 'm1n1': + return _prepare_m1n1(hardware, components, 'apple') + if kind != 'cascadia': + raise ValueError('Unsupported payload loader: ' + kind) + from .hardware import payload_profile + profile = payload_profile(device, 'cascadia', hardware) + inputs = ['boot/cascadia', 'src/orchard/image/cascadia.py', str(profile.relative_to(hardware)), + 'src/orchard/buildenv.py'] + def produce(stage): + command(hardware / 'boot/cascadia/build', '--profile', profile, '--output', stage) + return cached_component(components, 'loader-cascadia-' + device, + lambda: recipe_key(hardware, inputs), produce) / 'loader.bin' RELEASES = 'https://github.com/OrchardLinux/orchard/releases/download' @@ -630,19 +662,24 @@ def prepare_components(hardware=None, cache=None, *, device, kernel=None, if kernel is not None and kernel_tree is not None: raise ValueError('Give a staged kernel (--kernel-dir) or a kernel tree (--kernel-tree), not both') tree = kernels.check_tree(hardware, device, kernel_tree) if kernel_tree is not None else None + described = kernels.descriptor(hardware, device) + if pongo is not None and described['loader'] == 'cascadia': + raise ValueError('Cascadia payloads use recovery-autogo firmware stages') + if loader is None and described['loader'] == 'cascadia': + loader = prepare_payload_loader(hardware, cache, device=device) if loader is None and not has_loader_patches(hardware): raise ValueError(f'Missing loader patches for {device}; add a loader recipe or supply --loader FILE') with component_lock(cache): - if pongo is None: + if pongo is None and described['loader'] == 'm1n1': pongo = _prepare_pongo(hardware, components) if kernel is None: kernel = _prepare_kernel(hardware, cache, device, tree) if busybox is None: - busybox = _prepare_tools(hardware, components) / 'busybox' + busybox = _prepare_tools(hardware, components, described['arch']) / 'busybox' if loader is None: # One m1n1 starts every Apple device; the board only picks its DTB. loader = _prepare_m1n1(hardware, components, 'apple') - return {name: Path(path).resolve() for name, path in + return {name: Path(path).resolve() if path is not None else None for name, path in dict(kernel=kernel, busybox=busybox, loader=loader, pongo=pongo).items()} diff --git a/src/orchard/command.py b/src/orchard/command.py index 0a49f93e..dc83ca93 100644 --- a/src/orchard/command.py +++ b/src/orchard/command.py @@ -138,8 +138,14 @@ def main(): '--platform or --device; update the distribution ' '(orchard.toml [release] selects-platform = true)') from orchard.build import resolve_kernel_tree + selected = {} + if distro.settings(root)['selects_device']: + from orchard.build import resolve_device + selected_device = resolve_device(root, args.device) + if selected_device is not None: + selected['device'] = selected_device index = distro.build(root, version=args.version, platforms=platforms, - kernel_tree=resolve_kernel_tree(root, args.kernel_tree)) + kernel_tree=resolve_kernel_tree(root, args.kernel_tree), **selected) commands = index.parent.parent / f'{index.parent.name}.commands.sh' if args.explain: print(commands.read_text(), end='') diff --git a/src/orchard/distro.py b/src/orchard/distro.py index 72afe53d..33813f74 100644 --- a/src/orchard/distro.py +++ b/src/orchard/distro.py @@ -61,13 +61,16 @@ def settings(root): if kind != 'release': raise ValueError(f'{root} has no orchard.toml with [release]') value = {**value, 'name': name} - unknown = set(value) - {'name', 'build', 'selects-platform'} + unknown = set(value) - {'name', 'build', 'selects-platform', 'selects-device'} if unknown: raise ValueError(f'{path}: unknown settings {", ".join(sorted(unknown))}; ' - 'expected name, build, selects-platform') + 'expected name, build, selects-platform, selects-device') selects = value.get('selects-platform', False) if not isinstance(selects, bool): raise ValueError(f'{path}: selects-platform must be true or false') + selects_device = value.get('selects-device', False) + if not isinstance(selects_device, bool) or (selects_device and not selects): + raise ValueError(f'{path}: selects-device requires selects-platform and must be true or false') name, build = value.get('name'), value.get('build') if not isinstance(name, str) or not _NAME.fullmatch(name): raise ValueError(f'{path}: name must be the distro name of its index, e.g. "omarchy-a"') @@ -79,7 +82,7 @@ def settings(root): raise ValueError(f'{path}: the build entry {build} is not a file') if entry.suffix != '.py' and not os.access(entry, os.X_OK): raise ValueError(f'{path}: the build entry {build} is not executable') - return {'name': name, 'build': entry, 'selects_platform': selects} + return {'name': name, 'build': entry, 'selects_platform': selects, 'selects_device': selects_device} def default_version(now=None): @@ -88,7 +91,7 @@ def default_version(now=None): return now.strftime('dev-%Y%m%d-%H%M%S') -def build(root, *, version=None, platforms=None, kernel_tree=None): +def build(root, *, version=None, platforms=None, kernel_tree=None, device=None): """Run ROOT's build entry into out/releases/VERSION, check the release, record it; returns its index. PLATFORMS: a list of platform names or 'all', passed as --platform to an @@ -118,6 +121,10 @@ def build(root, *, version=None, platforms=None, kernel_tree=None): elif distro['selects_platform']: raise ValueError(f'{distro["name"]} selects its platforms: choose with --platform, --device, ' 'orchard.local.toml or the attached board') + if device is not None: + if not distro['selects_device']: + raise ValueError('Distribution build entry does not accept --device') + argv += ['--device', device] print('+', ' '.join(argv), flush=True) environment = {**os.environ, 'ORCHARD_PYTHON': sys.executable, 'ORCHARD_CHECKOUT': str(ROOT)} tree = '' @@ -185,7 +192,8 @@ def run(root, *, build_first=True, version=None, yes=False, progress=None, kerne if board is None: raise ValueError('No described device is attached: connect it, then run again') platforms = [ram.resolve_platform(root, ROOT, device=board)] - index = build(root, version=version, platforms=platforms, kernel_tree=kernel_tree) + index = build(root, version=version, platforms=platforms, kernel_tree=kernel_tree, + **({'device': board} if settings(root)['selects_device'] else {})) else: index = latest(root) with operation_lock(): diff --git a/src/orchard/image/elf.py b/src/orchard/image/elf.py index 48c6fc14..914421f3 100644 --- a/src/orchard/image/elf.py +++ b/src/orchard/image/elf.py @@ -1,35 +1,100 @@ -"""Read ELF64 little-endian DT_NEEDED names without executing target binaries.""" +"""Inspect little-endian ELF32/ELF64 dependencies without executing target code.""" import struct +def _parts(data): + if data[:4] != b'\x7fELF': + return None + if len(data) < 16 or data[5:7] != b'\x01\x01' or data[4] not in (1, 2): + raise ValueError('Unsupported ELF encoding') + width = data[4] + header_size, program_size = (52, 32) if width == 1 else (64, 56) + if len(data) < header_size: + raise ValueError('Incomplete ELF header') + machine = struct.unpack_from(' len(data)): + raise ValueError('ELF program headers exceed the file') + segments = [] + for index in range(count): + position = offset + size * index + if width == 1: + kind, file_offset, virtual, _, file_size, _, _, _ = struct.unpack_from('<8I', data, position) + else: + kind, _, file_offset, virtual, _, file_size, _, _ = struct.unpack_from(' len(data): + raise ValueError('ELF segment exceeds the file') + segments.append((kind, virtual, file_offset, file_size)) + return width, machine, segments + + +def architecture(data): + """Return Orchard's architecture for an ARM executable or shared object.""" + parsed = _parts(data) + if parsed is None: + raise ValueError('Expected an ELF executable') + width, machine, _ = parsed + if width == 2 and machine == 183: + return 'arm64' + if width == 1 and machine == 40: + flags = struct.unpack_from('> 24 != 5 or not flags & 0x400 or flags & 0x200: + raise ValueError('ARM recovery requires EABI5 hard-float executables') + return 'arm' + raise ValueError('Unsupported recovery ELF architecture') + + def needed(data): - if data[:6] != b'\x7fELF\x02\x01': + parsed = _parts(data) + if parsed is None: return set() - offset = struct.unpack_from(' length: + raise ValueError('ELF string table exceeds its load segment') + length = string_size + result = set() + for name in names: + if name >= length: + raise ValueError('ELF dependency offset exceeds the string table') + end = data.find(b'\0', start + name, start + length) + if end < 0: + raise ValueError('Unterminated ELF dependency name') + try: + text = data[start + name:end].decode('ascii') + except UnicodeDecodeError as error: + raise ValueError('Non-ASCII ELF dependency name') from error + if not text or '/' in text or text in ('.', '..'): + raise ValueError('Invalid ELF dependency name') + result.add(text) + return result raise ValueError('ELF string table is outside load segments') diff --git a/src/orchard/image/platform.py b/src/orchard/image/platform.py index 76a5e4fb..f3a1ec7c 100644 --- a/src/orchard/image/platform.py +++ b/src/orchard/image/platform.py @@ -17,6 +17,8 @@ import gzip import io import json +import re +import tomllib from pathlib import Path import stat import tarfile @@ -49,7 +51,7 @@ def boards(platform, hardware=None): ROOT_ARCH = 'arm64' # distributions build one arm64 root for every platform -def platforms(hardware=None): +def platforms(hardware=None, architecture=ROOT_ARCH): """Every platform with at least one described board whose kernel runs the shared arm64 root, sorted. A platform of another architecture (apple-armv7) has no platform layer until a root of its architecture exists.""" @@ -57,7 +59,7 @@ def platforms(hardware=None): from ..ram import boards_by_platform hardware = hardware or ROOT return sorted(platform for platform, boards in boards_by_platform(hardware).items() - if descriptor(hardware, sorted(boards)[0])['arch'] == ROOT_ARCH) + if architecture is None or descriptor(hardware, sorted(boards)[0])['arch'] == architecture) def kernel(platform, *, hardware=None, cache=None): @@ -79,7 +81,7 @@ def tools(hardware=None, cache=None, platform='apple-16k'): return prepare_tools(hardware, cache, device=sorted(boards(platform, hardware))[0]) / 'busybox' -def initrd(modules, release, busybox, *, hardware=None): +def initrd(modules, release, busybox, *, hardware=None, storage_modules=STORAGE_MODULES): """orchard's initrd for installed systems: BusyBox, the storage modules and their dependencies. MODULES is {path: (data, mode)} as module_files gives it. @@ -91,9 +93,9 @@ def initrd(modules, release, busybox, *, hardware=None): name, _, needs = line.partition(':') depends[name] = needs.split() by_name = {Path(name).name.split('.ko')[0].replace('_', '-'): name for name in depends} - loaded = [name for name in STORAGE_MODULES if name.replace('_', '-') in by_name] - if not loaded: - raise ValueError(f'Kernel bundle has no storage module: {", ".join(STORAGE_MODULES)}') + loaded = [name for name in storage_modules if name.replace('_', '-') in by_name] + if storage_modules and not loaded: + raise ValueError(f'Kernel bundle has no storage module: {", ".join(storage_modules)}') wanted, pending = [], [by_name[name.replace('_', '-')] for name in loaded] while pending: name = pending.pop() @@ -121,14 +123,31 @@ def initrd(modules, release, busybox, *, hardware=None): return output.read_bytes() +def storage_modules(hardware, platform): + """The platform's early-root modules; an empty list denotes built-in storage.""" + if not re.fullmatch(r'[a-z0-9][a-z0-9_-]*', platform): + raise ValueError('Invalid platform name') + settings = tomllib.loads((Path(hardware) / 'platforms' / (platform + '.toml')).read_text()) + modules = settings.get('root', {}).get('storage_modules', list(STORAGE_MODULES)) + if (not isinstance(modules, list) or + any(not isinstance(name, str) or not re.fullmatch(r'[a-zA-Z0-9_+-]+', name) for name in modules) or + len(modules) != len(set(modules))): + raise ValueError('Platform root.storage_modules must be distinct kernel module names') + return tuple(modules) + + def files(bundle, platform, board_dtbs, *, busybox, hardware=None): """(kernel release, {path: (data, mode)}) of PLATFORM's layer from a staged kernel BUNDLE.""" hardware = Path(hardware or ROOT) + from ..kernel import descriptor + described = descriptor(hardware, sorted(board_dtbs)[0]) bundle = Path(bundle) release, result = module_files(bundle / 'lib/modules') base = f'usr/lib/modules/{release}/' - result[base + 'vmlinuz'] = ((bundle / 'Image.gz').read_bytes(), 0o644) - result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware), 0o644) + result[base + 'vmlinuz'] = ((bundle / described['image']).read_bytes(), 0o644) + modules_to_load = storage_modules(hardware, platform) + result[base + 'initrd'] = (initrd(result, release, busybox, hardware=hardware, + storage_modules=modules_to_load), 0o644) for dtb in sorted(set(board_dtbs.values())): path = bundle / 'dtbs' / (dtb + '.dtb') if path.is_symlink() or not path.is_file(): @@ -145,7 +164,8 @@ def files(bundle, platform, board_dtbs, *, busybox, hardware=None): raise ValueError(f'System defaults belong under /usr/lib or /usr/share/alsa/ucm2: {name}') result[name] = (path.read_bytes(), 0o755 if path.stat().st_mode & 0o111 else 0o644) description = {'schema': SCHEMA, 'platform': platform, 'kernel_release': release, - 'boards': dict(sorted(board_dtbs.items()))} + 'boards': dict(sorted(board_dtbs.items())), 'architecture': described['arch'], + 'image': described['image']} result['usr/lib/orchard/platform.json'] = ((json.dumps(description, indent=2) + '\n').encode(), 0o644) result['usr/lib/orchard/platform.SHA256SUMS'] = (checksums(result), 0o644) return release, result @@ -166,21 +186,36 @@ def write(layer, output): return expanded, len(layer) -def build(platform, output_directory, *, hardware=None, cache=None, bundle=None, busybox=None): +def build(platform, output_directory, *, hardware=None, cache=None, bundle=None, busybox=None, + root_architecture=None, device=None): """PLATFORM's layer as OUTPUT_DIRECTORY/platform-PLATFORM.tar.gz, plus its receipt. BUNDLE is a staged kernel bundle and BUSYBOX orchard's static BusyBox; without them they are built or reused from the cache. """ board_dtbs = boards(platform, hardware) + if device is not None: + if device not in board_dtbs: + raise ValueError(f'Device {device} does not belong to platform {platform}') + board_dtbs = {device: board_dtbs[device]} + if root_architecture is not None: + from ..kernel import descriptor + architecture = descriptor(hardware or ROOT, sorted(board_dtbs)[0])['arch'] + if architecture != root_architecture: + raise ValueError(f'Platform {platform} architecture {architecture} differs from root {root_architecture}') bundle = Path(bundle) if bundle is not None else kernel(platform, hardware=hardware, cache=cache) busybox = Path(busybox) if busybox is not None else tools(hardware, cache, platform) + if root_architecture is not None: + from .elf import architecture as elf_architecture + if elf_architecture(busybox.read_bytes()) != root_architecture: + raise ValueError('Platform initrd BusyBox differs from the root architecture') release, layer = files(bundle, platform, board_dtbs, busybox=busybox, hardware=hardware) output = Path(output_directory) / f'platform-{platform}.tar.gz' expanded, members = write(layer, output) return {'file': output.name, 'bytes': output.stat().st_size, 'sha256': sha(output), 'compression': 'gzip', 'expanded_bytes': expanded, 'members': members, - 'kernel_release': release, 'boards': sorted(board_dtbs)} + 'kernel_release': release, 'boards': sorted(board_dtbs), + 'architecture': json.loads(layer['usr/lib/orchard/platform.json'][0])['architecture']} def main(): @@ -190,9 +225,13 @@ def main(): parser.add_argument('--kernel-dir', type=Path, help='a staged kernel bundle (default: build or reuse it)') parser.add_argument('--busybox', type=Path, help="orchard's static BusyBox (default: build or reuse it)") parser.add_argument('--receipt', type=Path, help="also write the layer's receipt (JSON) here") + parser.add_argument('--device', help='restrict the platform layer to this board') + parser.add_argument('--root-architecture', choices=('arm', 'arm64'), + help='require the platform to match this root architecture') args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=True) - receipt = build(args.platform, args.output, bundle=args.kernel_dir, busybox=args.busybox) + receipt = build(args.platform, args.output, bundle=args.kernel_dir, busybox=args.busybox, + root_architecture=args.root_architecture, device=args.device) if args.receipt: args.receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True) + '\n') print(json.dumps(receipt, indent=2)) diff --git a/src/orchard/image/prepare_runtime.py b/src/orchard/image/prepare_runtime.py index 9317506a..576b068c 100644 --- a/src/orchard/image/prepare_runtime.py +++ b/src/orchard/image/prepare_runtime.py @@ -10,7 +10,7 @@ from .runtime import files as runtime_files, recovery_files from .integrity import sha -from .elf import needed +from .elf import needed, architecture as elf_architecture from ..hardware import ROOT as PROJECT HERE = PROJECT / 'boot/initramfs/recovery' @@ -22,11 +22,18 @@ def copy(source, target, mode): target.chmod(mode) -def stage_runtime(base, receipt, root): +def stage_runtime(base, receipt, root, architecture='arm64'): spec = json.loads(receipt.read_text()) + root_architecture = spec.get('architecture', architecture) + if not isinstance(root_architecture, str): + raise ValueError('Python source root architecture is invalid') + root_architecture = {'aarch64': 'arm64', 'arm64': 'arm64', + 'armv7h': 'arm', 'armhf': 'arm', 'arm': 'arm'}.get(root_architecture) + if root_architecture != architecture: + raise ValueError('Python source root architecture differs from recovery') if base.stat().st_size != spec['bytes'] or sha(base) != spec['sha256']: raise ValueError('Python source root differs from receipt') - libraries = re.compile(r'usr/lib/(?:ld-linux-aarch64|lib(?:python[0-9.]+|c|m|z|zstd|bz2|lzma|ffi|ssl|crypto|readline|ncursesw|tinfo|uuid|expat|gcc_s|mpdec))[-.].*') + libraries = re.compile(r'usr/lib/(?:ld-linux-(?:aarch64|armhf)|lib(?:python[0-9.]+|c|m|z|zstd|bz2|lzma|ffi|ssl|crypto|readline|ncursesw|tinfo|uuid|expat|gcc_s|mpdec))[-.].*') with tarfile.open(base, 'r|gz') as archive: for member in archive: name = member.name.removeprefix('./') @@ -74,7 +81,10 @@ def stage_runtime(base, receipt, root): required = set() for path in (root / 'usr').rglob('*'): if path.is_file() and not path.is_symlink(): - required.update(needed(path.read_bytes())) + data = path.read_bytes() + if data.startswith(b'\x7fELF') and elf_architecture(data) != architecture: + raise ValueError('Recovery contains an executable for another architecture: ' + str(path)) + required.update(needed(data)) missing = set() for name in required: path = root / 'usr/lib' / name @@ -117,6 +127,7 @@ def main(): for name in ('busybox', 'dropbear', 'dropbearkey', 'wpa-supplicant', 'wpa-cli', 'iw', 'base', 'base-receipt', 'modules-dir', 'root-receipt', 'output'): parser.add_argument('--' + name, type=Path, required=True) + parser.add_argument('--architecture', choices=('arm', 'arm64'), default='arm64') args = parser.parse_args() root = args.output if root.exists(): @@ -129,7 +140,10 @@ def main(): (args.wpa_supplicant, 'usr/sbin/wpa_supplicant'), (args.wpa_cli, 'usr/bin/wpa_cli'), (args.iw, 'usr/sbin/iw')): copy(source, root / name, 0o755) - stage_runtime(args.base, args.base_receipt, root) + for source in (args.busybox, args.dropbear, args.dropbearkey, args.wpa_supplicant, args.wpa_cli, args.iw): + if elf_architecture(source.read_bytes()) != args.architecture: + raise ValueError('Recovery tool architecture differs: ' + str(source)) + stage_runtime(args.base, args.base_receipt, root, args.architecture) kernel_release, public = runtime_files(args.modules_dir) for name, (data, mode) in {**public, **recovery_files()}.items(): path = root / name diff --git a/src/orchard/image/recovery.py b/src/orchard/image/recovery.py index 7c2ee440..87babe68 100644 --- a/src/orchard/image/recovery.py +++ b/src/orchard/image/recovery.py @@ -19,10 +19,35 @@ def main(): parser.add_argument('--dtb', type=Path, action='append', required=True, help='the DTB of each device this recovery boots; repeat') parser.add_argument('--logo', type=Path, help="the distribution's m1n1 logo payload (see assemble.py)") + parser.add_argument('--platform', default='apple-16k', help='kernel platform and payload format') + parser.add_argument('--device', help='the board whose Cascadia handoff profile to use') diagnostics = parser.add_mutually_exclusive_group() diagnostics.add_argument('--verbose', action='store_true', help='enable visible Linux framebuffer console and kernel diagnostics') args = parser.parse_args() + from ..kernel import descriptor + from .platform import boards + described = descriptor(ROOT, args.platform) + profile = None + if described['loader'] == 'cascadia': + if len(args.dtb) != 1 or args.logo is not None: + parser.error('Cascadia recovery takes one board DTB and no m1n1 logo') + available = boards(described['platform']) + selected = args.device + if selected is None: + candidates = [board for board, dtb in available.items() if args.dtb[0].stem == dtb] + if len(candidates) != 1: + parser.error('Select the Cascadia board with --device') + selected = candidates[0] + if selected not in available or args.dtb[0].stem != available[selected]: + parser.error('The Cascadia board and supplied DTB differ') + from ..hardware import payload_profile, device as described_device + from .cascadia import check_compatible + try: + profile = payload_profile(selected, 'cascadia', ROOT) + check_compatible(args.dtb[0].read_bytes(), described_device(selected, ROOT)['device']['dt_compatible']) + except ValueError as error: + parser.error(str(error)) manifest = 'boot-verbose.json' if args.verbose else 'boot.json' if args.output.exists(): parser.error('choose a new output path') @@ -39,11 +64,14 @@ def main(): '--wpa-cli', str(args.wpa_cli), '--iw', str(args.iw), '--base', str(args.base), '--base-receipt', str(args.base_receipt), '--modules-dir', str(args.modules_dir), '--root-receipt', str(args.root_receipt), + '--architecture', described['arch'], '--output', str(stage / 'root')], check=True) subprocess.run([sys.executable, '-m', 'orchard.image.initramfs', '--root', str(stage / 'root'), '--output', str(initramfs)], check=True) - subprocess.run([sys.executable, '-m', 'orchard.image.assemble', + assembly = 'orchard.image.cascadia' if profile is not None else 'orchard.image.assemble' + subprocess.run([sys.executable, '-m', assembly, + *(['--profile', str(profile)] if profile is not None else []), '--manifest', str(HERE / manifest), '--loader', str(args.loader), *(f'--dtb={dtb}' for dtb in args.dtb), *(['--logo', str(args.logo)] if args.logo else []), diff --git a/src/orchard/inputs.py b/src/orchard/inputs.py index 3d84b442..1d9d7ba0 100644 --- a/src/orchard/inputs.py +++ b/src/orchard/inputs.py @@ -32,30 +32,57 @@ TOOLS = ('busybox', 'dropbear', 'dropbearkey', 'wpa_supplicant', 'wpa_cli', 'iw', 'orchard-write', 'resize2fs', 'tune2fs') -def resolve(platforms=None, explicit=None, *, hardware=None, cache=None, shared=None, kernel_tree=None): - """The inputs document for PLATFORMS (default: every arm64 platform). +def resolve(platforms=None, explicit=None, *, hardware=None, cache=None, shared=None, kernel_tree=None, + platform_loaders=None, platform_tools=None, device=None): + """Resolve one architecture's inputs; default to the existing arm64 platforms. - SHARED: {'m1n1', 'pongo', 'tools'} given explicitly; the rest is resolved. - KERNEL_TREE: a developer's kernel tree to build every selected platform's kernel from. + Per-platform loader/tool overrides keep offline builds independent of + orchard's cache. A root build must choose platforms of one architecture. """ - from .build import prepare_shared + from .build import prepare_shared, prepare_payload_loader, prepare_tools from .hardware import ROOT, revision from .image import platform + from .kernel import descriptor, layout hardware = Path(hardware or ROOT) explicit = explicit or {} - known = platform.platforms(hardware) - selected = list(platforms or known) - unknown = sorted(set(selected) - set(known)) + sorted(set(explicit) - set(selected)) + platform_loaders, platform_tools = platform_loaders or {}, platform_tools or {} + known = platform.platforms(hardware, architecture=None) + if device is not None: + selected_platform, _ = layout(hardware, device) + if platforms and list(platforms) != [selected_platform]: + raise ValueError('Selected device does not match the requested platform') + selected = [selected_platform] + else: + selected = list(platforms or platform.platforms(hardware)) + unknown = (set(selected) - set(known)) | ((set(explicit) | set(platform_loaders) | + set(platform_tools)) - set(selected)) if unknown: - raise ValueError(f'Unknown or unselected platform {", ".join(unknown)}; known: {", ".join(known)}') + raise ValueError(f'Unknown or unselected platform {", ".join(sorted(unknown))}; known: {", ".join(known)}') + descriptions = {name: descriptor(hardware, name) for name in selected} + architectures = {description['arch'] for description in descriptions.values()} + if len(architectures) != 1: + raise ValueError('A distribution root requires platforms of one architecture') + architecture = next(iter(architectures)) + board_sets = {name: platform.boards(name, hardware) for name in selected} + if device is not None: + board_sets[selected[0]] = {device: board_sets[selected[0]][device]} + for name, description in descriptions.items(): + if description['loader'] == 'cascadia' and len(board_sets[name]) != 1: + raise ValueError('Cascadia recovery requires --device to select one board profile') given = {key: Path(value) for key, value in (shared or {}).items() if value is not None} - found = {} if len(given) == 3 else prepare_shared(hardware, cache) - m1n1 = given.get('m1n1') or found['loader'] - pongo = given.get('pongo') or found['pongo'] - tools = given.get('tools') or Path(found['tools']) - for path in (m1n1, pongo, *(tools / name for name in TOOLS)): - if not Path(path).is_file(): - raise ValueError(f'Missing component {path}') + shared_document = {} + if architecture == 'arm64': + found = {} if len(given) == 3 else prepare_shared(hardware, cache) + m1n1 = given.get('m1n1') or found['loader'] + pongo = given.get('pongo') or found['pongo'] + tools = given.get('tools') or Path(found['tools']) + shared_document = {'m1n1': str(Path(m1n1).resolve()), 'pongo': str(Path(pongo).resolve()), + 'tools': {name: str((tools / name).resolve()) for name in TOOLS}} + for path in (m1n1, pongo, *(tools / name for name in TOOLS)): + if not Path(path).is_file(): + raise ValueError(f'Missing component {path}') + elif given: + raise ValueError('ARMv7 inputs use --platform-loader and --platform-tools overrides') try: orchard = revision() except ValueError: @@ -65,29 +92,46 @@ def bundle(name): return explicit[name] if kernel_tree is not None: from .build import prepare_kernel - return prepare_kernel(hardware, cache, device=sorted(platform.boards(name, hardware))[0], + return prepare_kernel(hardware, cache, device=sorted(board_sets[name])[0], kernel_tree=kernel_tree) return platform.kernel(name, hardware=hardware, cache=cache) - return { - 'schema': SCHEMA, - 'orchard': orchard, - 'platforms': {name: {'kernel': str(Path(bundle(name)).resolve()), - 'boards': platform.boards(name, hardware)} - for name in selected}, - 'shared': {'m1n1': str(Path(m1n1).resolve()), 'pongo': str(Path(pongo).resolve()), - 'tools': {name: str((tools / name).resolve()) for name in TOOLS}}, - } + records = {} + for name in selected: + description = descriptions[name] + board = sorted(board_sets[name])[0] + if architecture == 'arm64': + loader = platform_loaders.get(name) or m1n1 + target_tools = platform_tools.get(name) or tools + else: + loader = platform_loaders.get(name) or prepare_payload_loader(hardware, cache, device=board) + target_tools = platform_tools.get(name) or prepare_tools(hardware, cache, device=board) + target_tools = Path(target_tools) + for path in (loader, *(target_tools / tool for tool in TOOLS)): + if not Path(path).is_file(): + raise ValueError(f'Missing component {path}') + records[name] = {'kernel': str(Path(bundle(name)).resolve()), + 'boards': board_sets[name], + 'architecture': description['arch'], 'image': description['image'], + 'loader': {'kind': description['loader'], 'file': str(Path(loader).resolve())}, + 'tools': {tool: str((target_tools / tool).resolve()) for tool in TOOLS}} + return {'schema': SCHEMA, 'orchard': orchard, 'architecture': architecture, + 'platforms': records, 'shared': shared_document} def main(): parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) parser.add_argument('--output', type=Path, required=True, help='the JSON document to write') + parser.add_argument('--device', help='restrict inputs to this board and its loader profile') parser.add_argument('--platform', action='append', help='a platform to build for; repeat (default: every one)') parser.add_argument('--kernel-dir', action='append', default=[], metavar='PLATFORM=DIRECTORY', help="a platform's staged kernel bundle instead of orchard's") parser.add_argument('--m1n1', type=Path, help="orchard's m1n1 to use as it is") parser.add_argument('--pongo', type=Path, help='the Pongo to use as it is') parser.add_argument('--tools', type=Path, help='a directory with the static tools to use as they are') + parser.add_argument('--platform-loader', action='append', default=[], metavar='PLATFORM=FILE', + help='an explicit payload loader for one platform') + parser.add_argument('--platform-tools', action='append', default=[], metavar='PLATFORM=DIRECTORY', + help='an explicit static tool directory for one platform') parser.add_argument('--kernel-tree', type=Path, help='build the kernels from this kernel tree ' '(default: ORCHARD_KERNEL_TREE, which orchard build --kernel-tree sets)') args = parser.parse_args() @@ -97,6 +141,16 @@ def main(): if not separator: parser.error('--kernel-dir takes PLATFORM=DIRECTORY') explicit[name] = Path(path) + def overrides(values, option): + result = {} + for value in values: + name, separator, path = value.partition('=') + if not separator or not name or not path or name in result: + parser.error(option + ' takes distinct PLATFORM=PATH values') + result[name] = Path(path) + return result + platform_loaders = overrides(args.platform_loader, '--platform-loader') + platform_tools = overrides(args.platform_tools, '--platform-tools') try: import os tree = args.kernel_tree or (Path(os.environ['ORCHARD_KERNEL_TREE']) if os.environ.get('ORCHARD_KERNEL_TREE') @@ -105,7 +159,8 @@ def main(): explain.start() try: document = resolve(args.platform, explicit, shared={'m1n1': args.m1n1, 'pongo': args.pongo, - 'tools': args.tools}, kernel_tree=tree) + 'tools': args.tools}, kernel_tree=tree, + platform_loaders=platform_loaders, platform_tools=platform_tools, device=args.device) finally: steps = explain.stop() except ValueError as error: diff --git a/src/orchard/project.py b/src/orchard/project.py index d084147a..3bcf8343 100644 --- a/src/orchard/project.py +++ b/src/orchard/project.py @@ -10,6 +10,7 @@ [release] build = "tools/build.py" # relative to this file selects-platform = true # optional: the script takes --platform + selects-device = true # optional: preserve the chosen board as --device The file says what the project builds, never which board it is for; a developer's choices (platform, device, kernel tree) are arguments or diff --git a/src/orchard/ram.py b/src/orchard/ram.py index d0ba1eac..c24c47dd 100644 --- a/src/orchard/ram.py +++ b/src/orchard/ram.py @@ -20,8 +20,13 @@ IMAGE = Path(__file__).resolve().parent / 'image' # standalone scripts, run by path -def check_busybox(path): +def check_busybox(path, architecture='arm64'): data = path.read_bytes() + if architecture == 'arm': + from .image.elf import architecture as elf_architecture, _parts + if elf_architecture(data) != architecture or any(kind in (2, 3) for kind, *_ in _parts(data)[2]): + raise ValueError('BusyBox must be static ARM EABI5 hard-float ELF32') + return if len(data) < 64 or data[:6] != b'\x7fELF\x02\x01' or data[18:20] != b'\xb7\x00': raise ValueError('BusyBox must be little-endian AArch64 ELF64') offset = struct.unpack_from(' 128 * 1024 * 1024: raise ValueError('Payload exceeds the Pongo uploader limit (128 MiB)') - shutil.copyfile(pongo, work / 'pongo.bin') + if pongo is not None: + shutil.copyfile(pongo, work / 'pongo.bin') receipt = {'kind': 'orchard.ram-build', 'platform': platform, 'boards': sorted(boards), 'kernel_release': release, 'hardware_tested': False, # What `orchard run` needs to add a board's firmware layer. - 'assembly': {'loader': str(loader), 'kernel': str(kernel / 'Image.gz'), + 'assembly': {'loader': str(loader), 'kernel': str(kernel / described['image']), 'dtbs': [str(path) for path in dtbs], 'bootargs': bootargs}, - 'files': {name: digest(work / name) for name in ('linux.bin', 'pongo.bin')}, + 'files': {name: digest(work / name) for name in + (('linux.bin', 'pongo.bin') if pongo is not None else ('linux.bin',))}, 'sources': {str(p.resolve()): digest(p) for p in inputs}} - for name in ('linux.bin', 'linux.bin.json', 'pongo.bin', 'initramfs.cpio.gz'): + if profile is not None: + receipt['loader_format'] = 'cascadia' + receipt['boot_method'] = 'recovery-autogo' + receipt['assembly']['profile'] = str(profile) + for name in ('linux.bin', 'linux.bin.json', 'initramfs.cpio.gz', + *(('pongo.bin',) if pongo is not None else ())): os.replace(work / name, output / name) (output / 'build.json').write_text(json.dumps(receipt, indent=2) + '\n') print(f'Built {output / "linux.bin"}. No device was accessed.') return output -def assemble(bootargs, loader, dtbs, kernel, initramfs, output): - command(sys.executable, IMAGE / 'assemble.py', '--bootargs', bootargs, '--loader', loader, +def assemble(bootargs, loader, dtbs, kernel, initramfs, output, *, profile=None): + command(sys.executable, IMAGE / ('cascadia.py' if profile is not None else 'assemble.py'), + *(['--profile', profile] if profile is not None else []), + '--bootargs', bootargs, '--loader', loader, *(argument for dtb in dtbs for argument in ('--dtb', dtb)), '--kernel', kernel, '--initramfs', initramfs, '--output', output) @@ -218,7 +254,12 @@ def _build_directory(directory, *, hardware, cache, kernel, busybox, loader, pon hardware = Path(hardware).expanduser().resolve() platform = resolve_platform(directory, hardware, platform=platform, device=device) boards = boards_by_platform(hardware)[platform] - # Components are per platform; any of its boards selects them. + from .kernel import descriptor + if descriptor(hardware, platform)['loader'] == 'cascadia': + selected = build.resolve_device(directory, device) + if selected is None or selected not in boards: + raise ValueError('Select the Cascadia board with --device BOARD') + boards = {selected: boards[selected]} representative = sorted(boards)[0] cache = Path(cache).expanduser().resolve() if cache is not None else build.cache_root() output = directory / 'out' / platform @@ -302,10 +343,12 @@ def with_firmware(output, board, *, hardware=None, cache=None): image = work / 'image' image.mkdir() assemble(assembly['bootargs'], Path(assembly['loader']), [Path(path) for path in assembly['dtbs']], - Path(assembly['kernel']), combined, image / 'linux.bin') - shutil.copyfile(output / 'pongo.bin', image / 'pongo.bin') + Path(assembly['kernel']), combined, image / 'linux.bin', + profile=Path(assembly['profile']) if 'profile' in assembly else None) + if 'pongo.bin' in receipt['files']: + shutil.copyfile(output / 'pongo.bin', image / 'pongo.bin') layered = {**receipt, 'boards': [board], 'firmware': True, - 'files': {name: digest(image / name) for name in ('linux.bin', 'pongo.bin')}} + 'files': {name: digest(image / name) for name in receipt['files']}} layered.pop('overlay', None) layered['sources'] = {} (image / 'build.json').write_text(json.dumps(layered, indent=2) + '\n') @@ -318,7 +361,8 @@ def verify_ram(output): if 'device' in receipt and 'boards' not in receipt: # built before payloads were per platform receipt['boards'], receipt['platform'] = [receipt['device']], None if (receipt.get('kind') not in (None, 'orchard.ram-build') or not receipt.get('boards') or - set(receipt['files']) != {'linux.bin', 'pongo.bin'}): + set(receipt['files']) != ({'linux.bin'} if receipt.get('loader_format') == 'cascadia' else + {'linux.bin', 'pongo.bin'})): raise ValueError('Not a RAM-boot build') for name, expected in receipt['files'].items(): path = output / name diff --git a/tests/hardware_layout.py b/tests/hardware_layout.py index 754e6ce1..1b4c62b7 100644 --- a/tests/hardware_layout.py +++ b/tests/hardware_layout.py @@ -32,7 +32,7 @@ def patches(name): print(platform, dtb or '-') elif sys.argv[1] == '--platform': platform = resolve(sys.argv[2])[0] - print(json.dumps({'platform': platform, 'arch': 'arm64', 'image': 'Image.gz', + print(json.dumps({'platform': platform, 'arch': 'arm64', 'image': 'Image.gz', 'loader': 'm1n1', 'dtbs': 'arch/arm64/boot/dts/apple', 'upstream': 'kernel/upstream.lock', 'series': 'kernel/series/' + platform, 'config': 'kernel/configs/' + platform + '/config'})) diff --git a/tests/test_armv7_producers.py b/tests/test_armv7_producers.py new file mode 100644 index 00000000..166a8135 --- /dev/null +++ b/tests/test_armv7_producers.py @@ -0,0 +1,263 @@ +"""ARMv7 producer boundaries use synthetic ELF and DTB inputs, with no device access.""" +import gzip +import hashlib +import io +import json +from pathlib import Path +import struct +import tarfile +import tempfile +import unittest +from unittest import mock + +from orchard import build, inputs, distro +from orchard.hardware import ROOT +from orchard.image import elf, platform, prepare_runtime + + +def executable(architecture='arm', dynamic=False): + width = 1 if architecture == 'arm' else 2 + data = bytearray(384) + data[:7] = b'\x7fELF' + bytes((width, 1, 1)) + struct.pack_into(' Date: Fri, 6 Nov 2026 14:23:40 -0600 Subject: [PATCH 04/16] recovery: bind ARMv7 RAM sessions to guarded host identity Carry a descriptor-selected host USB identity policy through the checked Cascadia bootargs, bind each upload to its selected hardware and a fresh attempt, and revalidate that provenance in RAM. ARM64 recovery keeps its existing ADT identity path. Stage declared USB dependencies and defer manual storage modules until the host has serial and authenticated SSH. Reject manual drivers built into the kernel or selected as USB dependencies before assembly. Validated with 666 unit tests (one Linux-only skip) and independent cross-layer review. Physical P105 qualification remains pending. Assisted-by: LLM --- boot/initramfs/recovery/init | 31 ++-- boot/initramfs/recovery/usb-gadget | 7 +- chips/s5l8942x.toml | 3 + devices/p105/device.toml | 1 + docs/boot-contract.md | 47 +++++ platforms/apple-armv7.toml | 9 + runtime/recovery/device.py | 3 + runtime/recovery/identity.py | 141 ++++++++++++--- runtime/recovery/policy.py | 55 ++++++ src/orchard/image/cascadia.py | 103 ++++++++++- src/orchard/image/prepare_runtime.py | 38 ++++ src/orchard/image/recovery.py | 8 + src/orchard/image/runtime.py | 3 +- src/orchard/recovery_identity.py | 124 ++++++++++++++ src/orchard/recovery_policy.py | 35 ++++ tests/test_host_identity_handoff.py | 248 +++++++++++++++++++++++++++ tests/test_public_recovery.py | 134 +++++++++++++++ 17 files changed, 953 insertions(+), 37 deletions(-) create mode 100644 runtime/recovery/policy.py create mode 100644 src/orchard/recovery_identity.py create mode 100644 src/orchard/recovery_policy.py create mode 100644 tests/test_host_identity_handoff.py create mode 100644 tests/test_public_recovery.py diff --git a/boot/initramfs/recovery/init b/boot/initramfs/recovery/init index 4b6fd2fa..e8802263 100755 --- a/boot/initramfs/recovery/init +++ b/boot/initramfs/recovery/init @@ -17,19 +17,30 @@ case "$board" in apple,*) ;; *) echo "Not an Apple device for Orchard recovery: $board"; exec sh ;; esac -/sbin/recovery-usb -ip address add 172.16.42.1/30 dev usb0 -ip link set usb0 up -udhcpd -f /etc/udhcpd.conf & -echo "${board#apple,} recovery: USB serial ready; provision an SSH key before SSH starts." -# Respawn unconditionally: a USB re-enumeration hangs up the shell, which then -# exits non-zero, and serial is the only way to provision SSH. +# Serial survives networking failures and repeated USB enumeration. ( while true; do getty -n -l /bin/sh ttyGS0 115200 vt100; sleep 1; done ) & -# Internal storage comes up read-only in the background (storage-fence); the -# host installs over SSH once /run/orchard/storage-ready exists. -/usr/libexec/orchard/storage-fence > /run/orchard/storage.log 2>&1 & +if ! /sbin/recovery-usb > /run/orchard/usb.log 2>&1; then + echo 'Recovery USB setup failed; inspect /run/orchard/usb.log.' + while :; do sleep 60; done +fi +if ip address add 172.16.42.1/30 dev usb0 && ip link set usb0 up; then + udhcpd -f /etc/udhcpd.conf & +else + echo 'Recovery networking failed; USB serial remains available.' +fi +echo "${board#apple,} recovery: USB serial ready; provision an SSH key before SSH starts." +if storage=$(/usr/bin/python3 /usr/libexec/orchard/policy.py storage); then + if [ "$storage" = automatic ]; then + /usr/libexec/orchard/storage-fence > /run/orchard/storage.log 2>&1 & + else + echo manual > /run/orchard/storage-manual + echo 'Storage waits for a manual request after serial and authenticated SSH are ready.' + fi +else + echo 'Recovery storage policy failed; storage remains unloaded.' +fi # After an install, `orchard-device start` asks for the new root: if its boot # entry boots this same kernel, PID 1 switches into it, so the first start # needs no second DFU boot. Later starts go through the entry (orchard launch). diff --git a/boot/initramfs/recovery/usb-gadget b/boot/initramfs/recovery/usb-gadget index e5b7a826..4356b57a 100755 --- a/boot/initramfs/recovery/usb-gadget +++ b/boot/initramfs/recovery/usb-gadget @@ -1,6 +1,9 @@ #!/bin/sh # USB NCM networking and ACM serial, restricted to the attached host. set -eu +network=$(/usr/bin/python3 /usr/libexec/orchard/policy.py network) +modules=$(/usr/bin/python3 /usr/libexec/orchard/policy.py modules) +for module in $modules; do modprobe "$module"; done mkdir -p /sys/kernel/config mount -t configfs -o nodev,noexec,nosuid configfs /sys/kernel/config g=/sys/kernel/config/usb_gadget/linux-recovery @@ -16,8 +19,8 @@ echo 'Orchard Recovery' > "$g/strings/0x409/product" printf '%s\n' "$identity" > "$g/strings/0x409/serialnumber" echo 'USB network and serial' > "$g/configs/c.1/strings/0x409/configuration" echo 500 > "$g/configs/c.1/MaxPower" -mkdir "$g/functions/ncm.usb0" "$g/functions/acm.usb0" -ln -s "$g/functions/ncm.usb0" "$g/configs/c.1/ncm.usb0" +mkdir "$g/functions/$network.usb0" "$g/functions/acm.usb0" +ln -s "$g/functions/$network.usb0" "$g/configs/c.1/$network.usb0" ln -s "$g/functions/acm.usb0" "$g/configs/c.1/acm.usb0" set -- /sys/class/udc/* [ "$#" -eq 1 ] && [ -e "$1" ] diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index 1df5e145..32a4fc52 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -15,3 +15,6 @@ platform = "P105 on a Cascadia-based build" stages = "Project Cascadia P105 boot chain" reference_version = "Project Cascadia builds its A5 boot chain from 8.4.1 / 12H321" cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff awaits hardware qualification" + +# Public Cascadia RAM recovery correlates a guarded host USB identity with the live FDT and boot ID. +# Linux SoC ECID access and on-device public recovery identity remain unqualified. diff --git a/devices/p105/device.toml b/devices/p105/device.toml index bd06785c..95a55378 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -13,6 +13,7 @@ chip = 0x8942 board = 0x0a [boot] +# Public recovery uses a host-bound USB identity handoff; hardware qualification is pending. # Source-built Cascadia payload profile: cascadia.toml. External-initramfs handoff awaits testing. methods = ["jailbreak-kdfu", "recovery-autogo"] entries = ["pico-checkm8", "kdfu"] diff --git a/docs/boot-contract.md b/docs/boot-contract.md index aa7c72fc..290d9bec 100644 --- a/docs/boot-contract.md +++ b/docs/boot-contract.md @@ -199,3 +199,50 @@ the same m1n1 code, splash included. - Every change must be tested on both ways into m1n1: Pongo, and iBoot (`launcher-kit`, `./launch boot --bare`, then the client over the proxy). Device time goes through the user. Which SoCs have an m1n1 storage backend is in [bring-up/bootloader/experiments.md](bring-up/bootloader/experiments.md#per-device-menu). + +## Host-bound ARMv7 recovery identity + +Cascadia recovery templates carry a descriptor-selected identity policy in the +standard `/chosen/bootargs` string. These are userland parameters, not hardware +register descriptions. `orchard.recovery-policy=VALUE` and +`orchard.recovery-handoff=VALUE` encode schema-1 JSON objects as canonical, +unpadded base64url. Each token is bounded; the entire ARM command line remains +at most 1,023 ASCII bytes. Duplicate JSON keys, repeated parameters, unknown +fields, invalid types and incomplete records fail before recovery USB binds. + +The policy names `source = "host-usb"`, canonical `profile`, accepted live FDT +`compatibles`, and integer `chip` and `board` identifiers. The public runtime +contains the same policy, generated from the selected hardware descriptor. +A template contains the policy alone. The guarded transport verifies its +original digest and freshly queries the selected iRecovery endpoint for its +ECID, CPID, BDID and product. It then binds one nonzero random 128-bit +`handoff_id` to that observation and the original `template_sha256`. The handoff +also contains `schema`, `source`, `profile`, `ecid`, `chip` and `board`. The +transport rebuilds the checked CB32 configuration and FDT while retaining the +kernel and initramfs bytes, then rechecks the selected endpoint immediately +before its single upload. Its private attempt receipt records both image +hashes and the nonce. + +The runtime requires the live compatible, policy token and staged policy to +agree. It publishes `identity_source = "host-usb"`, the nonce and original +image digest with the kernel's fresh boot ID. This records a host observation; +it does not claim that Linux read a SoC ECID register. USB serial includes +`HND:` followed by the nonce, alongside the canonical board, ECID and boot ID. +The host requires that nonce before accepting this boot, then checks the full +provenance again through serial SSH bootstrap and authenticated status. A +later command may resume only through the matching private completed attempt +receipt, including the same profile, ECID, nonce, boot ID and template digest. +An unknown or changed handoff stops before a storage operation. + +The existing ARM64 path requires its live ADT MTD identity source. Absence of a +host-bound policy preserves that path; an invalid host-bound policy or missing +handoff never falls back to ADT. This handoff currently serves the guarded +host RAM-boot path. On-device startup needs its own checked identity source +before it can use this public recovery runtime. + +The platform recovery policy also selects USB function dependencies and early +storage activation. ARMv7 loads its declared ECM/ACM modules, keeps serial +available through network setup, and leaves SFTL unloaded until a manual +request after serial and authenticated SSH are ready. The module's finite +read policy bounds that later request. Hardware feature states change only +after the exact submitted stack passes its physical checks. diff --git a/platforms/apple-armv7.toml b/platforms/apple-armv7.toml index 05dba21e..eb21cb70 100644 --- a/platforms/apple-armv7.toml +++ b/platforms/apple-armv7.toml @@ -20,3 +20,12 @@ loader = "cascadia" # the payload format that boots this kernel # H2FMI is built in. The installed NAND backend supplies the early root # setup; the arm64 platforms' NVMe modules do not apply to this platform. storage_modules = [] + +[recovery] +# The guarded host transport binds a fresh USB identity to each RAM boot. +identity_source = "host-usb" +usb_network = "ecm" +usb_modules = ["libcomposite", "usb_f_ecm", "usb_f_acm"] +manual_modules = ["apple_sftl"] +# Mapping recovery begins only after serial and authenticated SSH are ready. +storage_activation = "manual" diff --git a/runtime/recovery/device.py b/runtime/recovery/device.py index bb217220..a1954aa1 100755 --- a/runtime/recovery/device.py +++ b/runtime/recovery/device.py @@ -47,6 +47,9 @@ def bootstrap(request): if type(request.get('ecid')) is not int or any(request.get(key) != record[key] for key in ('profile', 'ecid', 'boot_id')): raise ValueError('SSH bootstrap identity differs from this recovery boot') + if record.get('identity_source') == 'host-usb' and any(type(request.get(key)) is not type(record[key]) or request.get(key) != record[key] for key in + ('identity_source', 'handoff_id', 'template_sha256', 'chip', 'board')): + raise ValueError('SSH bootstrap handoff differs from this recovery attempt') public = request.get('public_key', '') if not isinstance(public, str) or not re.fullmatch(KEY, public): raise ValueError('Expected one Ed25519 public key') diff --git a/runtime/recovery/identity.py b/runtime/recovery/identity.py index 7b622a3d..ca91d859 100644 --- a/runtime/recovery/identity.py +++ b/runtime/recovery/identity.py @@ -1,4 +1,4 @@ -"""Publish the live ADT ECID and kernel boot ID before exposing recovery USB.""" +"""Publish the live ADT identity or a descriptor-bound host USB boot handoff.""" import json import os from pathlib import Path @@ -8,15 +8,93 @@ from adt import validate +try: + from recovery_identity import (HANDOFF_PARAMETER, POLICY_PARAMETER, object_json, + parameters, validate_policy) +except ModuleNotFoundError as error: + if error.name != 'recovery_identity': + raise + from orchard.recovery_identity import (HANDOFF_PARAMETER, POLICY_PARAMETER, object_json, + parameters, validate_policy) + RUN = Path('/run/orchard') BOOT_ID = Path('/proc/sys/kernel/random/boot_id') MTD = Path('/proc/mtd') DEV = Path('/dev') COMPATIBLE = Path('/proc/device-tree/compatible') +POLICY = Path('/etc/orchard/recovery-identity.json') +CMDLINE = Path('/proc/cmdline') +MAX_IDENTITY_BYTES = 2048 + + +def _bounded(path, maximum, *, protected=False): + try: + before = path.lstat() + if (not stat.S_ISREG(before.st_mode) or before.st_size > maximum or + (protected and (before.st_uid != 0 or before.st_mode & 0o022))): + raise ValueError('Identity input is not a protected regular file') + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, 'rb') as stream: + after = os.fstat(stream.fileno()) + if (not stat.S_ISREG(after.st_mode) or + (before.st_dev, before.st_ino) != (after.st_dev, after.st_ino) or + (protected and (after.st_uid != 0 or after.st_mode & 0o022))): + raise ValueError('Identity input changed while opening') + raw = stream.read(maximum + 1) + except OSError as error: + raise ValueError('Identity input is unavailable: ' + str(path)) from error + if len(raw) > maximum: + raise ValueError('Identity input exceeds its bound') + return raw + + +def _policy(): + if not POLICY.exists() and not POLICY.is_symlink(): + return None + return validate_policy(object_json(_bounded(POLICY, MAX_IDENTITY_BYTES, protected=True))) + + +def _compatible(policy): + raw = _bounded(COMPATIBLE, 4096) + if not raw.endswith(b'\0') or b'\0\0' in raw: + raise ValueError('Live compatible is incomplete') + try: + first = raw.split(b'\0')[0].decode('ascii') + except UnicodeError as error: + raise ValueError('Live compatible is invalid') from error + if first not in policy['compatibles']: + raise ValueError('Live compatible differs from the recovery identity policy') + return first + + +def _host_identity(policy): + _compatible(policy) + selected = parameters(_bounded(CMDLINE, MAX_IDENTITY_BYTES)) + if set(selected) != {POLICY_PARAMETER, HANDOFF_PARAMETER}: + raise ValueError('Recovery command line requires its policy and host USB handoff') + if selected[POLICY_PARAMETER] != policy: + raise ValueError('Recovery command-line policy differs from the staged descriptor') + handoff = selected[HANDOFF_PARAMETER] + if any(type(handoff[name]) is not type(policy[name]) or handoff[name] != policy[name] + for name in ('profile', 'chip', 'board')): + raise ValueError('Host USB handoff differs from the recovery identity policy') + handoff_id, template = handoff['handoff_id'], handoff['template_sha256'] + boot = uuid.UUID(BOOT_ID.read_text().strip()) + if not boot.int: + raise ValueError('Invalid live boot ID') + result = {'profile': policy['profile'], 'ecid': handoff['ecid'], 'boot_id': str(boot), + 'identity_source': 'host-usb', 'handoff_id': handoff_id, + 'template_sha256': template, 'chip': handoff['chip'], 'board': handoff['board']} + serial(result) + return result def profile(): - """The board's profile from its first devicetree compatible: \"apple,j71b\" -> \"j71b\".""" + """Use the staged descriptor for handoffs, or the existing ADT board spelling.""" + policy = _policy() + if policy is not None: + _compatible(policy) + return policy['profile'] board = COMPATIBLE.read_bytes().split(b'\0')[0].decode() if not re.fullmatch(r'apple,[a-z0-9]+', board): raise ValueError('Not an Apple board: ' + board) @@ -30,8 +108,17 @@ def cached(): info = path.lstat() if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o077: raise ValueError('Device identity record is not protected') - result = json.loads(path.read_text()) - if (result.get('profile') != profile() or type(result.get('ecid')) is not int or + policy = _policy() + if policy is not None: + result = object_json(_bounded(path, MAX_IDENTITY_BYTES, protected=True)) + expected = _host_identity(policy) + if (set(result) != set(expected) or + any(type(result[name]) is not type(value) or result[name] != value + for name, value in expected.items())): + raise ValueError('Device identity record differs from this boot handoff') + else: + result = json.loads(path.read_text()) + if policy is None and (result.get('profile') != profile() or type(result.get('ecid')) is not int or not 0 < result['ecid'] < 2**64 or result.get('boot_id') != str(uuid.UUID(BOOT_ID.read_text().strip()))): raise ValueError('Device identity record differs from this boot') @@ -41,21 +128,25 @@ def cached(): def initialize(): if cached() is not None: raise ValueError('Device identity was already published') - rows = re.findall(r'^mtd([0-9]+): ([0-9a-fA-F]+) [0-9a-fA-F]+ "adt"$', MTD.read_text(), re.M) - if len(rows) != 1: - raise ValueError('Expected one live ADT MTD entry') - number, size = rows[0][0], int(rows[0][1], 16) - if not 8 <= size <= 16 * 1024**2: - raise ValueError('Live ADT length outside bounds') - fd = os.open(DEV / ('mtd' + number), os.O_RDONLY | os.O_NOFOLLOW) - with os.fdopen(fd, 'rb') as stream: - if not stat.S_ISCHR(os.fstat(stream.fileno()).st_mode): - raise ValueError('Live ADT must be an MTD character device') - blob = stream.read(size) - if len(blob) != size: - raise ValueError('Live ADT read was truncated') - result = {'profile': profile(), 'ecid': validate(blob), - 'boot_id': str(uuid.UUID(BOOT_ID.read_text().strip()))} + policy = _policy() + if policy is not None: + result = _host_identity(policy) + else: + rows = re.findall(r'^mtd([0-9]+): ([0-9a-fA-F]+) [0-9a-fA-F]+ "adt"$', MTD.read_text(), re.M) + if len(rows) != 1: + raise ValueError('Expected one live ADT MTD entry') + number, size = rows[0][0], int(rows[0][1], 16) + if not 8 <= size <= 16 * 1024**2: + raise ValueError('Live ADT length outside bounds') + fd = os.open(DEV / ('mtd' + number), os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, 'rb') as stream: + if not stat.S_ISCHR(os.fstat(stream.fileno()).st_mode): + raise ValueError('Live ADT must be an MTD character device') + blob = stream.read(size) + if len(blob) != size: + raise ValueError('Live ADT read was truncated') + result = {'profile': profile(), 'ecid': validate(blob), + 'boot_id': str(uuid.UUID(BOOT_ID.read_text().strip()))} RUN.mkdir(mode=0o700, parents=True, exist_ok=True) if RUN.is_symlink(): raise ValueError('Linked runtime state refused') @@ -68,6 +159,14 @@ def initialize(): return result +def serial(record): + result = f'ECID:{record["ecid"]:016x} BOOT:{record["boot_id"]} BOARD:{record["profile"]}' + if record.get('identity_source') == 'host-usb': + result += ' HND:' + record['handoff_id'] + if len(result) > 126: + raise ValueError('Recovery identity exceeds the USB serial string limit') + return result + + if __name__ == '__main__': - record = initialize() - print(f'ECID:{record["ecid"]:016x} BOOT:{record["boot_id"]} BOARD:{record["profile"]}') + print(serial(initialize())) diff --git a/runtime/recovery/policy.py b/runtime/recovery/policy.py new file mode 100644 index 00000000..7a753692 --- /dev/null +++ b/runtime/recovery/policy.py @@ -0,0 +1,55 @@ +"""Read the descriptor-selected early recovery policy, with legacy defaults.""" +from pathlib import Path +import re +import stat +import os +import sys + +from recovery_identity import object_json, validate_policy + +CONFIG = Path('/etc/orchard/recovery.json') +IDENTITY = Path('/etc/orchard/recovery-identity.json') + + +def settings(): + if not CONFIG.exists() and not CONFIG.is_symlink(): + return {'usb_network': 'ncm', 'usb_modules': [], 'storage_activation': 'automatic'} + info = CONFIG.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022 or info.st_size > 2048: + raise ValueError('Recovery policy is not protected') + value = object_json(CONFIG.read_bytes()) + fields = {'schema', 'profile', 'usb_network', 'usb_modules', 'manual_modules', 'storage_activation'} + if (set(value) != fields or type(value['schema']) is not int or value['schema'] != 1 or + value['usb_network'] not in ('ncm', 'ecm') or value['storage_activation'] not in ('manual', 'automatic')): + raise ValueError('Invalid recovery runtime policy') + identity = validate_policy(object_json(IDENTITY.read_bytes())) + if value['profile'] != identity['profile']: + raise ValueError('Recovery policy profiles differ') + for key in ('usb_modules', 'manual_modules'): + names = value[key] + if (not isinstance(names, list) or any(not isinstance(name, str) or not re.fullmatch('[a-z0-9_-]+', name) for name in names) or + len(set(names)) != len(names)): + raise ValueError('Invalid recovery module policy') + normalized = lambda names: {name.replace('-', '_') for name in names} + if normalized(value['usb_modules']) & normalized(value['manual_modules']): + raise ValueError('A manual module cannot be loaded during USB setup') + return value + + +def loadable_modules(value, directory=None): + names = value['usb_modules'] + if not names: + return [] + directory = Path(directory) if directory is not None else Path('/lib/modules') / os.uname().release + builtins = {Path(line).stem.replace('-', '_') for line in + (directory / 'modules.builtin').read_text().splitlines() if line} + return [name for name in names if name.replace('-', '_') not in builtins] + + +if __name__ == '__main__': + value = settings() + selected = {'network': value['usb_network'], 'modules': ' '.join(loadable_modules(value)), + 'storage': value['storage_activation']} + if len(sys.argv) != 2 or sys.argv[1] not in selected: + raise SystemExit('Expected network, modules or storage') + print(selected[sys.argv[1]]) diff --git a/src/orchard/image/cascadia.py b/src/orchard/image/cascadia.py index 6b02caff..d3c899de 100644 --- a/src/orchard/image/cascadia.py +++ b/src/orchard/image/cascadia.py @@ -11,9 +11,16 @@ import json from pathlib import Path import struct +import sys import tomllib import zlib +if __package__: + from .. import recovery_identity as identity_protocol +else: # The distribution contract also runs this producer by file path. + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + import recovery_identity as identity_protocol + CONFIG_OFFSET = 0x7f00 CONFIG = struct.Struct('<4s16I') @@ -241,7 +248,7 @@ def chosen(data, bootargs, start, size): if not isinstance(bootargs, str) or any(char in bootargs for char in ('\0', '\n', '\r')): raise ValueError('Kernel arguments must be one line') encoded = bootargs.encode('utf-8') - if len(encoded) > 2047: + if len(encoded) > identity_protocol.MAX_CMDLINE_BYTES: raise ValueError('Kernel arguments exceed the ARM command-line bound') root, reservations, cpu = read_fdt(data) cell_bytes = root['properties'].get('#address-cells', struct.pack('>I', 2)) @@ -267,6 +274,16 @@ def chosen(data, bootargs, start, size): return write_fdt(root, reservations, cpu) +def _policy_compatible(data, policy): + check_compatible(data, policy['compatibles']) + tree, _, _ = read_fdt(data) + raw = tree['properties']['compatible'] + if not raw.endswith(b'\0') or b'\0\0' in raw: + raise ValueError('Recovery DTB compatible list is incomplete') + if raw.split(b'\0')[0].decode('ascii') not in policy['compatibles']: + raise ValueError('First DTB compatible differs from the recovery identity policy') + + def decompression_bound(kernel, dtb_size, described): """Conservatively bound the v6.12 ARM decompressor's relocated workspace. @@ -301,7 +318,7 @@ def decompression_bound(kernel, dtb_size, described): return bound -def assemble(loader, kernel, dtb, initramfs, described, bootargs, *, elf=None): +def assemble(loader, kernel, dtb, initramfs, described, bootargs, *, elf=None, identity_policy=None): validate_profile(described) verify_loader(loader, described, elf) if (len(kernel) < 0x30 or struct.unpack_from(' described['initrd_limit'] - described['initrd_address']: raise ValueError('Initramfs exceeds its reserved destination') + if identity_policy is not None: + policy = identity_protocol.validate_policy(identity_policy) + if identity_protocol.parameters(bootargs): + raise ValueError('Recovery template already contains identity parameters') + _policy_compatible(dtb, policy) + bootargs = bootargs + ' ' + identity_protocol.encode_parameter(identity_protocol.POLICY_PARAMETER, policy) tree = chosen(dtb, bootargs, described['initrd_address'], len(initramfs)) expanded_end = decompression_bound(kernel, len(tree), described) kernel_at = described['kernel_offset'] @@ -348,6 +371,77 @@ def append(name, payload): 'input_loader_sha256': digest(loader), 'input_dtb_sha256': digest(dtb)} +def _payload_parts(data): + """Validate a CB32 bundle before inspecting or binding its identity.""" + values = configuration(data) + if (values['image_size'] != len(data) or values['kernel_offset'] != 0x8000 or + data[:4] != bytes.fromhex('c0000cf1') or + values['crc32'] != zlib.crc32(data[CONFIG_OFFSET:CONFIG_OFFSET + CONFIG.size - 4])): + raise ValueError('Invalid Cascadia payload configuration') + kernel_at, dtb_at, dtb_size = (values[key] for key in ('kernel_offset', 'dtb_offset', 'dtb_size')) + initrd_at, initrd_size = values['initrd_offset'], values['initrd_size'] + if (not kernel_at + 0x3c <= dtb_at < dtb_at + dtb_size <= initrd_at or + initrd_at != aligned(dtb_at + dtb_size) or initrd_at + initrd_size != len(data) or + any(data[dtb_at + dtb_size:initrd_at])): + raise ValueError('Invalid Cascadia payload component ranges') + described = {'schema': 1, **{key: values[key] for key in PROFILE_FIELDS - {'schema'}}} + validate_profile(described) + if values['load_address'] + len(data) > values['initrd_address']: + raise ValueError('Cascadia payload overlaps its initramfs destination') + tree, _, _ = read_fdt(data[dtb_at:dtb_at + dtb_size]) + selected = next((child for child in tree['children'] if child['name'] == 'chosen'), None) + raw = selected['properties'].get('bootargs', b'') if selected else b'' + if not raw.endswith(b'\0') or b'\0' in raw[:-1]: + raise ValueError('Cascadia payload lacks one complete bootargs value') + try: + bootargs = raw[:-1].decode('utf-8') + except UnicodeError as error: + raise ValueError('Cascadia bootargs are not UTF-8') from error + return values, described, bootargs + + +def host_identity_policy(data): + """Return the declared host-bound recovery policy of an unbound template.""" + values, _, bootargs = _payload_parts(data) + if 'orchard.recovery-' not in bootargs: + return None + parameters = identity_protocol.parameters(bootargs) + if identity_protocol.HANDOFF_PARAMETER in parameters: + raise ValueError('A public recovery template already contains an identity handoff') + policy = parameters.get(identity_protocol.POLICY_PARAMETER) + if policy is None: + raise ValueError('Recovery identity parameters lack a policy') + start = values['dtb_offset'] + _policy_compatible(data[start:start + values['dtb_size']], policy) + return policy + + +def bind_host_identity(data, *, ecid, chip, board, profile, handoff_id, template_sha256): + """Bind a guarded host USB observation to one complete, revalidated bundle.""" + policy = host_identity_policy(data) + if policy is None: + raise ValueError('Payload does not declare host-bound recovery identity') + if (profile, chip, board) != (policy['profile'], policy['chip'], policy['board']): + raise ValueError('Selected device differs from the recovery identity policy') + if template_sha256 != digest(data): + raise ValueError('Recovery template digest differs before identity binding') + handoff = identity_protocol.validate_handoff({'schema': 1, 'source': 'host-usb', + 'profile': profile, 'ecid': ecid, 'chip': chip, 'board': board, + 'handoff_id': handoff_id, 'template_sha256': template_sha256}) + values, described, bootargs = _payload_parts(data) + loader = bytearray(data[:values['kernel_offset']]) + pristine = dict(values) + for key in ('dtb_offset', 'dtb_size', 'initrd_offset', 'initrd_size', 'image_size', 'crc32'): + pristine[key] = 0 + loader[CONFIG_OFFSET:CONFIG_OFFSET + CONFIG.size] = CONFIG.pack(*(pristine[key] for key in CONFIG_FIELDS)) + bootargs += ' ' + identity_protocol.encode_parameter(identity_protocol.HANDOFF_PARAMETER, handoff) + output, _ = assemble(bytes(loader), data[values['kernel_offset']:values['dtb_offset']], + data[values['dtb_offset']:values['dtb_offset'] + values['dtb_size']], + data[values['initrd_offset']:], described, bootargs) + _payload_parts(output) + return output + + def main(): parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) given = parser.add_mutually_exclusive_group(required=True) @@ -356,6 +450,7 @@ def main(): for name in ('loader', 'kernel', 'initramfs', 'profile', 'output'): parser.add_argument('--' + name, type=Path, required=True) parser.add_argument('--loader-elf', type=Path) + parser.add_argument('--identity-policy', type=Path, help='descriptor-selected public recovery identity policy') parser.add_argument('--dtb', action='append', type=Path, required=True) parser.add_argument('--logo', type=Path) args = parser.parse_args() @@ -376,7 +471,9 @@ def main(): bootargs = bootargs[len(prefix):-1] data, document = assemble(args.loader.read_bytes(), args.kernel.read_bytes(), args.dtb[0].read_bytes(), args.initramfs.read_bytes(), profile(args.profile), bootargs, - elf=args.loader_elf.read_bytes() if args.loader_elf else None) + elf=args.loader_elf.read_bytes() if args.loader_elf else None, + identity_policy=identity_protocol.validate_policy(identity_protocol.object_json(args.identity_policy.read_bytes())) + if args.identity_policy else None) except (ValueError, KeyError) as error: parser.error(str(error)) document['profile_sha256'] = digest(args.profile.read_bytes()) diff --git a/src/orchard/image/prepare_runtime.py b/src/orchard/image/prepare_runtime.py index 576b068c..7bc79ac1 100644 --- a/src/orchard/image/prepare_runtime.py +++ b/src/orchard/image/prepare_runtime.py @@ -122,13 +122,38 @@ def stage_runtime(base, receipt, root, architecture='arm64'): raise ValueError('Python runtime dependency closure did not converge') +def check_recovery_modules(root, release, recovery): + """Manual storage must stay modular; declared USB dependencies must exist.""" + normalized = lambda names: {name.replace('-', '_') for name in names} + if normalized(recovery['runtime']['usb_modules']) & normalized(recovery['runtime']['manual_modules']): + raise ValueError('A manual module cannot be loaded during USB setup') + directory = root / 'usr/lib/modules' / release + builtins = {Path(name).stem.replace('-', '_') for name in + (directory / 'modules.builtin').read_text().splitlines() if name} + available = {path.stem.replace('-', '_') for path in directory.rglob('*.ko')} + for name in recovery['runtime']['manual_modules']: + if name.replace('-', '_') in builtins: + raise ValueError('Manual recovery module is built in: ' + name) + for name in recovery['runtime']['usb_modules']: + if name.replace('-', '_') not in builtins | available: + raise ValueError('Recovery USB module is unavailable: ' + name) + + def main(): parser = argparse.ArgumentParser(description=__doc__) for name in ('busybox', 'dropbear', 'dropbearkey', 'wpa-supplicant', 'wpa-cli', 'iw', 'base', 'base-receipt', 'modules-dir', 'root-receipt', 'output'): parser.add_argument('--' + name, type=Path, required=True) parser.add_argument('--architecture', choices=('arm', 'arm64'), default='arm64') + parser.add_argument('--device', help='stage the selected board recovery policy') args = parser.parse_args() + recovery = None + if args.device: + from ..recovery_policy import settings + from ..kernel import descriptor + if descriptor(PROJECT, args.device)['arch'] != args.architecture: + parser.error('Recovery board architecture differs from the runtime') + recovery = settings(args.device) root = args.output if root.exists(): parser.error('Choose a new staging directory') @@ -150,6 +175,8 @@ def main(): path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) path.chmod(mode) + if recovery is not None: + check_recovery_modules(root, kernel_release, recovery) # The release this recovery belongs to; `orchard-device status` reports it. # Root images carry no kernel: the release's platform layer of the same # kernel bundle does, so the recovery and an installed root match. @@ -165,6 +192,17 @@ def main(): (root / 'etc/group').write_text('root:x:0:\n') (root / 'etc/shadow').write_text('root:!:0:0:99999:7:::\n') (root / 'etc/shadow').chmod(0o600) + if recovery is not None: + for name, value in (('recovery-identity.json', recovery['identity']), ('recovery.json', recovery['runtime'])): + path = root / 'etc/orchard' / name + path.write_text(json.dumps(value, sort_keys=True) + '\n') + path.chmod(0o644) + manual = recovery['runtime']['manual_modules'] + if manual: + blacklist = root / 'etc/modprobe.d/orchard-recovery.conf' + blacklist.parent.mkdir(parents=True, exist_ok=True) + blacklist.write_text(''.join('blacklist ' + name + '\n' for name in manual)) + blacklist.chmod(0o644) print('Public recovery staged:', root) diff --git a/src/orchard/image/recovery.py b/src/orchard/image/recovery.py index 87babe68..70d84a01 100644 --- a/src/orchard/image/recovery.py +++ b/src/orchard/image/recovery.py @@ -29,6 +29,11 @@ def main(): from .platform import boards described = descriptor(ROOT, args.platform) profile = None + selected = args.device + if selected: + from ..hardware import device as described_device + if described_device(selected)['kernel']['platform'] != described['platform']: + parser.error('Recovery board does not belong to the selected platform') if described['loader'] == 'cascadia': if len(args.dtb) != 1 or args.logo is not None: parser.error('Cascadia recovery takes one board DTB and no m1n1 logo') @@ -65,6 +70,7 @@ def main(): '--base', str(args.base), '--base-receipt', str(args.base_receipt), '--modules-dir', str(args.modules_dir), '--root-receipt', str(args.root_receipt), '--architecture', described['arch'], + *(['--device', selected] if selected else []), '--output', str(stage / 'root')], check=True) subprocess.run([sys.executable, '-m', 'orchard.image.initramfs', '--root', str(stage / 'root'), @@ -72,6 +78,8 @@ def main(): assembly = 'orchard.image.cascadia' if profile is not None else 'orchard.image.assemble' subprocess.run([sys.executable, '-m', assembly, *(['--profile', str(profile)] if profile is not None else []), + *(['--identity-policy', str(stage / 'root/etc/orchard/recovery-identity.json')] + if (stage / 'root/etc/orchard/recovery-identity.json').is_file() else []), '--manifest', str(HERE / manifest), '--loader', str(args.loader), *(f'--dtb={dtb}' for dtb in args.dtb), *(['--logo', str(args.logo)] if args.logo else []), diff --git a/src/orchard/image/runtime.py b/src/orchard/image/runtime.py index 577717ae..595f3bec 100644 --- a/src/orchard/image/runtime.py +++ b/src/orchard/image/runtime.py @@ -63,7 +63,8 @@ def recovery_files(): prefix = 'usr/libexec/orchard/' source = PROJECT / 'runtime/recovery' result = {prefix + name: ((source / name).read_bytes(), 0o755) - for name in ('device.py', 'identity.py', 'storage-fence', 'start.py')} + for name in ('device.py', 'identity.py', 'policy.py', 'storage-fence', 'start.py')} + result[prefix + 'recovery_identity.py'] = ((PROJECT / 'src/orchard/recovery_identity.py').read_bytes(), 0o644) result[prefix + 'adt.py'] = ((PROJECT / 'src/orchard/apple/adt.py').read_bytes(), 0o644) result['usr/bin/orchard-device'] = (b'#!/bin/sh\nexec /usr/bin/python3 /usr/libexec/orchard/device.py "$@"\n', 0o755) return result diff --git a/src/orchard/recovery_identity.py b/src/orchard/recovery_identity.py new file mode 100644 index 00000000..91c56bb0 --- /dev/null +++ b/src/orchard/recovery_identity.py @@ -0,0 +1,124 @@ +"""Bounded host USB identity parameters shared by payloads and RAM recovery.""" +import base64 +import binascii +import json +import re + +POLICY_PARAMETER = 'orchard.recovery-policy' +HANDOFF_PARAMETER = 'orchard.recovery-handoff' +MAX_ENCODED_BYTES = 2048 +MAX_CMDLINE_BYTES = 1023 +POLICY_FIELDS = frozenset(('schema', 'source', 'profile', 'compatibles', 'chip', 'board')) +HANDOFF_FIELDS = frozenset(('schema', 'source', 'profile', 'ecid', 'chip', 'board', + 'handoff_id', 'template_sha256')) + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Identity JSON repeats a field') + result[key] = value + return result + + +def object_json(raw): + """Decode one bounded JSON object with no duplicate keys or NUL bytes.""" + if not isinstance(raw, bytes) or len(raw) > MAX_ENCODED_BYTES or b'\0' in raw: + raise ValueError('Identity JSON exceeds its bound or contains a NUL') + try: + result = json.loads(raw.decode('utf-8'), object_pairs_hook=_pairs) + except (UnicodeError, json.JSONDecodeError) as error: + raise ValueError('Invalid identity JSON') from error + if not isinstance(result, dict): + raise ValueError('Identity JSON must be an object') + return result + + +def _selected(result, fields): + if (not isinstance(result, dict) or set(result) != fields or + type(result.get('schema')) is not int or result['schema'] != 1 or + result.get('source') != 'host-usb' or not isinstance(result.get('profile'), str) or + not re.fullmatch(r'[a-z0-9][a-z0-9._-]*', result['profile'])): + raise ValueError('Unsupported recovery identity fields') + if (type(result.get('chip')) is not int or not 0 < result['chip'] <= 0xffffffff or + type(result.get('board')) is not int or not 0 <= result['board'] <= 0xffffffff): + raise ValueError('Recovery identity has invalid chip identity') + + +def validate_policy(result): + _selected(result, POLICY_FIELDS) + compatibles = result.get('compatibles') + if (not isinstance(compatibles, list) or not 1 <= len(compatibles) <= 32 or + any(not isinstance(value, str) or not re.fullmatch(r'apple,[a-zA-Z0-9._+-]{1,120}', value) + for value in compatibles) or len(set(compatibles)) != len(compatibles)): + raise ValueError('Recovery identity policy has invalid compatibles') + return result + + +def validate_handoff(result): + _selected(result, HANDOFF_FIELDS) + if type(result.get('ecid')) is not int or not 0 < result['ecid'] < 2**64: + raise ValueError('Host USB handoff has an invalid ECID') + handoff_id, template = result.get('handoff_id'), result.get('template_sha256') + if (not isinstance(handoff_id, str) or not re.fullmatch(r'[0-9a-f]{32}', handoff_id) or + int(handoff_id, 16) == 0 or not isinstance(template, str) or + not re.fullmatch(r'[0-9a-f]{64}', template)): + raise ValueError('Host USB handoff has invalid attempt provenance') + return result + + +def _validate(name, value): + if name == POLICY_PARAMETER: + return validate_policy(value) + if name == HANDOFF_PARAMETER: + return validate_handoff(value) + raise ValueError('Unknown recovery identity parameter') + + +def encode_parameter(name, value): + value = _validate(name, value) + raw = json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode('ascii') + encoded = base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii') + if len(encoded) > MAX_ENCODED_BYTES: + raise ValueError('Recovery identity token exceeds its bound') + return name + '=' + encoded + + +def _decode(name, value): + if not 1 <= len(value) <= MAX_ENCODED_BYTES or not re.fullmatch(r'[A-Za-z0-9_-]+', value): + raise ValueError('Invalid recovery handoff token') + try: + raw = base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True) + except (ValueError, binascii.Error) as error: + raise ValueError('Invalid recovery handoff encoding') from error + if base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii') != value: + raise ValueError('Recovery handoff encoding is not canonical') + return _validate(name, object_json(raw)) + + +def parameters(cmdline): + """Return present identity parameters; callers decide which ones are required.""" + if isinstance(cmdline, bytes): + try: + cmdline = cmdline.decode('ascii') + except UnicodeError as error: + raise ValueError('Recovery command line is not ASCII') from error + if not isinstance(cmdline, str) or not cmdline.isascii(): + raise ValueError('Recovery command line is not ASCII') + # /proc/cmdline adds one newline to the kernel's bootargs string. + if cmdline.endswith('\n'): + cmdline = cmdline[:-1] + if len(cmdline) > MAX_CMDLINE_BYTES or any(value in cmdline for value in ('\0', '\r', '\n')): + raise ValueError('Recovery command line exceeds its bound or contains a control character') + result = {} + for token in cmdline.split(): + name, separator, value = token.partition('=') + if name not in (POLICY_PARAMETER, HANDOFF_PARAMETER): + if name.startswith('orchard.recovery-'): + raise ValueError('Unknown recovery identity parameter') + continue + if name in result or not separator: + raise ValueError('Repeated or incomplete recovery identity parameter') + result[name] = _decode(name, value) + return result diff --git a/src/orchard/recovery_policy.py b/src/orchard/recovery_policy.py new file mode 100644 index 00000000..065cdc81 --- /dev/null +++ b/src/orchard/recovery_policy.py @@ -0,0 +1,35 @@ +"""Descriptor-selected recovery identity and early runtime policy.""" +from pathlib import Path +import re +import tomllib + + +def settings(board, hardware=None): + from .hardware import ROOT, device + from .kernel import descriptor + from .recovery_identity import validate_policy + hardware = Path(hardware or ROOT) + selected = device(board, hardware) + platform = descriptor(hardware, board)['platform'] + table = tomllib.loads((hardware / 'platforms' / (platform + '.toml')).read_text()).get('recovery') + if table is None: + return None + expected = {'identity_source', 'usb_network', 'usb_modules', 'manual_modules', 'storage_activation'} + if (not isinstance(table, dict) or set(table) != expected or table['identity_source'] != 'host-usb' or + table['usb_network'] not in ('ecm', 'ncm') or table['storage_activation'] not in ('manual', 'automatic')): + raise ValueError('Invalid platform recovery policy') + modules = table['usb_modules'] + for key in ('usb_modules', 'manual_modules'): + names = table[key] + if (not isinstance(names, list) or any(not isinstance(name, str) or not re.fullmatch('[a-z0-9_-]+', name) for name in names) or + len(set(names)) != len(names)): + raise ValueError('Recovery modules must be distinct module names') + normalized = lambda names: {name.replace('-', '_') for name in names} + if normalized(table['usb_modules']) & normalized(table['manual_modules']): + raise ValueError('A manual module cannot be loaded during USB setup') + identity = validate_policy({'schema': 1, 'source': 'host-usb', 'profile': selected['device']['profile'], + 'compatibles': selected['device']['dt_compatible'], 'chip': selected['match']['dfu']['chip'], + 'board': selected['match']['dfu']['board']}) + runtime = {'schema': 1, 'profile': identity['profile'], 'usb_network': table['usb_network'], + 'usb_modules': modules, 'manual_modules': table['manual_modules'], 'storage_activation': table['storage_activation']} + return {'identity': identity, 'runtime': runtime} diff --git a/tests/test_host_identity_handoff.py b/tests/test_host_identity_handoff.py new file mode 100644 index 00000000..2a3e47c4 --- /dev/null +++ b/tests/test_host_identity_handoff.py @@ -0,0 +1,248 @@ +"""Descriptor-bound public recovery identity; fixtures never access a device.""" +import base64 +import importlib.util +import json +import os +from pathlib import Path +import stat +import sys +import tempfile +import unittest +from unittest import mock + +from orchard.apple import adt +from orchard import recovery_identity as codec + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('host_identity_fixture', ROOT/'runtime/recovery/identity.py') +identity = importlib.util.module_from_spec(spec) +with mock.patch.dict(sys.modules, {'adt': adt}): + spec.loader.exec_module(identity) +BOOT = '11111111-1111-4111-8111-111111111111' +POLICY = {'schema': 1, 'source': 'host-usb', 'profile': 'p105', + 'compatibles': ['apple,p105ap'], 'chip': 0x8942, 'board': 0x0a} +HANDOFF = {'schema': 1, 'source': 'host-usb', 'profile': 'p105', 'ecid': 123, + 'chip': 0x8942, 'board': 0x0a, 'handoff_id': 'a'*32, 'template_sha256': 'b'*64} + + +def raw_token(name, raw): + return name + '=' + base64.urlsafe_b64encode(raw).rstrip(b'=').decode() + + +class IdentityCodec(unittest.TestCase): + def test_roundtrip_partial_and_complete_parameters(self): + policy = codec.encode_parameter(codec.POLICY_PARAMETER, POLICY) + handoff = codec.encode_parameter(codec.HANDOFF_PARAMETER, HANDOFF) + self.assertEqual(codec.parameters('console=tty0'), {}) + self.assertEqual(codec.parameters('console=tty0 ' + policy), {codec.POLICY_PARAMETER: POLICY}) + self.assertEqual(codec.parameters((policy+' '+handoff+'\n').encode()), + {codec.POLICY_PARAMETER: POLICY, codec.HANDOFF_PARAMETER: HANDOFF}) + + def test_arm_command_line_boundary_and_truncated_handoff(self): + policy = codec.encode_parameter(codec.POLICY_PARAMETER, POLICY) + handoff = codec.encode_parameter(codec.HANDOFF_PARAMETER, HANDOFF) + tokens = policy + ' ' + handoff + full = 'x' * (1023-len(tokens)-1) + ' ' + tokens + self.assertEqual(len(full), 1023) + self.assertEqual(codec.parameters(full + '\n'), + {codec.POLICY_PARAMETER: POLICY, codec.HANDOFF_PARAMETER: HANDOFF}) + with self.assertRaisesRegex(ValueError, 'bound'): + codec.parameters('x' + full) + truncated = ('x' * (1024-len(handoff)-1) + ' ' + handoff)[:1023] + self.assertEqual(len(truncated), 1023) + with self.assertRaises(ValueError): + codec.parameters(truncated) + + def test_ambiguous_malformed_and_oversized_tokens_fail(self): + policy = codec.encode_parameter(codec.POLICY_PARAMETER, POLICY) + handoff = codec.encode_parameter(codec.HANDOFF_PARAMETER, HANDOFF) + duplicate = json.dumps(HANDOFF).replace('"ecid": 123', '"ecid": 0, "ecid": 123').encode() + for line in (policy+' '+policy, handoff+' '+handoff, codec.HANDOFF_PARAMETER, + policy+'=', codec.HANDOFF_PARAMETER+'=!', + raw_token(codec.HANDOFF_PARAMETER, duplicate), + raw_token(codec.HANDOFF_PARAMETER, b'{}\0'), + raw_token(codec.HANDOFF_PARAMETER, b'[]'), + 'orchard.recovery-unknown=e30', + handoff+'\0', handoff+'\r', handoff+'\n\n', 'x'*1024): + with self.subTest(line=line[:90]), self.assertRaises(ValueError): + codec.parameters(line) + # Equivalent decoded bytes with nonzero unused pad bits are rejected. + with self.assertRaisesRegex(ValueError, 'canonical'): + codec.parameters(codec.HANDOFF_PARAMETER+'=e31') + + def test_selected_fields_and_attempt_types_are_exact(self): + changes = ({'schema': True}, {'schema': 2}, {'source': 'configuration'}, {'profile': ''}, + {'ecid': True}, {'ecid': 0}, {'ecid': 2**64}, {'chip': True}, {'board': False}, + {'handoff_id': '0'*32}, {'handoff_id': 'A'*32}, {'handoff_id': 'a'*31}, + {'template_sha256': 'x'*64}, {'extra': 1}) + for change in changes: + with self.subTest(change=change), self.assertRaises(ValueError): + codec.encode_parameter(codec.HANDOFF_PARAMETER, {**HANDOFF, **change}) + for change in ({'compatibles': []}, {'compatibles': ['apple,p105ap']*2}, + {'compatibles': [None]}, {'compatibles': ['other,p105']}, + {'board': -1}, {'chip': 0}, {'schema': True}, {'extra': 1}): + with self.subTest(change=change), self.assertRaises(ValueError): + codec.encode_parameter(codec.POLICY_PARAMETER, {**POLICY, **change}) + + +class HostIdentity(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + for name, relative in (('RUN', 'run'), ('DEV', 'dev'), ('MTD', 'mtd'), + ('BOOT_ID', 'boot-id'), ('COMPATIBLE', 'compatible'), + ('POLICY', 'policy'), ('CMDLINE', 'cmdline')): + patch = mock.patch.object(identity, name, self.root/relative) + patch.start() + self.addCleanup(patch.stop) + identity.BOOT_ID.write_text(BOOT) + identity.COMPATIBLE.write_bytes(b'apple,p105ap\0apple,s5l8942x\0') + identity.POLICY.write_text(json.dumps(POLICY, indent=2)) + identity.POLICY.chmod(0o644) + self.command_line() + original_lstat, original_fstat = Path.lstat, os.fstat + + def root_stat(value): + fields = list(value) + fields[4] = 0 # Emulate root-owned image files on a non-root build host. + return os.stat_result(fields) + + def lstat(path): + result = original_lstat(path) + if path == identity.POLICY or path == identity.RUN/'identity.json': + result = root_stat(result) + return result + + patch = mock.patch.object(Path, 'lstat', lstat) + patch.start() + self.addCleanup(patch.stop) + patch = mock.patch.object(os, 'fstat', lambda fd: root_stat(original_fstat(fd))) + patch.start() + self.addCleanup(patch.stop) + + def command_line(self, *, policy=POLICY, handoff=HANDOFF): + tokens = ['console=tty0'] + if policy is not None: + tokens.append(codec.encode_parameter(codec.POLICY_PARAMETER, policy)) + if handoff is not None: + tokens.append(codec.encode_parameter(codec.HANDOFF_PARAMETER, handoff)) + identity.CMDLINE.write_text(' '.join(tokens)+'\n') + + def test_canonical_profile_live_board_attempt_and_private_cache(self): + with mock.patch.object(identity, 'validate', side_effect=AssertionError('ADT fallback')): + record = identity.initialize() + self.assertEqual(record, {'profile': 'p105', 'ecid': 123, 'boot_id': BOOT, + 'identity_source': 'host-usb', 'handoff_id': 'a'*32, + 'template_sha256': 'b'*64, 'chip': 0x8942, 'board': 0x0a}) + self.assertEqual(identity.profile(), 'p105') + self.assertEqual(identity.cached(), record) + self.assertEqual(stat.S_IMODE((identity.RUN/'identity.json').stat().st_mode), 0o600) + serial = identity.serial(record) + self.assertTrue(serial.endswith(' BOARD:p105 HND:'+'a'*32)) + self.assertLessEqual(len(serial), 126) + with self.assertRaisesRegex(ValueError, 'already published'): + identity.initialize() + + def test_missing_handoff_or_policy_token_does_not_fall_back_to_adt(self): + for policy, handoff in ((POLICY, None), (None, HANDOFF), (None, None)): + self.command_line(policy=policy, handoff=handoff) + with self.subTest(policy=policy, handoff=handoff), \ + mock.patch.object(identity, 'validate', side_effect=AssertionError('ADT fallback')), \ + self.assertRaisesRegex(ValueError, 'policy and host USB handoff'): + identity.initialize() + self.assertFalse((identity.RUN/'identity.json').exists()) + + def test_descriptor_policy_and_live_first_compatible_must_match(self): + for raw in (b'apple,p105\0', b'apple,j1\0', b'apple,s5l8942x\0apple,p105ap\0', + b'apple,p105ap', b'apple,p105ap\0\0'): + identity.COMPATIBLE.write_bytes(raw) + with self.subTest(raw=raw), self.assertRaises(ValueError): + identity.initialize() + identity.COMPATIBLE.write_bytes(b'apple,p105ap\0') + self.command_line(policy={**POLICY, 'compatibles': ['apple,p105ap', 'apple,j1']}) + with self.assertRaisesRegex(ValueError, 'staged descriptor'): + identity.initialize() + for key, value in (('profile', 'j1'), ('chip', 0x8945), ('board', 1)): + self.command_line(handoff={**HANDOFF, key: value}) + with self.subTest(key=key), self.assertRaisesRegex(ValueError, 'differs'): + identity.initialize() + + def test_policy_must_be_bounded_root_owned_and_not_writable_or_linked(self): + root_lstat = Path.lstat + + def unowned(path): + result = root_lstat(path) + if path == identity.POLICY: + fields = list(result) + fields[4] = 501 + result = os.stat_result(fields) + return result + + with mock.patch.object(Path, 'lstat', unowned), \ + self.assertRaisesRegex(ValueError, 'protected regular'): + identity.initialize() + identity.POLICY.chmod(0o664) + with self.assertRaisesRegex(ValueError, 'protected regular'): + identity.initialize() + identity.POLICY.chmod(0o644) + identity.POLICY.write_bytes(b'x'*2049) + with self.assertRaisesRegex(ValueError, 'protected regular'): + identity.initialize() + identity.POLICY.unlink() + identity.POLICY.symlink_to(self.root/'missing') + with self.assertRaisesRegex(ValueError, 'protected regular'): + identity.initialize() + + def test_cached_record_rechecks_attempt_digest_board_and_boot(self): + record = identity.initialize() + for key, value in (('ecid', 124), ('handoff_id', 'c'*32), ('template_sha256', 'd'*64)): + self.command_line(handoff={**HANDOFF, key: value}) + with self.subTest(key=key), self.assertRaisesRegex(ValueError, 'differs'): + identity.cached() + self.command_line() + identity.BOOT_ID.write_text('22222222-2222-4222-8222-222222222222') + with self.assertRaisesRegex(ValueError, 'differs'): + identity.cached() + identity.BOOT_ID.write_text(BOOT) + cache = identity.RUN/'identity.json' + cache.write_text(json.dumps({**record, 'ecid': 123.0})) + with self.assertRaisesRegex(ValueError, 'differs'): + identity.cached() + cache.write_text(json.dumps(record).replace('"ecid": 123', '"ecid": 0, "ecid": 123')) + with self.assertRaisesRegex(ValueError, 'repeats'): + identity.cached() + cache.write_text(json.dumps(record)) + identity.CMDLINE.unlink() + with self.assertRaisesRegex(ValueError, 'unavailable'): + identity.cached() + + def test_usb_serial_bound_is_checked_before_publication(self): + policy = {**POLICY, 'profile': 'p'*32} + identity.POLICY.write_text(json.dumps(policy)) + self.command_line(policy=policy, handoff={**HANDOFF, 'profile': policy['profile']}) + with self.assertRaisesRegex(ValueError, 'USB serial string limit'): + identity.initialize() + self.assertFalse((identity.RUN/'identity.json').exists()) + + def test_no_policy_keeps_the_adt_requirement_even_with_handoff_tokens(self): + identity.POLICY.unlink() + identity.COMPATIBLE.write_bytes(b'apple,j71b\0') + identity.MTD.write_text('') + with self.assertRaisesRegex(ValueError, 'one live ADT'): + identity.initialize() + + def test_unknown_or_malformed_command_line_never_publishes(self): + valid = identity.CMDLINE.read_text() + values = (valid+'orchard.recovery-unknown=e30', valid+codec.encode_parameter(codec.HANDOFF_PARAMETER, HANDOFF), + valid.replace('orchard.recovery-handoff=', 'orchard.recovery-handoff=='), + valid+'\0', 'x'*1024) + for value in values: + identity.CMDLINE.write_text(value) + with self.subTest(value=value[:80]), self.assertRaises(ValueError): + identity.initialize() + self.assertFalse((identity.RUN/'identity.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_public_recovery.py b/tests/test_public_recovery.py new file mode 100644 index 00000000..3a484bb0 --- /dev/null +++ b/tests/test_public_recovery.py @@ -0,0 +1,134 @@ +"""The public Cascadia producer and identity handoff operate without a device.""" +import hashlib +import unittest + +from orchard import recovery_identity as protocol +from orchard.recovery_policy import settings +from orchard.image import cascadia +from test_cascadia_loader import PROFILE, loader, kernel, tree + +POLICY = {'schema': 1, 'source': 'host-usb', 'profile': 'p105', + 'compatibles': ['apple,p105ap'], 'chip': 0x8942, 'board': 0x0a} + + +def payload(args='console=tty0 rdinit=/init', *, policy=POLICY, compatible=b'apple,p105ap\0apple,s5l8942x\0'): + root = tree() + root['properties']['compatible'] = compatible + return cascadia.assemble(loader(), kernel(), cascadia.write_fdt(root), b'public-initramfs', + PROFILE, args, identity_policy=policy)[0] + + +def bind(data, **changes): + values = dict(ecid=123, chip=0x8942, board=0x0a, profile='p105', + handoff_id='a'*32, template_sha256=hashlib.sha256(data).hexdigest()) + return cascadia.bind_host_identity(data, **{**values, **changes}) + + +class PublicRecovery(unittest.TestCase): + def test_platform_policy_uses_descriptor_identity_and_manual_storage(self): + selected = settings('p105') + self.assertEqual(selected['identity'], POLICY) + self.assertEqual(selected['runtime']['usb_network'], 'ecm') + self.assertEqual(selected['runtime']['manual_modules'], ['apple_sftl']) + self.assertEqual(selected['runtime']['storage_activation'], 'manual') + self.assertIsNone(settings('j71b')) + + def test_host_binding_keeps_kernel_initrd_and_template(self): + original = payload() + self.assertEqual(cascadia.host_identity_policy(original), POLICY) + bound = bind(original) + before, _, _ = cascadia._payload_parts(original) + after, _, args = cascadia._payload_parts(bound) + self.assertEqual(original[before['kernel_offset']:before['dtb_offset']], + bound[after['kernel_offset']:after['dtb_offset']]) + self.assertEqual(original[before['initrd_offset']:], bound[after['initrd_offset']:]) + fields = protocol.parameters(args) + self.assertEqual(fields[protocol.POLICY_PARAMETER], POLICY) + self.assertEqual(fields[protocol.HANDOFF_PARAMETER]['template_sha256'], hashlib.sha256(original).hexdigest()) + self.assertEqual(fields[protocol.HANDOFF_PARAMETER]['handoff_id'], 'a'*32) + with self.assertRaisesRegex(ValueError, 'already contains'): + cascadia.host_identity_policy(bound) + with self.assertRaisesRegex(ValueError, 'already contains'): + bind(bound) + + def test_selected_board_digest_and_complete_inputs_fail_before_binding(self): + original = payload() + for changes in ({'profile': 'j1'}, {'chip': 1}, {'board': 1}, {'ecid': True}, + {'handoff_id': '0'*32}, {'template_sha256': '0'*64}): + with self.subTest(changes=changes), self.assertRaises(ValueError): + bind(original, **changes) + with self.assertRaisesRegex(ValueError, 'First DTB compatible'): + payload(compatible=b'apple,s5l8942x\0apple,p105ap\0') + self.assertIsNone(cascadia.host_identity_policy(payload(policy=None))) + + def test_arm_command_line_full_bound_and_truncation(self): + example = payload('x') + _, _, args = cascadia._payload_parts(bind(example)) + padding = protocol.MAX_CMDLINE_BYTES - len(args) + self.assertGreater(padding, 0) + maximum = payload('x'*(padding+1)) + _, _, full = cascadia._payload_parts(bind(maximum)) + self.assertEqual(len(full), 1023) + with self.assertRaisesRegex(ValueError, 'command-line bound'): + bind(payload('x'*(padding+2))) + with self.assertRaises(ValueError): + protocol.parameters(full[:-1]) + + def test_corrupted_configuration_and_tree_are_refused(self): + data = payload() + bad = bytearray(data) + bad[cascadia.CONFIG_OFFSET+8] ^= 1 + with self.assertRaises(ValueError): + cascadia.host_identity_policy(bytes(bad)) + values = cascadia.configuration(data) + bad = bytearray(data) + bad[values['dtb_offset']] ^= 1 + with self.assertRaises(ValueError): + cascadia.host_identity_policy(bytes(bad)) + + +class RecoveryModulePolicy(unittest.TestCase): + def test_runtime_skips_only_recorded_builtin_modules(self): + import importlib.util + import sys + import tempfile + from pathlib import Path + from unittest import mock + path = Path(__file__).resolve().parents[1]/'runtime/recovery/policy.py' + specification = importlib.util.spec_from_file_location('recovery_policy_runtime', path) + runtime = importlib.util.module_from_spec(specification) + with mock.patch.dict(sys.modules, {'recovery_identity': protocol}): + specification.loader.exec_module(runtime) + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory/'modules.builtin').write_text('kernel/libcomposite.ko\nkernel/usb-f-ecm.ko\n') + value = {'usb_modules': ['libcomposite', 'usb_f_ecm', 'usb_f_acm']} + self.assertEqual(runtime.loadable_modules(value, directory), ['usb_f_acm']) + (directory/'modules.builtin').unlink() + with self.assertRaises(FileNotFoundError): + runtime.loadable_modules(value, directory) + + def test_builtin_usb_is_accepted_and_builtin_manual_storage_is_refused(self): + import tempfile + from pathlib import Path + from orchard.image.prepare_runtime import check_recovery_modules + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + modules = root/'usr/lib/modules/test' + modules.mkdir(parents=True) + selected = settings('p105') + overlapping = {'runtime': {**selected['runtime'], 'manual_modules': ['usb-f-ecm']}} + with self.assertRaisesRegex(ValueError, 'manual module cannot be loaded'): + check_recovery_modules(root, 'test', overlapping) + builtin = modules/'modules.builtin' + builtin.write_text('kernel/libcomposite.ko\nkernel/usb_f_ecm.ko\nkernel/usb_f_acm.ko\n') + check_recovery_modules(root, 'test', selected) + builtin.write_text(builtin.read_text() + 'kernel/apple-sftl.ko\n') + with self.assertRaisesRegex(ValueError, 'Manual recovery module is built in'): + check_recovery_modules(root, 'test', selected) + builtin.write_text('') + with self.assertRaisesRegex(ValueError, 'USB module is unavailable'): + check_recovery_modules(root, 'test', selected) + for name in selected['runtime']['usb_modules']: + (modules/(name+'.ko')).write_bytes(b'fixture') + check_recovery_modules(root, 'test', selected) From 000f135c3b8d2b68f8a2a19ae20d8b920ffd08ae Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:44:59 -0600 Subject: [PATCH 05/16] kernel: scope shared patch layers by platform ARMv7 and ARM64 use different kernel bases while driver patches belong in shared subsystem directories. An adjacent patch scope lets those directories carry both platforms without applying a patch to the wrong base. Add schema-1 .patch.toml scopes and scoped-only series layers. Export writes scopes and missing layers with --platform, preserving existing patch numbers. Builds, conversion and regeneration use the same selector, and published source keys include selected scope metadata. Refuse malformed scopes and duplicate selection before build or export work starts. Existing patch bytes, ordered platform selections and published source keys are unchanged. Cover scope validation, export planning and deterministic kernel/m1n1 regeneration with host tests. Assisted-by: LLM --- CONTRIBUTING.md | 20 ++ kernel/README.md | 11 + kernel/build | 9 +- kernel/series.py | 92 +++++++- src/orchard/published.py | 3 +- tests/test_patch_scopes.py | 429 +++++++++++++++++++++++++++++++++++ tools/kernel-tree/README.md | 30 ++- tools/kernel-tree/convert.py | 64 ++++-- tools/kernel-tree/export.py | 100 +++++++- tools/kernel-tree/regenerate | 18 +- 10 files changed, 721 insertions(+), 55 deletions(-) create mode 100644 tests/test_patch_scopes.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index de1d4b2b..85ceccf3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,6 +58,20 @@ the follow-ups under "Open work", so you can pick them up from there. Patches are applied in the layer order of `kernel/series/`; within a directory, in file-name order. +Platforms with different upstream bases can share a subsystem directory. +An adjacent `.patch.toml` restricts a patch to named platforms: + +```toml +schema = 1 +platforms = ["apple-armv7"] +``` + +A plain directory entry includes its unscoped patches and those scoped to +that platform. `scoped patches/drivers/usb` includes only patches explicitly +scoped to the platform. Use this form when adding a shared directory to a +series whose upstream base differs from the existing patches. Each patch +must be selected once; repeated and overlapping layers are refused. + - `kernel/patches/drivers//`: changes to drivers and their bindings, including new drivers. Use the subsystem directory that matches the kernel's (`spi`, `input`, `power`, …); add one when none fits. @@ -70,6 +84,12 @@ directory, in file-name order. Number a new patch with the next free number in its directory. Don't renumber existing patches. +After testing the source branch on hardware, use +`tools/kernel-tree/export.py --platform ` to write scoped patches +and add missing scoped layers. Repeat `--platform` for each platform that +shares the generated kernel branch. The export keeps subsystem directories +and assigns the next free patch number. + A driver patch filed under `devices/` hides that it changes behaviour on other devices, which is why the split matters. diff --git a/kernel/README.md b/kernel/README.md index e9a8addc..42a803be 100644 --- a/kernel/README.md +++ b/kernel/README.md @@ -9,6 +9,17 @@ the platform. A device name resolves to its platform through `DIRECTORY/.dtb`. Git and Podman are required. `series.py --platform NAME` prints the checked descriptor of a platform or device. +`series/` lists patch directories in application order. Each +directory contributes patches recursively in filename order. A patch can +have an adjacent `.patch.toml` with `schema = 1` and `platforms = ["name"]` to +restrict where it applies. Plain directory entries select unscoped patches +and scopes matching the platform. A `scoped patches/drivers/` +entry selects only matching scopes, so platforms on different kernel bases +can use the same subsystem layout. The selector rejects unknown platforms, +malformed or orphaned scopes, paths outside the kernel directory and duplicate +patch selection. See [Contributing](../CONTRIBUTING.md#where-kernel-patches-go) +and [kernel-tree](../tools/kernel-tree/README.md) for export and regeneration. + `orchard build` runs it; build with `orchard build --device j71b` (platform `apple-16k` plus the j71b DTB) or `orchard build --platform apple-16k`, not with `kernel/build` directly. It holds the shared tree's lock and switches the diff --git a/kernel/build b/kernel/build index 814bee7c..bf46308a 100755 --- a/kernel/build +++ b/kernel/build @@ -42,9 +42,12 @@ if [[ -n $tree ]]; then fi config="$here/../$(field config)" lock="$here/../$(field upstream)" patches=() -while IFS= read -r patch; do - patches+=("$patch") -done < <("$python" "$here/series.py" "$device") +selected_patches=$("$python" "$here/series.py" "$device") +if [[ -n $selected_patches ]]; then + while IFS= read -r patch; do + patches+=("$patch") + done <<< "$selected_patches" +fi [[ -f "$config" ]] || { echo "Missing kernel config for platform $platform" >&2; exit 1; } command -v git >/dev/null || { echo 'Git is required' >&2; exit 1; } if [[ ${ORCHARD_BUILD_NATIVE:-0} != 1 ]]; then diff --git a/kernel/series.py b/kernel/series.py index 037dfbb0..54a89ada 100755 --- a/kernel/series.py +++ b/kernel/series.py @@ -80,17 +80,91 @@ def platform(name): ('arch', 'image', 'dtbs', 'upstream', 'series', 'config', 'defconfig', 'branch', 'loader')}} -def patches(name): - series = REPO / platform(name)['series'] - result = [] - for line in series.read_text().splitlines(): - line = line.split('#', 1)[0].strip() +def _kernel_path(path, what): + resolved = path.resolve() + if ROOT.resolve() not in resolved.parents: + raise ValueError(f'{what} leaves the kernel directory: {path}') + return resolved + + +def series_layers(name): + """Ordered (directory, scoped-only) entries, with one occurrence per layer.""" + described = platform(name) + series = REPO / described['series'] + result, seen = [], set() + for number, raw in enumerate(series.read_text().splitlines(), 1): + line = raw.split('#', 1)[0].strip() if not line: continue - layer = (ROOT / line).resolve() - if ROOT not in layer.parents or not layer.is_dir(): - raise ValueError(f'Invalid patch layer in {series}: {line}') - result += sorted(layer.rglob('*.patch')) + words = line.split() + scoped = words[0] == 'scoped' + if len(words) != (2 if scoped else 1): + raise ValueError(f'Invalid patch layer in {series}:{number}: {line}') + relative = words[-1] + if Path(relative).is_absolute() or '..' in Path(relative).parts: + raise ValueError(f'Invalid patch layer in {series}:{number}: {line}') + layer = _kernel_path(ROOT / relative, 'Patch layer') + if not layer.is_dir(): + raise ValueError(f'Invalid patch layer in {series}:{number}: {line}') + if layer in seen: + raise ValueError(f'Duplicate patch layer in {series}:{number}: {line}') + seen.add(layer) + result.append((layer, scoped)) + return result + + +def patch_platforms(patch): + """An adjacent PATCH.patch.toml restricts that patch to named platforms.""" + _kernel_path(patch, 'Patch') + if not patch.is_file(): + raise ValueError(f'Missing patch: {patch}') + sidecar = Path(str(patch) + '.toml') + if not sidecar.exists() and not sidecar.is_symlink(): + return None + _kernel_path(sidecar, 'Patch scope') + if not sidecar.is_file() or sidecar.is_symlink(): + raise ValueError(f'Patch scope must be a regular file: {sidecar}') + try: + value = tomllib.loads(sidecar.read_text()) + except (OSError, UnicodeError, tomllib.TOMLDecodeError) as error: + raise ValueError(f'Invalid patch scope {sidecar}: {error}') from error + names = value.get('platforms') + if (set(value) != {'schema', 'platforms'} or type(value.get('schema')) is not int or + value['schema'] != 1 or not isinstance(names, list) or not names or + any(not isinstance(name, str) or not NAME.fullmatch(name) for name in names) or + len(set(names)) != len(names)): + raise ValueError(f'Invalid patch scope {sidecar}: expected schema 1 and distinct platform names') + for name in names: + if not (PLATFORMS / f'{name}.toml').is_file(): + raise ValueError(f'Unknown platform {name} in patch scope {sidecar}') + platform(name) + return frozenset(names) + + +def layer_patches(layer): + """Patches and validated scopes, also used to preflight new export layers.""" + _kernel_path(layer, 'Patch layer') + for sidecar in sorted(layer.rglob('*.patch.toml')): + _kernel_path(sidecar, 'Patch scope') + patch = Path(str(sidecar).removesuffix('.toml')) + if not patch.is_file(): + raise ValueError(f'Orphan patch scope: {sidecar}') + return [(patch, patch_platforms(patch)) for patch in sorted(layer.rglob('*.patch'))] + + +def patches(name): + """Select each patch once, preserving layer order and filename order.""" + selected = platform(name)['platform'] + result, seen = [], set() + for layer, scoped in series_layers(name): + for patch, scope in layer_patches(layer): + if (scope is None and scoped) or (scope is not None and selected not in scope): + continue + identity = patch.resolve() + if identity in seen: + raise ValueError(f'Patch selected by multiple layers: {patch}') + seen.add(identity) + result.append(patch) return result diff --git a/src/orchard/published.py b/src/orchard/published.py index f383d789..ac44d7b0 100644 --- a/src/orchard/published.py +++ b/src/orchard/published.py @@ -41,11 +41,12 @@ def series(*args): return subprocess.check_output([sys.executable, str(hardware / 'kernel/series.py'), *args], text=True) described = json.loads(series('--platform', platform)) patches = [str(Path(line).resolve().relative_to(hardware.resolve())) for line in series(platform).splitlines()] + scopes = [path + '.toml' for path in patches if (hardware / (path + '.toml')).is_file()] return { 'platform': platform, 'build': {key: described[key] for key in ('arch', 'image', 'dtbs')}, 'recipes': recipe_entries(hardware, [described['upstream'], described['series'], described['config'], - 'kernel/Containerfile', 'kernel/install-toolchain', *patches]), + 'kernel/Containerfile', 'kernel/install-toolchain', *patches, *scopes]), 'producer': KERNEL_PRODUCER, } diff --git a/tests/test_patch_scopes.py b/tests/test_patch_scopes.py new file mode 100644 index 00000000..5e6fd6ac --- /dev/null +++ b/tests/test_patch_scopes.py @@ -0,0 +1,429 @@ +"""Shared subsystem directories can carry patches for different kernel bases.""" +import json +import os +from pathlib import Path +import runpy +import subprocess +import sys +import tempfile +import unittest + +from orchard import published + +ROOT = Path(__file__).resolve().parents[1] +EXPORT = ROOT / 'tools/kernel-tree/export.py' +CONVERT = ROOT / 'tools/kernel-tree/convert.py' +REGENERATE = ROOT / 'tools/kernel-tree/regenerate' + + +def write(root, path, text): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + return target + + +def git(root, *args, input=None): + return subprocess.run(['git', '-C', str(root), '-c', 'user.name=Test Author', + '-c', 'user.email=test@example.invalid', *args], input=input, + check=True, capture_output=True).stdout.decode().strip() + + +def commit(root, message): + git(root, 'add', '.') + git(root, 'commit', '-qm', message, '--no-gpg-sign') + return git(root, 'rev-parse', 'HEAD') + + +def layout(root): + """Minimal hardware tree with separate ARMv7 and shared ARM64 branches.""" + root = root.resolve() + write(root, 'kernel/series.py', (ROOT / 'kernel/series.py').read_text()) + write(root, 'kernel/upstream.lock', 'url=https://example.invalid/linux\ncommit=unused\n') + write(root, 'kernel/sources.toml', '') + write(root, 'kernel/Containerfile', 'FROM scratch\n') + write(root, 'kernel/install-toolchain', '# toolchain\n') + (root / 'kernel/patches/drivers/usb').mkdir(parents=True) + for name, arch, branch in (('armv7', 'arm', 'armv7'), + ('arm64-4k', 'arm64', 'main'), + ('arm64-16k', 'arm64', 'main')): + write(root, f'platforms/{name}.toml', f'''schema = 1 +kind = "hardware.platform" +name = "{name}" +[kernel] +arch = "{arch}" +image = "{'zImage' if arch == 'arm' else 'Image.gz'}" +dtbs = "arch/{arch}/boot/dts/apple" +upstream = "kernel/upstream.lock" +series = "kernel/series/{name}" +config = "kernel/configs/{name}/config" +defconfig = "orchard_{name.replace('-', '')}_defconfig" +branch = "{branch}" +loader = "{'cascadia' if arch == 'arm' else 'm1n1'}" +''') + write(root, f'kernel/configs/{name}/config', 'CONFIG_TEST=y\n') + write(root, f'kernel/series/{name}', + '' if arch == 'arm' else 'patches/drivers\n') + return root + + +class ScopedPatches(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = layout(Path(self.temporary.name) / 'orchard') + self.api = runpy.run_path(str(self.root / 'kernel/series.py')) + + def patch(self, path, platforms=None): + patch = write(self.root, f'kernel/patches/{path}', '# synthetic patch\n') + if platforms is not None: + write(self.root, f'kernel/patches/{path}.toml', + 'schema = 1\nplatforms = ' + json.dumps(platforms) + '\n') + return patch + + def selected(self, name): + return [str(path.relative_to(self.root / 'kernel/patches')) + for path in self.api['patches'](name)] + + def test_normal_and_scoped_layers_preserve_layer_and_filename_order(self): + self.patch('drivers/usb/0001-common.patch') + self.patch('drivers/usb/0003-arm64.patch', ['arm64-4k', 'arm64-16k']) + self.patch('drivers/usb/0002-armv7.patch', ['armv7']) + self.patch('devices/a5/0001-board.patch', ['armv7']) + write(self.root, 'kernel/series/armv7', + 'scoped patches/devices/a5 # board first\nscoped patches/drivers/usb\n') + self.assertEqual(self.selected('armv7'), + ['devices/a5/0001-board.patch', 'drivers/usb/0002-armv7.patch']) + self.assertEqual(self.selected('arm64-4k'), + ['drivers/usb/0001-common.patch', 'drivers/usb/0003-arm64.patch']) + write(self.root, 'kernel/series/armv7', 'patches/drivers/usb\n') + self.assertEqual(self.selected('armv7'), + ['drivers/usb/0001-common.patch', 'drivers/usb/0002-armv7.patch']) + + def test_armv7_patch_leaves_arm64_selection_and_published_keys_unchanged(self): + self.patch('drivers/usb/0001-common.patch') + selected = self.selected('arm64-4k') + keys = {name: published.source_key(self.root, name) + for name in ('arm64-4k', 'arm64-16k')} + self.patch('drivers/usb/0002-armv7.patch', ['armv7']) + write(self.root, 'kernel/series/armv7', 'scoped patches/drivers/usb\n') + self.assertEqual(self.selected('arm64-4k'), selected) + for name, key in keys.items(): + self.assertEqual(published.source_key(self.root, name), key) + + def test_selected_scope_is_in_published_inputs(self): + patch = self.patch('drivers/usb/0001-armv7.patch', ['armv7']) + write(self.root, 'kernel/series/armv7', 'scoped patches/drivers/usb\n') + before = published.source_key(self.root, 'armv7') + with Path(str(patch) + '.toml').open('a') as sidecar: + sidecar.write('# provenance changed\n') + self.assertNotEqual(published.source_key(self.root, 'armv7'), before) + + def test_invalid_scopes_fail_instead_of_silently_skipping(self): + patch = self.patch('drivers/usb/0001.patch') + invalid = ('schema = 2\nplatforms = ["armv7"]\n', + 'schema = true\nplatforms = ["armv7"]\n', + 'schema = 1\nplatforms = []\n', + 'schema = 1\nplatforms = ["armv7", "armv7"]\n', + 'schema = 1\nplatforms = ["unknown"]\n', + 'schema = 1\nplatforms = ["../armv7"]\n', + 'schema = 1\nplatforms = [3]\n', + 'schema = 1\nplatforms = ["armv7"]\nextra = 1\n', + 'schema = 1\nplatforms = "armv7"\n', + 'invalid toml = [\n') + for text in invalid: + with self.subTest(text=text): + Path(str(patch) + '.toml').write_text(text) + with self.assertRaises(ValueError): + self.selected('arm64-4k') + + def test_orphan_scopes_and_symlinks_are_refused(self): + sidecar = write(self.root, 'kernel/patches/drivers/usb/0001.patch.toml', + 'schema = 1\nplatforms = ["armv7"]\n') + with self.assertRaisesRegex(ValueError, 'Orphan'): + self.selected('arm64-4k') + sidecar.unlink() + patch = self.patch('drivers/usb/0001.patch') + outside = write(Path(self.temporary.name), 'outside.toml', + 'schema = 1\nplatforms = ["armv7"]\n') + sidecar.symlink_to(outside) + with self.assertRaisesRegex(ValueError, 'leaves the kernel'): + self.selected('arm64-4k') + sidecar.unlink() + patch.unlink() + patch.symlink_to(outside) + with self.assertRaisesRegex(ValueError, 'leaves the kernel'): + self.selected('arm64-4k') + + def test_layer_containment_and_duplicate_selection(self): + self.patch('drivers/usb/0001.patch') + for text, message in (('patches/drivers\npatches/drivers\n', 'Duplicate patch layer'), + ('patches/drivers\nscoped patches/drivers\n', 'Duplicate patch layer'), + ('patches/drivers\npatches/drivers/usb\n', 'multiple layers'), + ('../outside\n', 'Invalid patch layer'), + ('/tmp\n', 'Invalid patch layer'), + ('scoped patches/drivers extra\n', 'Invalid patch layer')): + with self.subTest(text=text): + write(self.root, 'kernel/series/armv7', text) + with self.assertRaisesRegex(ValueError, message): + self.selected('armv7') + (self.root / 'kernel/patches/escape').symlink_to(Path(self.temporary.name)) + write(self.root, 'kernel/series/armv7', 'patches/escape\n') + with self.assertRaisesRegex(ValueError, 'leaves the kernel'): + self.selected('armv7') + + def test_producer_stops_on_invalid_selection_before_starting_build_tools(self): + self.patch('drivers/usb/0001.patch', ['unknown']) + write(self.root, 'kernel/series/armv7', 'scoped patches/drivers/usb\n') + build = write(self.root, 'kernel/build', (ROOT / 'kernel/build').read_text()) + marker = self.root.parent / 'podman-called' + fake = write(self.root.parent, 'bin/podman', f'#!/bin/sh\ntouch "{marker}"\nexit 1\n') + fake.chmod(0o755) + output, work = self.root.parent / 'output', self.root.parent / 'work' + result = subprocess.run(['bash', str(build), 'armv7', '--output', str(output), + '--work-dir', str(work)], capture_output=True, text=True, + env={**os.environ, 'ORCHARD_PYTHON': sys.executable, + 'PATH': str(fake.parent) + os.pathsep + os.environ['PATH']}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Unknown platform', result.stderr) + self.assertFalse(marker.exists()) + self.assertFalse(output.exists()) + self.assertFalse(work.exists()) + + +class ScopedExport(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = layout(Path(self.temporary.name) / 'orchard') + self.linux = Path(self.temporary.name) / 'linux' + git(Path(self.temporary.name), 'init', '-q', '-b', 'main', str(self.linux)) + write(self.linux, 'drivers/usb/test.c', 'int old;\n') + self.base = commit(self.linux, 'base') + git(self.linux, 'switch', '-qc', 'work') + write(self.linux, 'drivers/usb/test.c', 'int new;\n') + self.change = commit(self.linux, 'usb: test: Update value') + + def export(self, *args): + return subprocess.run([sys.executable, str(EXPORT), '--orchard', str(self.root), + '--linux', str(self.linux), *args, 'work'], + capture_output=True, text=True) + + def test_export_writes_scope_and_missing_layer_without_renumbering(self): + old = write(self.root, 'kernel/patches/drivers/usb/0042-existing.patch', '# preserved\n') + result = self.export('--platform', 'armv7') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(old.read_text(), '# preserved\n') + new = next((self.root / 'kernel/patches/drivers/usb').glob('0043-*.patch')) + self.assertEqual(Path(str(new) + '.toml').read_text(), + 'schema = 1\nplatforms = ["armv7"]\n') + self.assertEqual((self.root / 'kernel/series/armv7').read_text(), + 'scoped patches/drivers/usb\n') + api = runpy.run_path(str(self.root / 'kernel/series.py')) + self.assertEqual(api['patches']('armv7'), [new]) + self.assertEqual(api['patches']('arm64-4k'), [old]) + + def test_existing_layer_is_reused_and_shared_platform_order_is_deterministic(self): + result = self.export('--platform', 'arm64-4k', '--platform', 'arm64-16k', + '--platform', 'arm64-4k') + self.assertEqual(result.returncode, 0, result.stderr) + new = next((self.root / 'kernel/patches/drivers/usb').glob('*.patch')) + self.assertEqual(Path(str(new) + '.toml').read_text(), + 'schema = 1\nplatforms = ["arm64-16k", "arm64-4k"]\n') + self.assertEqual((self.root / 'kernel/series/arm64-4k').read_text(), 'patches/drivers\n') + self.assertEqual((self.root / 'kernel/series/arm64-16k').read_text(), 'patches/drivers\n') + + def test_shared_series_receives_each_new_layer_once(self): + descriptor = self.root / 'platforms/arm64-4k.toml' + descriptor.write_text(descriptor.read_text().replace('series/arm64-4k', 'series/arm64-16k')) + write(self.root, 'kernel/series/arm64-16k', '') + result = self.export('--platform', 'arm64-4k', '--platform', 'arm64-16k') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.root / 'kernel/series/arm64-16k').read_text(), + 'scoped patches/drivers/usb\n') + + def test_planned_layer_overlaps_fail_before_writes(self): + write(self.root, 'kernel/series/armv7', 'scoped patches/drivers/usb\n') + existing = write(self.root, 'kernel/patches/drivers/usb/0001-existing.patch', '# preserved\n') + sidecar = write(self.root, 'kernel/patches/drivers/usb/0001-existing.patch.toml', + 'schema = 1\nplatforms = ["armv7"]\n') + result = self.export('--platform', 'armv7', '--topic', f'{self.change}=drivers') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Planned layers', result.stderr) + self.assertEqual(list((self.root / 'kernel/patches').rglob('*.patch')), [existing]) + self.assertEqual((self.root / 'kernel/series/armv7').read_text(), + 'scoped patches/drivers/usb\n') + existing.unlink() + sidecar.unlink() + write(self.root, 'kernel/series/armv7', '') + write(self.linux, 'drivers/usb/test.c', 'int newer;\n') + second = commit(self.linux, 'usb: test: Update again') + result = self.export('--platform', 'armv7', '--topic', f'{self.change}=drivers', + '--topic', f'{second}=drivers/usb') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Planned layers', result.stderr) + self.assertEqual(list((self.root / 'kernel/patches').rglob('*.patch')), []) + + def test_export_checks_orphans_in_a_new_layer_before_writing(self): + write(self.root, 'kernel/patches/drivers/mtd/orphan.patch.toml', 'invalid = true\n') + result = self.export('--platform', 'armv7', '--topic', f'{self.change}=drivers/mtd') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Orphan patch scope', result.stderr) + self.assertEqual(list((self.root / 'kernel/patches').rglob('*.patch')), []) + self.assertEqual((self.root / 'kernel/series/armv7').read_text(), '') + + def test_dry_run_reserves_nothing_and_export_is_reproducible(self): + before = {str(p.relative_to(self.root)): p.read_bytes() + for p in self.root.rglob('*') if p.is_file()} + for _ in range(2): + result = self.export('--platform', 'armv7', '--dry-run') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('drivers/usb/0001', result.stdout) + self.assertEqual({str(p.relative_to(self.root)): p.read_bytes() + for p in self.root.rglob('*') if p.is_file()}, before) + result = self.export('--platform', 'armv7') + self.assertEqual(result.returncode, 0, result.stderr) + first = next((self.root / 'kernel/patches/drivers/usb').glob('*.patch')) + expected = first.read_bytes() + first.unlink() + Path(str(first) + '.toml').unlink() + result = self.export('--platform', 'armv7') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(first.read_bytes(), expected) + self.assertEqual((self.root / 'kernel/series/armv7').read_text(), + 'scoped patches/drivers/usb\n') + + def test_unknown_incomplete_and_escaping_scopes_fail_before_writes(self): + for args, error in ((('--platform', 'unknown'), 'Unknown platform'), + (('--platform', 'arm64-4k'), 'shares kernel branch'), + (('--platform', 'armv7', '--topic', f'{self.change}=../escape'), + 'Invalid patch topic')): + with self.subTest(args=args): + result = self.export(*args) + self.assertNotEqual(result.returncode, 0) + self.assertIn(error, result.stderr) + self.assertEqual(list((self.root / 'kernel/patches').rglob('*.patch')), []) + + def test_a5_dts_topic_is_inferred(self): + git(self.linux, 'reset', '--hard', self.base) + write(self.linux, 'arch/arm/boot/dts/apple/s5l8942x-p105.dts', '/dts-v1/;\n') + commit(self.linux, 'ARM: dts: apple: Add P105') + result = self.export('--platform', 'armv7', '--dry-run') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('devices/s5l8942x/0001', result.stdout) + + def test_converter_uses_the_committed_selector_and_is_deterministic(self): + # A foreign patch follows the exported ARMv7 patch in the same subsystem. + result = self.export('--platform', 'armv7') + self.assertEqual(result.returncode, 0, result.stderr) + armv7 = next((self.root / 'kernel/patches/drivers/usb').glob('*.patch')) + git(self.linux, 'switch', '-qc', 'foreign', self.base) + write(self.linux, 'drivers/usb/test.c', 'int arm64;\n') + commit(self.linux, 'usb: test: Update ARM64 value') + foreign = write(self.root, 'kernel/patches/drivers/usb/0002-arm64.patch', + git(self.linux, 'format-patch', '--stdout', '--no-signature', '-1') + '\n') + write(self.root, str(foreign.relative_to(self.root)) + '.toml', + 'schema = 1\nplatforms = ["arm64-4k", "arm64-16k"]\n') + git(self.root.parent, 'init', '-q', '-b', 'main', str(self.root)) + revision = commit(self.root, 'Add scoped patches') + # Dirty files must have no effect on regeneration from a commit. + write(self.root, 'kernel/series/armv7', 'patches/drivers\n') + for platform, patch, expected in (('armv7', armv7, 'int new;\n'), + ('arm64-4k', foreign, 'int arm64;\n')): + converted = [] + for _ in range(2): + mapping = self.root.parent / f'{platform}.map' + result = subprocess.run([sys.executable, str(CONVERT), '--orchard', str(self.root), + '--commit', revision, '--layers', 'kernel', + '--series', f'series/{platform}', '--platform', platform, + '--target', str(self.linux), '--base', self.base, + '--map', str(mapping)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + tip = git(self.linux, 'rev-parse', 'refs/orchard/converted') + converted.append(tip) + self.assertEqual(git(self.linux, 'show', f'{tip}:drivers/usb/test.c') + '\n', expected) + self.assertEqual(mapping.read_text().split('\t')[0], str(patch.relative_to(self.root))) + self.assertEqual(converted[0], converted[1]) + + def test_scope_removal_retains_its_conversion_provenance(self): + result = self.export('--platform', 'armv7') + self.assertEqual(result.returncode, 0, result.stderr) + write(self.root, 'kernel/series/armv7', 'patches/drivers/usb\n') + git(self.root.parent, 'init', '-q', '-b', 'main', str(self.root)) + commit(self.root, 'Add scoped patch') + patch = next((self.root / 'kernel/patches/drivers/usb').glob('*.patch')) + Path(str(patch) + '.toml').unlink() + git(self.root, '-c', 'user.name=Scope Remover', 'commit', '-qam', 'Remove scope', '--no-gpg-sign') + mapping = self.root.parent / 'mapping' + result = subprocess.run([sys.executable, str(CONVERT), '--orchard', str(self.root), + '--commit', 'HEAD', '--layers', 'kernel', '--series', 'series/armv7', + '--target', str(self.linux), '--base', self.base, '--map', str(mapping)], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + converted = mapping.read_text().split('\t')[1].strip() + self.assertEqual(git(self.linux, 'log', '-1', '--format=%cn', converted), 'Scope Remover') + + def test_regeneration_is_repeatable_and_rejects_divergent_shared_branches(self): + result = self.export('--platform', 'armv7') + self.assertEqual(result.returncode, 0, result.stderr) + write(self.root, 'kernel/upstream.lock', f'url=unused\ncommit={self.base}\n') + write(self.root, 'devices/.keep', '') + for script in ('convert.py', 'regenerate'): + write(self.root, f'tools/kernel-tree/{script}', (ROOT / f'tools/kernel-tree/{script}').read_text()) + # The toolchain leaf is replaced; regeneration still archives sources, + # converts patches, writes configs and creates the generated branches. + fake = write(self.root.parent, 'bin/podman', f'''#!{sys.executable} +import pathlib +import sys +if sys.argv[1:3] == ['image', 'exists']: + sys.exit(0) +sys.stdin.buffer.read() +out = pathlib.Path(sys.argv[sys.argv.index('-v') + 1].split(':')[0]) +for line in (out / 'platforms').read_text().splitlines(): + name, arch, defconfig = line.split() + (out / name / 'defconfig').write_bytes((out / name / 'config').read_bytes()) +''') + fake.chmod(0o755) + m1n1 = self.root.parent / 'm1n1' + git(self.root.parent, 'init', '-q', '-b', 'main', str(m1n1)) + write(m1n1, 'README', 'base\n') + m1n1_base = commit(m1n1, 'm1n1 base') + write(m1n1, 'README', 'patched\n') + commit(m1n1, 'm1n1: Update README') + write(self.root, 'boot/m1n1/patches/orchard/0001-readme.patch', + git(m1n1, 'format-patch', '--stdout', '--no-signature', '-1') + '\n') + git(m1n1, 'reset', '--hard', m1n1_base) + write(self.root, 'boot/m1n1/upstream.lock', f'url=unused\ncommit={m1n1_base}\n') + write(self.root, 'boot/m1n1/series', 'patches/orchard\n') + write(self.root, 'boot/m1n1/sources.toml', '') + git(self.root.parent, 'init', '-q', '-b', 'main', str(self.root)) + revision = commit(self.root, 'Add scoped regeneration fixture') + env = {**os.environ, 'PATH': str(fake.parent) + os.pathsep + os.environ['PATH']} + + def regenerate(revision): + return subprocess.run([sys.executable, str(self.root / 'tools/kernel-tree/regenerate'), + '--linux', str(self.linux), '--m1n1', str(m1n1), revision], + env=env, capture_output=True, text=True) + + tips = [] + for _ in range(2): + result = regenerate(revision) + self.assertEqual(result.returncode, 0, result.stderr) + tips.append(tuple(git(self.linux, 'rev-parse', branch) for branch in ('armv7', 'main')) + + (git(m1n1, 'rev-parse', 'main'),)) + self.assertEqual(git(self.linux, 'show', 'armv7:drivers/usb/test.c'), 'int new;') + self.assertEqual(git(self.linux, 'show', 'main:drivers/usb/test.c'), 'int old;') + self.assertEqual(git(m1n1, 'show', 'main:README'), 'patched') + self.assertEqual(tips[0], tips[1]) + new = next((self.root / 'kernel/patches/drivers/usb').glob('*.patch')) + Path(str(new) + '.toml').write_text('schema = 1\nplatforms = ["armv7", "arm64-4k"]\n') + bad = commit(self.root, 'Make shared branch scopes inconsistent') + result = regenerate(bad) + self.assertNotEqual(result.returncode, 0) + self.assertIn('select different patches', result.stderr) + self.assertEqual(tuple(git(self.linux, 'rev-parse', branch) for branch in ('armv7', 'main')) + + (git(m1n1, 'rev-parse', 'main'),), tips[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/kernel-tree/README.md b/tools/kernel-tree/README.md index b7c4fb18..2a35a194 100644 --- a/tools/kernel-tree/README.md +++ b/tools/kernel-tree/README.md @@ -23,6 +23,27 @@ files reproduce the branch. The topic comes from the files a commit touches; if they point to several topics it asks for `--topic SHA=TOPIC`. Changes to an Orchard defconfig belong in `kernel/configs//config` instead. +When a topic applies to one platform's kernel base, pass `--platform` after +testing the branch on hardware: + +```sh +~/orchard/tools/kernel-tree/export.py --orchard ~/orchard --linux . \ + --base armv7 --platform apple-armv7 my-work +``` + +Each exported patch receives an adjacent `.patch.toml` with schema 1 and the +selected platform names. Export adds `scoped patches/` to the selected +series when no existing layer covers that directory. A scoped layer selects +only explicitly matching patches, which lets ARMv7 share subsystem directories +with ARM64. Ordinary layers keep including unscoped patches and skip patches +scoped to another platform. Existing patch files keep their names and bytes. +`--dry-run` prints the plan without writing files or reserving numbers. + +Repeat `--platform` for every platform on the same generated branch, such as +`--platform apple-4k --platform apple-16k`. Regeneration requires those +platforms to select the same ordered patches. Without `--platform`, export +keeps writing unscoped patches and leaves the series files alone. + On macOS the kernel tree needs a case-sensitive volume, and the kernel builds in a Linux container. @@ -39,7 +60,8 @@ per patch and one merge per patch directory, adds the platforms' defconfigs (checked to reproduce `kernel/configs`), and moves the branch; m1n1 `main` likewise. It pushes nothing. -Regeneration is stable: a commit keeps the committer and date of the orchard -commit that last changed its patch file, and a merge cites the orchard commit -that last changed its directory, so topics before the first changed one keep -their SHAs. Branches based on an older `main` rebase cleanly onto a new one. +Regeneration uses the committed platform selector, including patch scopes. +A commit keeps the committer and date of the orchard commit that last changed +its patch file or adjacent scope, and a merge cites the latest change to its +selected patch inputs. Topics before the first changed one keep their SHAs. +Branches based on an older `main` rebase cleanly onto a new one. diff --git a/tools/kernel-tree/convert.py b/tools/kernel-tree/convert.py index eb821358..261eb6a7 100755 --- a/tools/kernel-tree/convert.py +++ b/tools/kernel-tree/convert.py @@ -22,6 +22,7 @@ import argparse import atexit import os +import runpy import shutil import subprocess import sys @@ -36,6 +37,7 @@ parser.add_argument('--target', required=True, type=Path, help='linux or m1n1 repository') parser.add_argument('--base', required=True, help='upstream base in the target repository') parser.add_argument('--series', help='series file relative to --layers, e.g. series/apple-16k') +parser.add_argument('--platform', help='kernel platform for patch selection (default: series file name)') parser.add_argument('--refs', help='topic refs go to refs/topics//') parser.add_argument('--map', type=Path, help='write "patch pathcommit" lines here') args = parser.parse_args() @@ -49,12 +51,13 @@ def run(*cmd, env=None, input=None, cwd=None): commit = run('git', '-C', args.orchard, 'rev-parse', '--short', args.commit) -work = Path(tempfile.mkdtemp(prefix='kernel-tree-')) +work = Path(tempfile.mkdtemp(prefix='kernel-tree-')).resolve() atexit.register(shutil.rmtree, work, ignore_errors=True) index = work / 'index' source = work / 'src' source.mkdir() -archive = subprocess.run(['git', '-C', args.orchard, 'archive', commit, args.layers], +archive_paths = [args.layers, 'platforms'] if args.layers == 'kernel' else [args.layers] +archive = subprocess.run(['git', '-C', args.orchard, 'archive', commit, *archive_paths], capture_output=True, check=True).stdout run('tar', '-x', '-C', source, input=archive) layers = source / args.layers @@ -69,30 +72,45 @@ def orchard_log(fmt, *rest): return run('git', '-C', args.orchard, 'log', f'--format={fmt}', commit, *rest) -def last_change(path): - """Committer of the orchard commit that last changed path.""" - name, email, date = orchard_log('%cn\t%ce\t%cD', '-1', '--', path).split('\t') +def last_change(*paths): + """Committer of the latest change to the patch or its selection metadata.""" + name, email, date = orchard_log('%cn\t%ce\t%cD', '-1', '--', *paths).split('\t') return {'GIT_COMMITTER_NAME': name, 'GIT_COMMITTER_EMAIL': email, 'GIT_COMMITTER_DATE': date} -# Patch order exactly as the build applies it: series layers in order; the -# kernel's series.py takes every patch below a layer (rglob), m1n1's build -# only the layer's own files. -if args.series: - series_file, find = layers / args.series, 'rglob' -elif (layers / 'series').is_file(): - series_file, find = layers / 'series', 'glob' +# Kernel builds and regeneration use the selector from the same snapshot. +# m1n1 retains its existing series contract. +if args.layers == 'kernel': + if not args.series: + sys.exit('kernel has one series per platform; name one with --series') + series_api = runpy.run_path(str(layers / 'series.py')) + selected_platform = args.platform or Path(args.series).name + described = series_api['platform'](selected_platform) + if (source / described['series']).resolve() != (layers / args.series).resolve(): + sys.exit(f'{args.series} is not {selected_platform}\'s patch series') + patches = series_api['patches'](selected_platform) else: - sys.exit(f'{args.layers} has one series per platform; name one with --series') -if not series_file.is_file(): - sys.exit(f'No series {args.series} in {args.layers} at {commit}') -series = series_file.read_text() + if args.platform: + sys.exit('--platform applies only to kernel patch layers') + series_file = layers / (args.series or 'series') + if not series_file.is_file(): + sys.exit(f'No series {args.series} in {args.layers} at {commit}') + patches = [] + find = 'rglob' if args.series else 'glob' + for line in series_file.read_text().splitlines(): + line = line.split('#', 1)[0].strip() + if line: + patches += sorted(getattr(layers / line, find)('*.patch')) refs = args.refs or Path(args.layers).name -patches = [] -for line in series.splitlines(): - line = line.split('#', 1)[0].strip() - if line: - patches += sorted(getattr(layers / line, find)('*.patch')) + + +def patch_inputs(path): + inputs = [str(path.relative_to(source))] + # Include the potential path even when absent so removing a scope keeps + # the deletion's provenance. Git ignores paths with no history. + if args.layers == 'kernel': + inputs.append(str(path.relative_to(source)) + '.toml') + return inputs topics = [] for p in patches: @@ -134,7 +152,7 @@ def source_note(topic): message = run('git', 'stripspace', input=(fields['Subject'] + ('\n\n' + body if body.strip() else '\n')).encode()) git('apply', '--cached', str(p)) - committer = last_change(rel) + committer = last_change(*patch_inputs(p)) tip = git('commit-tree', git('write-tree'), '-p', tip, input=(message + '\n').encode(), env={ 'GIT_AUTHOR_NAME': fields['Author'], 'GIT_AUTHOR_EMAIL': fields['Email'], 'GIT_AUTHOR_DATE': fields['Date'], **committer}) @@ -142,7 +160,7 @@ def source_note(topic): git('update-ref', f'refs/topics/{refs}/{topic}', tip) # The merge is authored and committed as the topic's most recent change. newest = orchard_log('%h\t%an\t%ae\t%cn\t%ce\t%cD', '-1', '--', - *(str(p.relative_to(source)) for p in files)) + *(path for p in files for path in patch_inputs(p))) changed, an, ae, cn, ce, date = newest.split('\t') count = len(files) message = (f'Merge {topic} ({count} patch{"es" if count != 1 else ""})\n\n' diff --git a/tools/kernel-tree/export.py b/tools/kernel-tree/export.py index 9017e684..371721e0 100755 --- a/tools/kernel-tree/export.py +++ b/tools/kernel-tree/export.py @@ -18,12 +18,13 @@ a commit is exported as usual. usage: export.py --orchard DIR --linux DIR [--base REV] [--topic SHA=TOPIC ...] - [--dry-run] BRANCH + [--platform NAME ...] [--dry-run] BRANCH """ import argparse import json import os import re +import runpy import subprocess import sys import tempfile @@ -35,11 +36,15 @@ parser.add_argument('--base', default='main', help='where BRANCH started (default: main)') parser.add_argument('--topic', action='append', default=[], metavar='SHA=TOPIC', help='topic for one commit, e.g. 1a2b3c=drivers/input') +parser.add_argument('--platform', action='append', default=[], metavar='NAME', + help='scope exported patches to this platform; repeat for shared branches') parser.add_argument('--dry-run', action='store_true', help='show the plan, write nothing') parser.add_argument('branch') args = parser.parse_args() args.orchard, args.linux = args.orchard.resolve(), args.linux.resolve() patches = args.orchard / "kernel/patches" +series_api = runpy.run_path(str(args.orchard / 'kernel/series.py')) +selected_platforms = sorted(set(args.platform)) def git(*cmd, input=None): @@ -61,7 +66,7 @@ def git(*cmd, input=None): def topic_of(path): - m = re.match(r'arch/arm(64)?/boot/dts/apple/([a-z]+[0-9]+[a-z]?)[-.]', path) + m = re.match(r'arch/arm(64)?/boot/dts/apple/([a-z][a-z0-9]*)[-.]', path) if m: return f'devices/{m[2]}' for prefix, topic in sorted(DRIVERS.items(), key=lambda kv: -len(kv[0])): @@ -89,12 +94,34 @@ def topic_of(path): DEFCONFIG = re.compile(r'arch/(arm|arm64)/configs/orchard_[a-z0-9_]+_defconfig$') # defconfig path -> (platform, arch, defconfig name, orchard config path), from orchard's descriptors. -platforms = {} +platforms, descriptions = {}, {} for descriptor in sorted((args.orchard / 'platforms').glob('*.toml')): - described = json.loads(subprocess.run([sys.executable, str(args.orchard / 'kernel/series.py'), '--platform', - descriptor.stem], capture_output=True, check=True, text=True).stdout) + described = series_api['platform'](descriptor.stem) + descriptions[descriptor.stem] = described platforms[f'arch/{described["arch"]}/configs/{described["defconfig"]}'] = ( descriptor.stem, described['arch'], described['defconfig'], described['config']) +for name in selected_platforms: + if name not in descriptions: + sys.exit(f'Unknown platform {name}; choose from {", ".join(descriptions)}') + shared = {p for p, value in descriptions.items() + if value['branch'] == descriptions[name]['branch']} + missing = shared - set(selected_platforms) + if missing: + sys.exit(f'{name} shares kernel branch {descriptions[name]["branch"]} with ' + f'{", ".join(sorted(missing))}; include those with --platform too') +# Refuse malformed existing selection metadata before creating any files. +for name in descriptions: + series_api['patches'](name) + + +def topic_directory(topic): + if (not isinstance(topic, str) or not topic or Path(topic).is_absolute() or + any(not re.fullmatch(r'[a-z0-9][a-z0-9_-]*', part) for part in topic.split('/'))): + sys.exit(f'Invalid patch topic: {topic}') + directory = (patches / topic).resolve() + if patches.resolve() not in directory.parents: + sys.exit(f'Patch topic leaves kernel/patches: {topic}') + return directory plan, problems, configs = [], [], set() for sha in commits: @@ -104,6 +131,8 @@ def topic_of(path): for path in changed: if path not in platforms: problems.append(f'{sha[:12]} {subject}: {path} is no platform\'s defconfig') + if selected_platforms and path in platforms and platforms[path][0] not in selected_platforms: + problems.append(f'{sha[:12]} {subject}: {path} is outside the selected platforms') configs.add(path) files = [f for f in files if f not in changed] if not files: @@ -119,6 +148,7 @@ def topic_of(path): f'{", ".join(sorted(candidates)) or "no topic"}; name one with ' f'--topic {sha[:12]}=') continue + topic_directory(topic) plan.append((sha, subject, topic)) if problems: sys.exit('Not exported:\n ' + '\n '.join(problems)) @@ -126,7 +156,7 @@ def topic_of(path): numbers = {} writes = [] for sha, subject, topic in plan: - directory = patches / topic + directory = topic_directory(topic) if topic not in numbers: existing = [int(p.name[:4]) for p in directory.glob('[0-9][0-9][0-9][0-9]-*.patch')] numbers[topic] = max(existing, default=0) @@ -136,16 +166,70 @@ def topic_of(path): print(f'{topic}/{numbers[topic]:04d} {subject}{new}') for path in sorted(configs): print(f'{platforms[path][3]} from {path} on {args.branch}') +layer_additions = {} +for name in selected_platforms: + layers = series_api['series_layers'](name) + missing = [] + for _, _, topic, _ in writes: + directory = topic_directory(topic) + if any(layer == directory or layer in directory.parents for layer, _ in layers): + continue + relative = directory.relative_to(args.orchard / 'kernel').as_posix() + entry = f'scoped {relative}' + if entry not in missing: + missing.append(entry) + layer_additions[name] = missing + planned_layers = layers + [(args.orchard / 'kernel' / entry.split()[1], True) + for entry in missing] + # A new parent layer can overlap a pre-existing child layer. Check both + # existing selected patches and future exports before creating any files. + seen = set() + for layer, scoped in planned_layers: + for patch, scope in series_api['layer_patches'](layer): + if (scope is None and scoped) or (scope is not None and name not in scope): + continue + if patch.resolve() in seen: + sys.exit(f'Planned layers for {name} select {patch} more than once') + seen.add(patch.resolve()) + for _, _, topic, _ in writes: + directory = topic_directory(topic) + count = sum(layer == directory or layer in directory.parents + for layer, _ in planned_layers) + if count != 1: + sys.exit(f'Planned layers for {name} overlap at {topic}; use non-overlapping topics') + print(f'Patch scope: {name}') + for entry in missing: + print(f'{descriptions[name]["series"]} add {entry}') + if args.dry_run: sys.exit(0) written = [] for sha, subject, topic, number in writes: - directory = patches / topic + directory = topic_directory(topic) directory.mkdir(parents=True, exist_ok=True) name = git('format-patch', '--zero-commit', '--no-signature', '--start-number', str(number), '-o', str(directory), '-1', sha, '--', '.', ':(exclude,glob)arch/*/configs/orchard_*_defconfig') - written.append(Path(name)) + path = Path(name) + written.append(path) + if selected_platforms: + Path(str(path) + '.toml').write_text('schema = 1\nplatforms = [' + + ', '.join(json.dumps(name) for name in selected_platforms) + ']\n') + +series_updates = {} +for name, additions in layer_additions.items(): + series = (args.orchard / descriptions[name]['series']).resolve() + entries = series_updates.setdefault(series, []) + entries.extend(entry for entry in additions if entry not in entries) +for series, additions in series_updates.items(): + if additions: + text = series.read_text() + series.write_text(text + ('' if not text or text.endswith('\n') else '\n') + + '\n'.join(additions) + '\n') +for name in selected_platforms: + selected = set(series_api['patches'](name)) + if not set(written).issubset(selected): + sys.exit(f'Exported patches are missing from {name}') # Defconfig changes: the branch's defconfig, expanded, becomes orchard's config. IMAGE = 'localhost/orchard-hardware-kernel-toolchain:bookworm' diff --git a/tools/kernel-tree/regenerate b/tools/kernel-tree/regenerate index 7076b911..112a4f04 100755 --- a/tools/kernel-tree/regenerate +++ b/tools/kernel-tree/regenerate @@ -74,18 +74,22 @@ try: def lock(path): return dict(l.split('=', 1) for l in (snapshot / path).read_text().splitlines() if '=' in l) - def layers(path): - """The series' layer lines, without comments: what is applied.""" - return [l.split('#', 1)[0].strip() for l in (snapshot / path).read_text().splitlines() - if l.split('#', 1)[0].strip()] + def selected(name): + """What the snapshot's kernel build applies, including patch scopes.""" + return run(sys.executable, snapshot / 'kernel/series.py', name).splitlines() + selections = {name: selected(name) for name in platforms} + # Validate every branch before fetching, creating refs or running a toolchain. for branch, names in sorted(branches.items()): first = described[names[0]] for name in names[1:]: if lock(described[name]['upstream']) != lock(first['upstream']): sys.exit(f'{name} and {names[0]} share branch {branch} but not their upstream base') - if layers(described[name]['series']) != layers(first['series']): - sys.exit(f'{name} and {names[0]} share branch {branch} but not their patch layers') + if selections[name] != selections[names[0]]: + sys.exit(f'{name} and {names[0]} share branch {branch} but select different patches') + + for branch, names in sorted(branches.items()): + first = described[names[0]] git(args.linux, 'check-ref-format', '--branch', branch) base = lock(first['upstream']) if subprocess.run(['git', '-C', args.linux, 'cat-file', '-e', base['commit'] + '^{commit}'], @@ -106,7 +110,7 @@ try: subprocess.run([sys.executable, '-I', HERE / 'convert.py', '--orchard', ORCHARD, '--commit', commit, '--layers', 'kernel', '--series', Path(first['series']).relative_to('kernel'), - '--refs', branch, '--target', args.linux, '--base', base['commit'], + '--platform', names[0], '--refs', branch, '--target', args.linux, '--base', base['commit'], '--map', args.linux / '.git' / f'orchard-patch-map-{branch.replace("/", "-")}'], check=True) converted = git(args.linux, 'rev-parse', 'refs/orchard/converted') From 7deef1cf0cd92d7b2eb5d492f4666e77f2e83e1e Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:43:46 -0600 Subject: [PATCH 06/16] p105: record PPN geometry and read qualification boundaries Record the measured geometry, native address views and the proposed kernel read path. Keep the chip evidence and device capability comments consistent with the pending exact-revision qualification. All 34 catalog tests pass; regenerating the matrix keeps capability rows. Assisted-by: LLM --- chips/s5l8942x.toml | 1 + devices/p105/device.toml | 2 ++ docs/bring-up/p105-nand-reads.md | 42 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 docs/bring-up/p105-nand-reads.md diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index 32a4fc52..b49e83a5 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -14,6 +14,7 @@ stages = ["iBSS", "iBEC"] platform = "P105 on a Cascadia-based build" stages = "Project Cascadia P105 boot chain" reference_version = "Project Cascadia builds its A5 boot chain from 8.4.1 / 12H321" +nand_reads = "Measured P105 64 GB PPN geometry has eight CAUs per bus, 2088 blocks per CAU and 128 normal pages per block; exact ARMv7 read-topic qualification is pending (docs/bring-up/p105-nand-reads.md)." cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff awaits hardware qualification" # Public Cascadia RAM recovery correlates a guarded host USB identity with the live FDT and boot ID. diff --git a/devices/p105/device.toml b/devices/p105/device.toml index 95a55378..ab53a130 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -40,6 +40,8 @@ accelerometer = "missing" gyroscope = "missing" light_sensor = "missing" touch_id = "missing" +# PPN geometry/read topic and native-map provider await exact-build qualification. +# See docs/bring-up/p105-nand-reads.md. storage_read = "missing" storage_write = "missing" wifi = "untested" diff --git a/docs/bring-up/p105-nand-reads.md b/docs/bring-up/p105-nand-reads.md new file mode 100644 index 00000000..528bf571 --- /dev/null +++ b/docs/bring-up/p105-nand-reads.md @@ -0,0 +1,42 @@ +# P105 PPN geometry and reads + +The measured 64 GB P105 package exposes two H2FMI buses. Each bus has eight +CAUs with 2,088 blocks per CAU and 128 normal-view pages per block. A page +contains 16 KiB of data and 64 bytes of metadata. The reference two-CAU +package uses a different address layout, so the driver queries both PPN +parameter copies and accepts only the measured profiles. + +The 64 GB row places the page in bits 0 through 6, the block in bits 7 +through 18 and the CAU from bit 19. Four row cycles carry the address. +Context reads use the separately selected SLC view. The MTD byte offset +remains 64 bits throughout, including addresses above 4 GiB. + +The proposed read path validates completion status before asking for payload, +restores inherited controller state on exit, and stops a bus after a +transport or cleanup failure. A failed zero-byte read returns a terminal +error, which prevents MEMREAD from repeating the same page indefinitely. +Page programming belongs to a separate topic. + +## Evidence and scope + +The source branch starts at current ARMv7 base +`97d6283e22e8f989a9d1027d99dc519b5f1f7f81` and ends at +[`fe8c2f6960bd9d9fa0b051560598069497b31d82`](https://github.com/shaynelegault/linux/commit/fe8c2f6960bd9d9fa0b051560598069497b31d82). +The actual read functions passed 140,146 assertions under ASan and UBSan. +Focused ARMv7 driver and KUnit objects compiled. Synthetic public KUnit +fixtures check both parameter profiles, address uniqueness, context-view +bounds and malformed responses. + +Earlier device captures supplied the geometry and native logical-page +crosschecks. Exact-revision hardware qualification remains pending. The new +candidate must establish USB recovery before a bounded read-only session; +its source omits the program and erase paths. + +Raw PPN reads expose physical flash pages. Apple's VFL and SwissPPNFTL maps +still determine the logical medium, with LwVM describing its partitions. +The dependent kernel translation reader follows those native maps. +Encrypted iOS volume contents retain their encryption at this layer. + +P105 storage capability states await qualification of the logical provider. +The read topic also changes the imported reference profile's shared +controller code; that profile needs a regression check before merge. From c6b9cb69da74de2150889ce56400d5f2bdfe52be Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:00:43 -0600 Subject: [PATCH 07/16] mtd: export scoped PPN geometry and read support Export the current ARMv7 H2FMI read source through kernel-tree/export.py. Keep the existing import intact and scope the new MTD layer to apple-armv7. The generated mail patch reproduces source fe8c2f696 exactly. The owner approved draft exports before physical qualification. The new source has focused ARMv7 and sanitizer results; exact-build device checks remain pending. Public logical storage capability stays gated. Assisted-by: LLM --- ...Discover-and-validate-PPN-package-ge.patch | 824 ++++++++++++++++++ ...ver-and-validate-PPN-package-ge.patch.toml | 2 + kernel/series/apple-armv7 | 1 + 3 files changed, 827 insertions(+) create mode 100644 kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch create mode 100644 kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch.toml diff --git a/kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch b/kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch new file mode 100644 index 00000000..3747151b --- /dev/null +++ b/kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch @@ -0,0 +1,824 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 11:29:40 -0600 +Subject: [PATCH] mtd: apple-h2fmi: Discover and validate PPN package geometry + +Query both PPN parameter pages before registering MTD devices. The measured +64 GB P105 has eight CAUs per bus with 128 pages per block; the imported +fixed geometry aliases addresses on that profile. Retain support for the +reference two-CAU package and reject unsupported or mismatched profiles. + +Use four row address cycles for the supported PPN protocol. Check status +before requesting payload, restore the inherited controller state on every +exit, and stop I/O after a transport or cleanup failure. Report a terminal +error for a failed zero-byte MTD read so MEMREAD cannot repeat that page. +Keep page programming and capture interfaces outside this read change. + +Add synthetic KUnit tests for parameter parsing and address uniqueness, +including offsets beyond 4 GiB and the context address view. A host harness +of the actual read functions passed 140146 ASan/UBSan assertions. Exact +current-base hardware qualification is pending. + +Assisted-by: LLM +--- + drivers/mtd/devices/Kconfig | 11 +- + drivers/mtd/devices/Makefile | 2 + + drivers/mtd/devices/mtd-apple-h2fmi-test.c | 172 ++++++++++++ + drivers/mtd/devices/mtd-apple-h2fmi.c | 251 ++++++++++-------- + drivers/mtd/devices/mtd-apple-h2fmi.h | 122 +++++++++ + .../testing/kunit/configs/apple_h2fmi.config | 3 + + 6 files changed, 445 insertions(+), 116 deletions(-) + create mode 100644 drivers/mtd/devices/mtd-apple-h2fmi-test.c + create mode 100644 drivers/mtd/devices/mtd-apple-h2fmi.h + create mode 100644 tools/testing/kunit/configs/apple_h2fmi.config + +diff --git a/drivers/mtd/devices/Kconfig b/drivers/mtd/devices/Kconfig +index e73b65842..2b6e1efae 100644 +--- a/drivers/mtd/devices/Kconfig ++++ b/drivers/mtd/devices/Kconfig +@@ -213,7 +213,7 @@ config MTD_APPLE_H2FMI + Apple's flash controller as it appears on the A5 (S5L8940X/8942X), + driving the Hynix PPN packages the iPad mini 1 boots iOS from. The + package does its own ECC, so this is a plain MTD and not a raw-NAND +- driver. One MTD per die; four on this board. ++ driver. One MTD per CAU; geometry is read from each package. + + READ ONLY. The driver's opcode allowlist contains no program and no + erase, so it cannot damage whatever the NAND already holds -- which +@@ -224,6 +224,15 @@ config MTD_APPLE_H2FMI + + If unsure, say N. + ++config MTD_APPLE_H2FMI_KUNIT_TEST ++ tristate "KUnit tests for Apple H2FMI PPN geometry" if !KUNIT_ALL_TESTS ++ depends on KUNIT ++ default KUNIT_ALL_TESTS ++ help ++ Test PPN parameter parsing, row encoding and page-status handling. ++ Both measured PPN package profiles are covered. ++ These tests use synthetic parameter pages and do not access hardware. ++ + if MTD_DOCG3 + config BCH_CONST_M + default 14 +diff --git a/drivers/mtd/devices/Makefile b/drivers/mtd/devices/Makefile +index 8135496fe..cc0a852d2 100644 +--- a/drivers/mtd/devices/Makefile ++++ b/drivers/mtd/devices/Makefile +@@ -22,3 +22,5 @@ obj-$(CONFIG_MTD_APPLE_H2FMI) += mtd-apple-h2fmi.o + + + CFLAGS_docg3.o += -I$(src) ++ ++obj-$(CONFIG_MTD_APPLE_H2FMI_KUNIT_TEST) += mtd-apple-h2fmi-test.o +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi-test.c b/drivers/mtd/devices/mtd-apple-h2fmi-test.c +new file mode 100644 +index 000000000..5308972e7 +--- /dev/null ++++ b/drivers/mtd/devices/mtd-apple-h2fmi-test.c +@@ -0,0 +1,172 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++#include ++#include ++ ++#include "mtd-apple-h2fmi.h" ++ ++static const u32 ppn_16gb[] = { ++ 2, 4, 1064, 11, 256, 128, 8, 1, 2, 16448, ++}; ++ ++static const u32 ppn_64gb[] = { ++ 8, 3, 2088, 12, 128, 64, 7, 1, 2, 16448, ++}; ++ ++static void h2fmi_test_params(u8 *params, const u32 *fields) ++{ ++ int i; ++ ++ memset(params, 0, H2FMI_PARAMS_SIZE); ++ memcpy(params, "PPN Device Info", sizeof("PPN Device Info") - 1); ++ for (i = 0; i < ARRAY_SIZE(ppn_64gb); i++) ++ put_unaligned_le32(fields[i], params + 0x10 + i * sizeof(u32)); ++} ++ ++static void h2fmi_geometry_test(struct kunit *test) ++{ ++ u8 params[H2FMI_PARAMS_SIZE]; ++ struct h2fmi_geometry g; ++ ++ h2fmi_test_params(params, ppn_16gb); ++ KUNIT_ASSERT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_EXPECT_EQ(test, g.caus, 2U); ++ KUNIT_EXPECT_EQ(test, g.slc_pages, 128U); ++ KUNIT_EXPECT_EQ(test, (u64)g.blocks * g.pages * H2FMI_PAGE_DATA, 4462739456ULL); ++ h2fmi_test_params(params, ppn_64gb); ++ KUNIT_ASSERT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_EXPECT_EQ(test, g.caus, 8U); ++ KUNIT_EXPECT_EQ(test, g.slc_pages, 64U); ++ KUNIT_EXPECT_EQ(test, (u64)g.blocks * g.pages * H2FMI_PAGE_DATA, 4378853376ULL); ++ KUNIT_EXPECT_EQ(test, 2ULL * g.caus * g.blocks * g.pages * H2FMI_PAGE_DATA, ++ 70061654016ULL); ++} ++ ++static void h2fmi_invalid_geometry_test(struct kunit *test) ++{ ++ u8 params[H2FMI_PARAMS_SIZE]; ++ struct h2fmi_geometry g, sentinel; ++ int i; ++ ++ memset(&sentinel, 0xa5, sizeof(sentinel)); ++ for (i = 0; i < ARRAY_SIZE(ppn_64gb); i++) { ++ h2fmi_test_params(params, ppn_64gb); ++ put_unaligned_le32(0, params + 0x10 + i * sizeof(u32)); ++ g = sentinel; ++ KUNIT_EXPECT_LT(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_EXPECT_MEMEQ(test, &g, &sentinel, sizeof(g)); ++ } ++ h2fmi_test_params(params, ppn_64gb); ++ KUNIT_EXPECT_EQ(test, h2fmi_parse_geometry(params, sizeof(params) - 1, &g), -EINVAL); ++ KUNIT_EXPECT_EQ(test, h2fmi_parse_geometry(params, 0, &g), -EINVAL); ++ params[0] ^= 1; ++ KUNIT_EXPECT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), -EINVAL); ++} ++ ++static void h2fmi_row_test(struct kunit *test) ++{ ++ u8 params[H2FMI_PARAMS_SIZE]; ++ struct h2fmi_geometry g; ++ u32 row, previous = 0; ++ unsigned int cau, block, page; ++ ++ h2fmi_test_params(params, ppn_64gb); ++ KUNIT_ASSERT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_ASSERT_EQ(test, h2fmi_page_row(&g, 0, 2048, 0, &row), 0); ++ KUNIT_EXPECT_EQ(test, row, 0x40000U); ++ KUNIT_ASSERT_EQ(test, h2fmi_page_row(&g, 1, 0, 0, &row), 0); ++ KUNIT_EXPECT_EQ(test, row, 0x80000U); ++ for (cau = 0; cau < g.caus; cau++) { ++ for (block = 0; block < g.blocks; block++) { ++ for (page = 0; page < g.pages; page++) { ++ h2fmi_page_row(&g, cau, block, page, &row); ++ if (cau || block || page) { ++ if (row <= previous) { ++ KUNIT_FAIL(test, "PPN row alias"); ++ return; ++ } ++ } ++ previous = row; ++ } ++ } ++ } ++ KUNIT_EXPECT_EQ(test, previous, 0x3c13ffU); ++ row = 0xdeadbeef; ++ KUNIT_EXPECT_EQ(test, h2fmi_page_row(&g, 8, 0, 0, &row), -EINVAL); ++ KUNIT_EXPECT_EQ(test, h2fmi_page_row(&g, 0, 2088, 0, &row), -EINVAL); ++ KUNIT_EXPECT_EQ(test, h2fmi_page_row(&g, 0, 0, 128, &row), -EINVAL); ++ KUNIT_EXPECT_EQ(test, row, 0xdeadbeefU); ++ h2fmi_test_params(params, ppn_16gb); ++ KUNIT_ASSERT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_ASSERT_EQ(test, h2fmi_page_row(&g, 1, 1063, 255, &row), 0); ++ KUNIT_EXPECT_EQ(test, row, 0xc27ffU); ++} ++ ++static void h2fmi_slc_row_test(struct kunit *test) ++{ ++ u8 params[H2FMI_PARAMS_SIZE]; ++ struct h2fmi_geometry g; ++ u32 row, previous = 0; ++ unsigned int cau, block, page; ++ ++ h2fmi_test_params(params, ppn_64gb); ++ KUNIT_ASSERT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_ASSERT_EQ(test, h2fmi_slc_row(&g, 0, 2087, 0, &row), 0); ++ KUNIT_EXPECT_EQ(test, row, 0x441380U); ++ for (cau = 0; cau < g.caus; cau++) { ++ for (block = 0; block < g.blocks; block++) { ++ for (page = 0; page < g.slc_pages; page++) { ++ h2fmi_slc_row(&g, cau, block, page, &row); ++ if (row <= previous || !(row & 0x400000)) { ++ KUNIT_FAIL(test, "PPN SLC row alias or mode bit lost"); ++ return; ++ } ++ previous = row; ++ } ++ } ++ } ++ KUNIT_EXPECT_EQ(test, previous, 0x7c13bfU); ++ row = 0xdeadbeef; ++ KUNIT_EXPECT_EQ(test, h2fmi_slc_row(&g, 8, 0, 0, &row), -EINVAL); ++ KUNIT_EXPECT_EQ(test, h2fmi_slc_row(&g, 0, 2088, 0, &row), -EINVAL); ++ KUNIT_EXPECT_EQ(test, h2fmi_slc_row(&g, 0, 0, 64, &row), -EINVAL); ++ KUNIT_EXPECT_EQ(test, row, 0xdeadbeefU); ++ h2fmi_test_params(params, ppn_16gb); ++ KUNIT_ASSERT_EQ(test, h2fmi_parse_geometry(params, sizeof(params), &g), 0); ++ KUNIT_ASSERT_EQ(test, h2fmi_slc_row(&g, 1, 1063, 127, &row), 0); ++ KUNIT_EXPECT_EQ(test, row, 0x8c277fU); ++ KUNIT_EXPECT_EQ(test, h2fmi_slc_row(&g, 0, 0, 128, &row), -EINVAL); ++} ++ ++static void h2fmi_status_test(struct kunit *test) ++{ ++ unsigned int status; ++ ++ for (status = 0; status <= 0xff; status++) { ++ int result = h2fmi_page_status(status); ++ ++ if (status == PPN_ST_GOOD || status == PPN_ST_ERASED) ++ KUNIT_EXPECT_EQ(test, result, 0); ++ else if (status == PPN_ST_REFRESH) ++ KUNIT_EXPECT_EQ(test, result, 1); ++ else ++ KUNIT_EXPECT_EQ(test, result, -EBADMSG); ++ } ++} ++ ++static struct kunit_case h2fmi_test_cases[] = { ++ KUNIT_CASE(h2fmi_geometry_test), ++ KUNIT_CASE(h2fmi_invalid_geometry_test), ++ KUNIT_CASE(h2fmi_row_test), ++ KUNIT_CASE(h2fmi_status_test), ++ KUNIT_CASE(h2fmi_slc_row_test), ++ {} ++}; ++ ++static struct kunit_suite h2fmi_test_suite = { ++ .name = "apple-h2fmi", ++ .test_cases = h2fmi_test_cases, ++}; ++ ++kunit_test_suite(h2fmi_test_suite); ++ ++MODULE_LICENSE("GPL"); +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi.c b/drivers/mtd/devices/mtd-apple-h2fmi.c +index bdb89a700..c83bc9624 100644 +--- a/drivers/mtd/devices/mtd-apple-h2fmi.c ++++ b/drivers/mtd/devices/mtd-apple-h2fmi.c +@@ -1,6 +1,6 @@ + // SPDX-License-Identifier: GPL-2.0-only + /* +- * Apple S5L H2FMI + Hynix PPN NAND -- READ ONLY. ++ * Apple S5L H2FMI + Hynix PPN NAND. + * + * The iPad mini 1 has two PPN packages, one on CE0 of each of the SoC's two + * flash buses. "PPN" means the package has its own controller: it does the +@@ -14,23 +14,22 @@ + * this is a port of, tools/nand/ppn.c, which read the whole NAND bit-for-bit + * repeatably. docs/research/p105-nand.md is the notebook. + * +- * Geometry: 2 buses x 1 CE x 2 CAUs x 1064 blocks x 256 pages. A page comes +- * off the bus as 16448 bytes -- four times (1024 data, 16 metadata, 3072 +- * data). The metadata is the FTL's, and becomes MTD's OOB; strip it and a +- * page is 16 KiB of data with 64 bytes of OOB. One MTD is exported per die, +- * which is also how the verified raw dumps in docs are laid out, so a fresh +- * read can be diffed against them byte for byte. ++ * Geometry is read from each PPN package before registering any MTDs. ++ * The supported packages have one CE per bus and either two or eight CAUs. ++ * A page has 16448 bytes: four chunks of 1024 data bytes, 16 metadata ++ * bytes and 3072 more data bytes. The metadata is the FTL's, and becomes MTD's OOB; strip it and a ++ * page is 16 KiB of data with 64 bytes of OOB. One MTD is exported per ++ * CAU. Raw MTD offsets are independent of the iOS logical block addresses. + * +- * READ ONLY, and structurally so: the opcode allowlist below has no program +- * (80/10, PPN boot page 8A/17) and no erase (60/D0, PPN 67) in it, and there +- * is no code path that could issue one. Writing is a separate job that costs +- * iOS the first time it runs; see the README. ++ * This interface exposes reads through MTD. Logical-address translation and ++ * allocation belong to the shared FTL above the physical page provider. + */ + + #include + #include + #include + #include ++#include + #include + #include + #include +@@ -38,31 +37,10 @@ + #include + #include + ++#include "mtd-apple-h2fmi.h" ++ + #define H2FMI_BUSES 2 +-#define H2FMI_CAUS 2 +-#define H2FMI_DIES (H2FMI_BUSES * H2FMI_CAUS) +-#define H2FMI_BLOCKS 1064 +-#define H2FMI_PAGES 256 +- +-#define H2FMI_CHUNKS 4 /* per page */ +-#define H2FMI_CHUNK_LO 1024 /* data before the metadata */ +-#define H2FMI_CHUNK_META 16 +-#define H2FMI_CHUNK_HI 3072 /* data after it */ +-#define H2FMI_CHUNK_RAW (H2FMI_CHUNK_LO + H2FMI_CHUNK_META + H2FMI_CHUNK_HI) +- +-#define H2FMI_PAGE_RAW (H2FMI_CHUNKS * H2FMI_CHUNK_RAW) /* 16448 */ +-#define H2FMI_PAGE_DATA (H2FMI_CHUNKS * (H2FMI_CHUNK_LO + H2FMI_CHUNK_HI)) +-#define H2FMI_PAGE_OOB (H2FMI_CHUNKS * H2FMI_CHUNK_META) /* 64 */ +-#define H2FMI_ERASESIZE ((u32)H2FMI_PAGES * H2FMI_PAGE_DATA) /* 4 MiB */ +- +-/* Data page and erase block are both powers of two (16 KiB and 4 MiB), so an +- * offset splits with shifts and masks. This is a 32-bit kernel and the MTD is +- * 4 GiB, so every offset is 64-bit: a plain / or % here would be a link error +- * for __aeabi_uldivmod, which is exactly the kind of thing that only shows up +- * at the end of a build. */ +-#define H2FMI_PAGE_SHIFT 14 +-#define H2FMI_PAGES_SHIFT 8 /* 256 pages a block */ +-#define H2FMI_PAGE_MASK ((u32)H2FMI_PAGE_DATA - 1) ++#define H2FMI_PAGE_MASK (H2FMI_PAGE_DATA - 1) + + /* The ADT's ce-bitmap is 0x101: CE0 on each bus, and nothing else. Addressing + * a CE with no chip behind it wedges the FMC in DDR -- it waits for a DQS that +@@ -91,22 +69,11 @@ + #define FMC_NAND_STATUS 0x48 + #define FMC_STATUS_MASK 0x4c + +-/* Page status: 0x40 good, 0x42 good but due a refresh, 0x45 every page of a +- * retired block, 0x49 erased. */ +-#define PPN_ST_GOOD 0x40 +-#define PPN_ST_REFRESH 0x42 +-#define PPN_ST_RETIRED 0x45 +-#define PPN_ST_ERASED 0x49 +- +-/* row for the page-read command */ +-#define PPN_ROW(cau, blk, pg) \ +- ((u32)(pg) | (u32)(blk) << 8 | (u32)(cau) << 19) +- + /* The allowlist is the safety property, not a convenience: a sequence that is + * not on it does not run. Nothing here programs or erases. */ + static const u8 h2fmi_read_only_ops[][2] = { +- { 0xff, 0x00 }, /* reset */ + { 0x70, 0x00 }, /* read status */ ++ { 0x92, 0x97 }, /* PPN device parameters */ + { 0x77, 0x7d }, /* PPN operation status */ + { 0x77, 0x00 }, /* PPN end of operation */ + { 0x7a, 0x00 }, /* PPN data out */ +@@ -119,6 +86,8 @@ struct h2fmi_bus { + void __iomem *ecc; + struct mutex lock; /* one command sequence on this bus at a time */ + u8 *page; /* one raw page; only touched under lock */ ++ struct h2fmi_geometry geometry; ++ bool failed; /* block further I/O in this driver instance */ + }; + + struct h2fmi_die { +@@ -130,8 +99,9 @@ struct h2fmi_die { + struct h2fmi { + struct device *dev; + struct h2fmi_bus bus[H2FMI_BUSES]; +- struct mtd_info mtd[H2FMI_DIES]; +- struct h2fmi_die die[H2FMI_DIES]; ++ struct mtd_info *mtd; ++ struct h2fmi_die *die; ++ unsigned int ndies; + }; + + /* +@@ -156,7 +126,7 @@ static int h2fmi_wait(struct h2fmi *fmi, void __iomem *base, u32 off, + + /* + * FMC_CMD holds the first command byte, and the second one in bits 15:8. +- * Every sequence goes through here, which is where the allowlist bites. ++ * Every command sequence passes through the read-operation allowlist. + */ + static int h2fmi_cmd_write(struct h2fmi *fmi, struct h2fmi_bus *b, u8 op, u8 op2) + { +@@ -196,8 +166,11 @@ static int h2fmi_cmd1(struct h2fmi *fmi, struct h2fmi_bus *b, u8 op) + static int h2fmi_cmd_addr_cmd(struct h2fmi *fmi, struct h2fmi_bus *b, + u8 op, u8 op2, u32 addr, int naddr) + { +- int ret = h2fmi_cmd_write(fmi, b, op, op2); ++ int ret; + ++ if (naddr < 1 || naddr > sizeof(addr)) ++ return -EINVAL; ++ ret = h2fmi_cmd_write(fmi, b, op, op2); + if (ret) + return ret; + writel(addr, b->fmc + FMC_ADDR0); +@@ -306,27 +279,61 @@ static void h2fmi_device_reset(struct h2fmi_bus *b, u32 on, u32 if_ctrl) + writel(if_ctrl, b->fmc + FMC_IF_CTRL); + } + +-/* One raw page, into b->page. Caller holds b->lock. */ +-static int h2fmi_read_raw_page(struct h2fmi *fmi, struct h2fmi_bus *b, +- u32 row, u8 *st) ++/* Caller holds b->lock. All exits release CE and restore inherited state. */ ++static int h2fmi_read_transaction(struct h2fmi *fmi, struct h2fmi_bus *b, ++ u8 op, u8 op2, u32 address, int naddr, ++ u8 *buffer, u32 length, u8 *status) + { +- u32 on = readl(b->fmc + FMC_ON); +- u32 if_ctrl = readl(b->fmc + FMC_IF_CTRL); +- int ret; ++ u32 on, if_ctrl; ++ int ret, end_ret; ++ ++ *status = 0; ++ if (b->failed) ++ return -EIO; ++ on = readl(b->fmc + FMC_ON); ++ if_ctrl = readl(b->fmc + FMC_IF_CTRL); ++ if (on != 5) ++ return -EIO; + +- *st = 0; + writel(1u << H2FMI_CE, b->fmc + FMC_CE_CTRL); + h2fmi_clear_irqs(b); +- +- ret = h2fmi_cmd_addr_cmd(fmi, b, 0x0a, 0x37, row, 3); +- if (!ret) +- ret = h2fmi_status(fmi, b, st); ++ ret = h2fmi_cmd_addr_cmd(fmi, b, op, op2, address, naddr); + if (!ret) +- ret = h2fmi_data_out(fmi, b, b->page, H2FMI_PAGE_RAW); +- +- h2fmi_cmd1(fmi, b, 0x77); ++ ret = h2fmi_status(fmi, b, status); ++ /* An error status has no data phase. Device parameters require GOOD; ++ * normal pages also admit REFRESH and ERASED. Keep the status for the ++ * MTD reader and bad-block query without copying a stale page buffer. ++ */ ++ if (!ret && (op == 0x92 ? *status == PPN_ST_GOOD : ++ h2fmi_page_status(*status) >= 0)) ++ ret = h2fmi_data_out(fmi, b, buffer, length); ++ ++ /* Stop PIO before ending the operation, as in the parameter reader. */ ++ writel(6, b->fmi + FMI_CONTROL); ++ end_ret = h2fmi_cmd1(fmi, b, 0x77); + writel(0, b->fmc + FMC_CE_CTRL); + h2fmi_device_reset(b, on, if_ctrl); ++ if (ret || end_ret) { ++ b->failed = true; ++ return ret ? ret : end_ret; ++ } ++ return 0; ++} ++ ++static int h2fmi_read_geometry(struct h2fmi *fmi, struct h2fmi_bus *b) ++{ ++ u8 status = 0; ++ int ret; ++ ++ mutex_lock(&b->lock); ++ ret = h2fmi_read_transaction(fmi, b, 0x92, 0x97, 0, 1, ++ b->page, H2FMI_PARAMS_SIZE, &status); ++ if (!ret && status != PPN_ST_GOOD) ++ ret = -EIO; ++ if (!ret) ++ ret = h2fmi_parse_geometry(b->page, H2FMI_PARAMS_SIZE, ++ &b->geometry); ++ mutex_unlock(&b->lock); + return ret; + } + +@@ -353,14 +360,6 @@ static void h2fmi_split_page(const u8 *raw, u8 *data, u8 *oob) + } + } + +-/* +- * The one place that turns an MTD offset into a page and reads it. Whole +- * pages are always read -- the PPN cannot hand over less than one -- and the +- * caller takes what it wants out of the bounce buffer. +- * +- * `from` must be page-aligned. The page's status byte comes back in *stp; a +- * negative return means the bus failed, not that the data is bad. +- */ + static int h2fmi_read_page_at(struct mtd_info *mtd, loff_t from, + u8 *data, u8 *oob, u8 *stp) + { +@@ -368,18 +367,31 @@ static int h2fmi_read_page_at(struct mtd_info *mtd, loff_t from, + struct h2fmi *fmi = die->fmi; + struct h2fmi_bus *b = &fmi->bus[die->bus]; + u64 page = (u64)from >> H2FMI_PAGE_SHIFT; +- u32 blk = (u32)(page >> H2FMI_PAGES_SHIFT); +- u32 pg = (u32)page & (H2FMI_PAGES - 1); ++ u32 blk = page >> b->geometry.page_bits; ++ u32 pg = page & (b->geometry.pages - 1); ++ u32 row; + u8 st = 0; + int ret; + +- if (blk >= H2FMI_BLOCKS) ++ if (from < 0 || from >= mtd->size || (from & H2FMI_PAGE_MASK)) + return -EINVAL; ++ ret = h2fmi_page_row(&b->geometry, die->cau, blk, pg, &row); ++ if (ret) ++ return ret; + + mutex_lock(&b->lock); +- ret = h2fmi_read_raw_page(fmi, b, PPN_ROW(die->cau, blk, pg), &st); +- if (!ret) +- h2fmi_split_page(b->page, data, oob); ++ /* Supported PPN 1.5 packages require four row cycles, even when the ++ * fourth byte is zero. This is independent of the geometry's width. ++ */ ++ ret = h2fmi_read_transaction(fmi, b, 0x0a, 0x37, row, 4, ++ b->page, H2FMI_PAGE_RAW, &st); ++ if (!ret && h2fmi_page_status(st) >= 0) { ++ if (st == PPN_ST_ERASED && ++ memchr_inv(b->page, 0xff, H2FMI_PAGE_RAW)) ++ ret = -EBADMSG; ++ else ++ h2fmi_split_page(b->page, data, oob); ++ } + mutex_unlock(&b->lock); + + *stp = st; +@@ -398,17 +410,12 @@ static int h2fmi_read_page_at(struct mtd_info *mtd, loff_t from, + */ + static int h2fmi_bitflips(struct mtd_info *mtd, u8 st) + { +- switch (st) { +- case PPN_ST_GOOD: +- case PPN_ST_ERASED: +- return 0; +- case PPN_ST_REFRESH: +- return 1; +- default: ++ int ret = h2fmi_page_status(st); ++ ++ if (ret < 0) + dev_err_ratelimited(mtd->dev.parent, + "%s: page status 0x%02x\n", mtd->name, st); +- return -EBADMSG; +- } ++ return ret; + } + + /* +@@ -463,14 +470,22 @@ static int h2fmi_mtd_read_oob(struct mtd_info *mtd, loff_t from, + } + + ret = h2fmi_read_page_at(mtd, base, dst, oob, &st); +- if (ret) ++ if (ret) { ++ /* MEMREAD retries ECC errors. No bytes were returned, so ++ * report a terminal error instead of repeating this page. ++ */ ++ if (ret == -EBADMSG) ++ ret = -EIO; + break; ++ } + + bf = h2fmi_bitflips(mtd, st); + if (bf < 0) { +- ret = bf; ++ mtd->ecc_stats.failed++; ++ ret = -EIO; + break; + } ++ mtd->ecc_stats.corrected += bf; + maxbitflips = max(maxbitflips, bf); + + if (len) { +@@ -514,27 +529,20 @@ static int h2fmi_mtd_block_isbad(struct mtd_info *mtd, loff_t ofs) + ret = h2fmi_read_page_at(mtd, ofs, NULL, NULL, &st); + if (ret) + return ret; +- switch (st) { +- case PPN_ST_GOOD: +- case PPN_ST_REFRESH: +- case PPN_ST_ERASED: +- return 0; +- case PPN_ST_RETIRED: ++ if (st == PPN_ST_RETIRED) + return 1; +- default: +- /* An unknown status does not establish that the block is good. */ +- return -EIO; +- } ++ return h2fmi_page_status(st) < 0 ? -EIO : 0; + } + + static int h2fmi_setup_mtd(struct h2fmi *fmi, int idx) + { + struct mtd_info *mtd = &fmi->mtd[idx]; + struct h2fmi_die *die = &fmi->die[idx]; ++ const struct h2fmi_geometry *g = &fmi->bus[0].geometry; + + die->fmi = fmi; +- die->bus = idx / H2FMI_CAUS; +- die->cau = idx % H2FMI_CAUS; ++ die->bus = idx / g->caus; ++ die->cau = idx % g->caus; + + mtd->priv = die; + mtd->dev.parent = fmi->dev; +@@ -544,14 +552,10 @@ static int h2fmi_setup_mtd(struct h2fmi *fmi, int idx) + return -ENOMEM; + + mtd->type = MTD_NANDFLASH; +- /* No MTD_WRITEABLE on purpose: this driver cannot write, and the NAND +- * it is pointed at holds somebody's iOS. MTD_NO_ERASE goes with it -- +- * add_mtd_device() rejects an erasesize with no ->_erase behind it +- * unless the device says erasing is not a thing it does. Both come off +- * when the write path lands. */ ++ /* PPN page reads expose the package metadata through MTD OOB. */ + mtd->flags = MTD_NO_ERASE; +- mtd->size = (u64)H2FMI_BLOCKS * H2FMI_ERASESIZE; +- mtd->erasesize = H2FMI_ERASESIZE; ++ mtd->erasesize = g->pages * H2FMI_PAGE_DATA; ++ mtd->size = (u64)g->blocks * mtd->erasesize; + mtd->writesize = H2FMI_PAGE_DATA; + mtd->writebufsize = H2FMI_PAGE_DATA; + mtd->oobsize = H2FMI_PAGE_OOB; +@@ -578,7 +582,6 @@ static int h2fmi_probe(struct platform_device *pdev) + int i, ret; + + BUILD_BUG_ON(H2FMI_PAGE_DATA != (1 << H2FMI_PAGE_SHIFT)); +- BUILD_BUG_ON(H2FMI_PAGES != (1 << H2FMI_PAGES_SHIFT)); + BUILD_BUG_ON(H2FMI_PAGE_RAW != 16448); + + fmi = devm_kzalloc(&pdev->dev, sizeof(*fmi), GFP_KERNEL); +@@ -612,7 +615,24 @@ static int h2fmi_probe(struct platform_device *pdev) + i, readl(b->fmc + FMC_ON), readl(b->fmc + FMC_IF_CTRL)); + } + +- for (i = 0; i < H2FMI_DIES; i++) { ++ /* Query both packages before publishing any part of the inventory. */ ++ for (i = 0; i < H2FMI_BUSES; i++) { ++ ret = h2fmi_read_geometry(fmi, &fmi->bus[i]); ++ if (ret) ++ return dev_err_probe(&pdev->dev, ret, ++ "bus %d: PPN discovery failed\n", i); ++ if (i && memcmp(&fmi->bus[0].geometry, &fmi->bus[i].geometry, ++ sizeof(fmi->bus[0].geometry))) ++ return dev_err_probe(&pdev->dev, -EINVAL, ++ "PPN package geometries disagree\n"); ++ } ++ fmi->ndies = H2FMI_BUSES * fmi->bus[0].geometry.caus; ++ fmi->mtd = devm_kcalloc(&pdev->dev, fmi->ndies, sizeof(*fmi->mtd), GFP_KERNEL); ++ fmi->die = devm_kcalloc(&pdev->dev, fmi->ndies, sizeof(*fmi->die), GFP_KERNEL); ++ if (!fmi->mtd || !fmi->die) ++ return -ENOMEM; ++ ++ for (i = 0; i < fmi->ndies; i++) { + ret = h2fmi_setup_mtd(fmi, i); + if (ret) { + while (--i >= 0) +@@ -622,8 +642,9 @@ static int h2fmi_probe(struct platform_device *pdev) + } + + platform_set_drvdata(pdev, fmi); +- dev_info(&pdev->dev, "%d dies, %u blocks of %u KiB each, read-only\n", +- H2FMI_DIES, H2FMI_BLOCKS, H2FMI_ERASESIZE / 1024); ++ dev_info(&pdev->dev, "%u CAUs, %u blocks of %u KiB each\n", ++ fmi->ndies, fmi->bus[0].geometry.blocks, ++ fmi->mtd[0].erasesize / 1024); + return 0; + } + +@@ -632,7 +653,7 @@ static void h2fmi_remove(struct platform_device *pdev) + struct h2fmi *fmi = platform_get_drvdata(pdev); + int i; + +- for (i = 0; i < H2FMI_DIES; i++) ++ for (i = 0; i < fmi->ndies; i++) + mtd_device_unregister(&fmi->mtd[i]); + } + +@@ -652,5 +673,5 @@ static struct platform_driver h2fmi_driver = { + }; + module_platform_driver(h2fmi_driver); + +-MODULE_DESCRIPTION("Apple S5L H2FMI / Hynix PPN NAND, read-only"); ++MODULE_DESCRIPTION("Apple S5L H2FMI / Hynix PPN NAND"); + MODULE_LICENSE("GPL"); +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi.h b/drivers/mtd/devices/mtd-apple-h2fmi.h +new file mode 100644 +index 000000000..ad573a5c0 +--- /dev/null ++++ b/drivers/mtd/devices/mtd-apple-h2fmi.h +@@ -0,0 +1,122 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef MTD_APPLE_H2FMI_H ++#define MTD_APPLE_H2FMI_H ++ ++#include ++#include ++#include ++#include ++ ++#define H2FMI_PARAMS_SIZE 512 ++#define H2FMI_CHUNKS 4 ++#define H2FMI_CHUNK_LO 1024 ++#define H2FMI_CHUNK_META 16 ++#define H2FMI_CHUNK_HI 3072 ++#define H2FMI_CHUNK_RAW 4112 ++#define H2FMI_PAGE_RAW 16448 ++#define H2FMI_PAGE_DATA 16384 ++#define H2FMI_PAGE_OOB 64 ++#define H2FMI_PAGE_SHIFT 14 ++ ++#define PPN_ST_GOOD 0x40 ++#define PPN_ST_REFRESH 0x42 ++#define PPN_ST_RETIRED 0x45 ++#define PPN_ST_ERASED 0x49 ++ ++struct h2fmi_geometry { ++ u32 caus; ++ u32 blocks; ++ u32 pages; ++ u32 page_bits; ++ u32 block_bits; ++ u32 cau_bits; ++ u32 slc_pages; ++}; ++ ++/* Restrict discovery to the two measured PPN 1.5 package profiles. The ++ * parameter response does not identify a PPN protocol revision. The existing ++ * platform compatible continues to select that protocol explicitly. ++ */ ++static inline int h2fmi_parse_geometry(const u8 *params, size_t len, ++ struct h2fmi_geometry *geometry) ++{ ++ struct h2fmi_geometry g; ++ u32 cell_bits, cells, raw_bytes; ++ ++ if (len != H2FMI_PARAMS_SIZE || ++ memcmp(params, "PPN Device Info", sizeof("PPN Device Info") - 1)) ++ return -EINVAL; ++ ++ g.caus = get_unaligned_le32(params + 0x10); ++ g.cau_bits = get_unaligned_le32(params + 0x14); ++ g.blocks = get_unaligned_le32(params + 0x18); ++ g.block_bits = get_unaligned_le32(params + 0x1c); ++ g.pages = get_unaligned_le32(params + 0x20); ++ g.slc_pages = get_unaligned_le32(params + 0x24); ++ g.page_bits = get_unaligned_le32(params + 0x28); ++ cell_bits = get_unaligned_le32(params + 0x2c); ++ cells = get_unaligned_le32(params + 0x30); ++ raw_bytes = get_unaligned_le32(params + 0x34); ++ if (raw_bytes != H2FMI_PAGE_RAW || cell_bits != 1 || cells != 2) ++ return -EOPNOTSUPP; ++ if (g.caus == 2 && g.cau_bits == 4 && g.blocks == 1064 && ++ g.block_bits == 11 && g.pages == 256 && g.slc_pages == 128 && ++ g.page_bits == 8) { ++ *geometry = g; ++ return 0; ++ } ++ if (g.caus == 8 && g.cau_bits == 3 && g.blocks == 2088 && ++ g.block_bits == 12 && g.pages == 128 && g.slc_pages == 64 && ++ g.page_bits == 7) { ++ *geometry = g; ++ return 0; ++ } ++ return -EOPNOTSUPP; ++} ++ ++static inline int h2fmi_page_row(const struct h2fmi_geometry *g, ++ unsigned int cau, unsigned int block, ++ unsigned int page, u32 *row) ++{ ++ if (cau >= g->caus || block >= g->blocks || page >= g->pages) ++ return -EINVAL; ++ *row = page | block << g->page_bits | ++ cau << (g->page_bits + g->block_bits); ++ return 0; ++} ++ ++/* The VFL bootstrap's packed requests set the bit above the CAU field. ++ * Keep this view separate from normal MTD offsets. Its address encoding ++ * follows the saved firmware; live context selection is a separate step. ++ */ ++static inline int h2fmi_slc_row(const struct h2fmi_geometry *g, ++ unsigned int cau, unsigned int block, ++ unsigned int page, u32 *row) ++{ ++ u32 value; ++ int ret; ++ ++ if (page >= g->slc_pages) ++ return -EINVAL; ++ ret = h2fmi_page_row(g, cau, block, page, &value); ++ if (ret) ++ return ret; ++ *row = value | 1u << (g->page_bits + g->block_bits + g->cau_bits); ++ return 0; ++} ++ ++/* Unknown statuses must never lead to a data-out command or a good block. */ ++static inline int h2fmi_page_status(u8 status) ++{ ++ switch (status) { ++ case PPN_ST_GOOD: ++ case PPN_ST_ERASED: ++ return 0; ++ case PPN_ST_REFRESH: ++ return 1; ++ default: ++ return -EBADMSG; ++ } ++} ++ ++#endif +diff --git a/tools/testing/kunit/configs/apple_h2fmi.config b/tools/testing/kunit/configs/apple_h2fmi.config +new file mode 100644 +index 000000000..8577f01ee +--- /dev/null ++++ b/tools/testing/kunit/configs/apple_h2fmi.config +@@ -0,0 +1,3 @@ ++CONFIG_KUNIT=y ++CONFIG_MTD=y ++CONFIG_MTD_APPLE_H2FMI_KUNIT_TEST=y diff --git a/kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch.toml b/kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0001-mtd-apple-h2fmi-Discover-and-validate-PPN-package-ge.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] diff --git a/kernel/series/apple-armv7 b/kernel/series/apple-armv7 index e6b88fc7..fdd10d76 100644 --- a/kernel/series/apple-armv7 +++ b/kernel/series/apple-armv7 @@ -2,3 +2,4 @@ # to kernel/, on torvalds v6.12 (kernel/upstream-v6.12.lock). Every *.patch # below each directory is applied in path order. patches/import/cascadia +scoped patches/drivers/mtd From c97d28bc0587cac10bdc4d2f9bd7601f8b78dd66 Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:53:52 -0600 Subject: [PATCH 08/16] p105: record complete-page programming qualification boundaries Document the default-off H2FMI program primitive and its host checks. Keep logical storage_write missing while exact-build programming and translation-layer persistence remain unqualified. Record chip evidence and the shared reference-profile regression requirement. All 34 catalog tests pass. Matrix regeneration leaves capability rows unchanged; the P105 device and A5 chip schemas validate. Assisted-by: LLM --- chips/s5l8942x.toml | 1 + devices/p105/device.toml | 2 ++ docs/bring-up/p105-nand-program.md | 48 ++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 docs/bring-up/p105-nand-program.md diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index b49e83a5..566fa06c 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -15,6 +15,7 @@ platform = "P105 on a Cascadia-based build" stages = "Project Cascadia P105 boot chain" reference_version = "Project Cascadia builds its A5 boot chain from 8.4.1 / 12H321" nand_reads = "Measured P105 64 GB PPN geometry has eight CAUs per bus, 2088 blocks per CAU and 128 normal pages per block; exact ARMv7 read-topic qualification is pending (docs/bring-up/p105-nand-reads.md)." +nand_program = "The P105 complete-page PPN program candidate defaults off, verifies blank destination and full readback, and exposes no erase operation; exact-revision hardware qualification is pending (docs/bring-up/p105-nand-program.md)." cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff awaits hardware qualification" # Public Cascadia RAM recovery correlates a guarded host USB identity with the live FDT and boot ID. diff --git a/devices/p105/device.toml b/devices/p105/device.toml index ab53a130..d9fe9d27 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -43,6 +43,8 @@ touch_id = "missing" # PPN geometry/read topic and native-map provider await exact-build qualification. # See docs/bring-up/p105-nand-reads.md. storage_read = "missing" +# The default-off complete-page primitive does not provide a qualified logical writer. +# See docs/bring-up/p105-nand-program.md for its checks and pending hardware work. storage_write = "missing" wifi = "untested" bluetooth_input = "untested" diff --git a/docs/bring-up/p105-nand-program.md b/docs/bring-up/p105-nand-program.md new file mode 100644 index 00000000..e468562d --- /dev/null +++ b/docs/bring-up/p105-nand-program.md @@ -0,0 +1,48 @@ +# P105 complete-page PPN programming + +The proposed H2FMI program primitive accepts one complete physical page: 16 KiB +of data and 64 metadata bytes. `CONFIG_MTD_APPLE_H2FMI_WRITE` defaults to `n`. +Enabling it publishes raw MTD programming through the measured PPN command +path. Read-only qualification builds leave the option disabled. + +The caller supplies every data and metadata byte. The driver serializes the +transaction with the bus lock, checks the destination's erased status and +every raw byte, then programs the page and validates completion. Success +requires a fresh read with good status and an exact comparison of the whole +page. MTD reports progress only after that readback succeeds. An uncertain +completion or mismatched readback stops further I/O on the bus. + +The request must align to a physical page and include a full data/OOB pair. +Block zero and the two reserved package-tail blocks are excluded. This +primitive exposes no erase callback and owns no free-space map. A dependent +VFL/SFTL writer must decide which blank pages it can allocate while preserving +Apple's native mapping records. + +The driver binds the shared `apple,s5l8940x-fmi` compatible, which P105 uses +in this source tree. Its imported two-CAU parameter profile also needs a +regression check before merge; the measured 64 GB profile does not qualify +that hardware. + +## Source and evidence + +The candidate is +[`cef7165b078f1d52c36fd9a38467e999ab2fefdd`](https://github.com/shaynelegault/linux/commit/cef7165b078f1d52c36fd9a38467e999ab2fefdd), +on the geometry/read topic +`fe8c2f6960bd9d9fa0b051560598069497b31d82`. The program fixture exercised +the actual driver functions with fake MMIO and passed 134,181 checks under +ASan/UBSan. Cases cover blank detection, request bounds, completion statuses, +transport failures and full readback mismatches. The read fixture retained +its 140,146 checks. Strict checkpatch reported no errors or warnings. + +Focused ARMv7 object builds passed with programming enabled and disabled; +the KUnit fixture also compiled. Those builds used the existing qualified +ARMv7 generated headers. Their source hashes match this candidate, but they +were object checks rather than a complete current-base kernel build or a +kernel KUnit run. + +Earlier bounded device programming supplied protocol evidence. This exact +revision still needs an approved physical test, including the failure and +recovery paths. It has no new physical-programming result. Raw page readback +does not establish logical-map persistence after restart or recovery after +interrupted writes. P105 `storage_write` remains `missing` until the logical +backend earns that capability through device qualification. From bf9775578c5ffd86df29d728eea32e545a45347d Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:01:25 -0600 Subject: [PATCH 09/16] mtd: export default-off verified PPN programming Export source cef7165b0 after the geometry/read dependency, preserving the scoped MTD series. Require complete pages, blank destinations and exact synchronous readback. The controller supplies no erase operation. The generated patch reproduces the source commit exactly. Programming stays disabled in the recovery candidate, and exact-revision physical qualification remains pending under the approved draft export order. Assisted-by: LLM --- ...Add-verified-complete-page-programmi.patch | 385 ++++++++++++++++++ ...erified-complete-page-programmi.patch.toml | 2 + 2 files changed, 387 insertions(+) create mode 100644 kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch create mode 100644 kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch.toml diff --git a/kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch b/kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch new file mode 100644 index 00000000..41098896 --- /dev/null +++ b/kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch @@ -0,0 +1,385 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 11:49:24 -0600 +Subject: [PATCH] mtd: apple-h2fmi: Add verified complete-page programming + +Add the measured PPN program sequence behind a default-off Kconfig option. +Require complete data and metadata, an erased destination and exact readback +before publishing progress. A transport failure, uncertain completion or +readback mismatch stops further I/O on that bus. + +Use the native program-completion status rule rather than the managed-read +status classifier. Keep allocation and shared FTL ownership policy outside +this controller interface. The driver supplies no erase operation. + +The actual program functions passed 134181 ASan/UBSan assertions and the read +path retained its 140146 assertions. Focused ARMv7 driver compilation passed +with programming enabled and disabled; KUnit source compilation passed. +The exact new candidate still needs hardware qualification. + +Assisted-by: LLM +--- + drivers/mtd/devices/Kconfig | 18 +- + drivers/mtd/devices/mtd-apple-h2fmi-test.c | 14 ++ + drivers/mtd/devices/mtd-apple-h2fmi.c | 237 +++++++++++++++++++++ + drivers/mtd/devices/mtd-apple-h2fmi.h | 7 + + 4 files changed, 272 insertions(+), 4 deletions(-) + +diff --git a/drivers/mtd/devices/Kconfig b/drivers/mtd/devices/Kconfig +index 2b6e1efae..637358e20 100644 +--- a/drivers/mtd/devices/Kconfig ++++ b/drivers/mtd/devices/Kconfig +@@ -207,7 +207,7 @@ config MTD_ST_SPI_FSM + for a subset of connected Serial Flash devices. + + config MTD_APPLE_H2FMI +- tristate "Apple S5L H2FMI / PPN NAND (read-only)" ++ tristate "Apple S5L H2FMI / PPN NAND" + depends on ARCH_APPLE_S5L || COMPILE_TEST + help + Apple's flash controller as it appears on the A5 (S5L8940X/8942X), +@@ -215,15 +215,25 @@ config MTD_APPLE_H2FMI + package does its own ECC, so this is a plain MTD and not a raw-NAND + driver. One MTD per CAU; geometry is read from each package. + +- READ ONLY. The driver's opcode allowlist contains no program and no +- erase, so it cannot damage whatever the NAND already holds -- which +- on an un-restored device is somebody's iOS install. ++ The default interface is read-only. Optional verified full-page ++ programming requires MTD_APPLE_H2FMI_WRITE. An Apple-compatible FTL ++ must own allocation and recovery before modifying shared NAND. + + Say Y to read the NAND from Linux, for instance to check a dump or + to mount iOS's own partitions. + + If unsure, say N. + ++config MTD_APPLE_H2FMI_WRITE ++ bool "Apple H2FMI PPN page programming (EXPERIMENTAL)" ++ depends on MTD_APPLE_H2FMI ++ default n ++ help ++ Build synchronous full-page programming with data and metadata ++ readback. A destination must be erased; erase operations are not ++ implemented. Allocation and recovery belong to an Apple-compatible ++ FTL that owns the shared NAND. Enable this only with such a caller. ++ + config MTD_APPLE_H2FMI_KUNIT_TEST + tristate "KUnit tests for Apple H2FMI PPN geometry" if !KUNIT_ALL_TESTS + depends on KUNIT +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi-test.c b/drivers/mtd/devices/mtd-apple-h2fmi-test.c +index 5308972e7..419c04cc8 100644 +--- a/drivers/mtd/devices/mtd-apple-h2fmi-test.c ++++ b/drivers/mtd/devices/mtd-apple-h2fmi-test.c +@@ -153,12 +153,26 @@ static void h2fmi_status_test(struct kunit *test) + } + } + ++static void h2fmi_program_status_test(struct kunit *test) ++{ ++ unsigned int status; ++ ++ for (status = 0; status <= 0xff; status++) { ++ bool ready = status & 0x40; ++ bool failed = status & 0x11; ++ ++ KUNIT_EXPECT_EQ(test, h2fmi_program_status(status), ++ ready && !failed ? 0 : -EIO); ++ } ++} ++ + static struct kunit_case h2fmi_test_cases[] = { + KUNIT_CASE(h2fmi_geometry_test), + KUNIT_CASE(h2fmi_invalid_geometry_test), + KUNIT_CASE(h2fmi_row_test), + KUNIT_CASE(h2fmi_status_test), + KUNIT_CASE(h2fmi_slc_row_test), ++ KUNIT_CASE(h2fmi_program_status_test), + {} + }; + +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi.c b/drivers/mtd/devices/mtd-apple-h2fmi.c +index c83bc9624..6f17833a3 100644 +--- a/drivers/mtd/devices/mtd-apple-h2fmi.c ++++ b/drivers/mtd/devices/mtd-apple-h2fmi.c +@@ -23,6 +23,8 @@ + * + * This interface exposes reads through MTD. Logical-address translation and + * allocation belong to the shared FTL above the physical page provider. ++ * Optional programming accepts an erased full page and verifies its raw ++ * data and metadata synchronously before reporting completion. + */ + + #include +@@ -360,6 +362,237 @@ static void h2fmi_split_page(const u8 *raw, u8 *data, u8 *oob) + } + } + ++static int h2fmi_write_cmd(struct h2fmi *fmi, struct h2fmi_bus *b, ++ u8 op, u8 op2) ++{ ++ if (!IS_ENABLED(CONFIG_MTD_APPLE_H2FMI_WRITE)) ++ return -EROFS; ++ if (!((op == 0x8a || op == 0x17 || op == 0x79) && !op2)) ++ return -EPERM; ++ writel(op | op2 << 8, b->fmc + FMC_CMD); ++ return 0; ++} ++ ++static int h2fmi_write_cmd1(struct h2fmi *fmi, struct h2fmi_bus *b, u8 op) ++{ ++ int ret = h2fmi_write_cmd(fmi, b, op, 0); ++ ++ if (ret) ++ return ret; ++ writel(1, b->fmc + FMC_RW_CTRL); ++ return h2fmi_wait(fmi, b->fmc, FMC_STATUS, 1, 1); ++} ++ ++/* Four row-address cycles follow the PPN program command. */ ++static int h2fmi_write_address(struct h2fmi *fmi, struct h2fmi_bus *b, ++ u32 row) ++{ ++ int ret = h2fmi_write_cmd(fmi, b, 0x8a, 0); ++ ++ if (ret) ++ return ret; ++ writel(row, b->fmc + FMC_ADDR0); ++ writel(3, b->fmc + FMC_ADDRNUM); ++ writel(9, b->fmc + FMC_RW_CTRL); ++ return h2fmi_wait(fmi, b->fmc, FMC_STATUS, 9, 9); ++} ++ ++/* Same PIO FIFO used by iBEC's feature writer, with CONTROL=5 for data in. ++ * Every chunk must finish before the next one or the program confirm. ++ */ ++static int h2fmi_data_in(struct h2fmi *fmi, struct h2fmi_bus *b, ++ const u8 *buffer, u32 length) ++{ ++ u32 offset, index, value; ++ int ret; ++ ++ if (!length || length & 3) ++ return -EINVAL; ++ for (offset = 0; offset < length; ) { ++ u32 bytes = min_t(u32, length - offset, 0x400); ++ ++ writel(5, b->fmi + FMI_CONFIG); ++ writel((bytes << 8 & 0x7fc00) | 1, ++ b->fmi + FMI_PIO_CONFIG); ++ writel(5, b->fmi + FMI_CONTROL); ++ ret = readl_poll_timeout(b->fmi + FMI_DMA_STATUS, value, ++ value & 0x18, 0, 100000); ++ if (ret) ++ return ret; ++ for (index = 0; index < bytes; index += 4) ++ writel(get_unaligned_le32(buffer + offset + index), ++ b->fmi + FMI_DATA); ++ ret = h2fmi_wait(fmi, b->fmi, FMI_STATUS, 2, 2); ++ if (ret) ++ return ret; ++ offset += bytes; ++ } ++ return 0; ++} ++ ++/* Program status uses 79, as in iBEC's write_bootpage path. */ ++static int h2fmi_write_status(struct h2fmi *fmi, struct h2fmi_bus *b, u8 *status) ++{ ++ u32 value; ++ int ret = h2fmi_write_cmd1(fmi, b, 0x79); ++ ++ if (ret) ++ return ret; ++ h2fmi_clear_irqs(b); ++ writel(0x100, b->fmi + FMI_INTEN); ++ writel(readl(b->fmc + FMC_IF_CTRL) & ~0x100000u, ++ b->fmc + FMC_IF_CTRL); ++ writel(0x4040, b->fmc + FMC_STATUS_MASK); ++ writel(0, b->fmc + FMC_DATANUM); ++ writel(0x20, b->fmc + FMC_INTMASK); ++ writel(0x50, b->fmc + FMC_RW_CTRL); ++ ret = readl_poll_timeout(b->fmc + FMC_STATUS, value, ++ value & 0x20, 0, 2000000); ++ if (!ret) ++ *status = readl(b->fmc + FMC_NAND_STATUS); ++ writel(0, b->fmc + FMC_RW_CTRL); ++ h2fmi_clear_irqs(b); ++ return ret; ++} ++ ++/* Caller holds the bus lock. A failed mutation permanently stops this bus. */ ++static int h2fmi_program_transaction(struct h2fmi *fmi, struct h2fmi_bus *b, ++ u32 row, const u8 *raw) ++{ ++ u32 on, if_ctrl; ++ u8 status = 0; ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_MTD_APPLE_H2FMI_WRITE)) ++ return -EROFS; ++ if (b->failed) ++ return -EIO; ++ on = readl(b->fmc + FMC_ON); ++ if_ctrl = readl(b->fmc + FMC_IF_CTRL); ++ if (on != 5) ++ return -EIO; ++ writel(1u << H2FMI_CE, b->fmc + FMC_CE_CTRL); ++ h2fmi_clear_irqs(b); ++ ret = h2fmi_write_address(fmi, b, row); ++ if (!ret) ++ ret = h2fmi_data_in(fmi, b, raw, H2FMI_PAGE_RAW); ++ if (!ret) ++ ret = h2fmi_write_cmd1(fmi, b, 0x17); ++ if (!ret) ++ ret = h2fmi_write_status(fmi, b, &status); ++ /* CMD 79 returns NAND write status, not the managed read result. ++ * iBEC 12H321 tests failure bits 0 and 4 after ready bit 6. ++ */ ++ if (!ret) ++ ret = h2fmi_program_status(status); ++ writel(0, b->fmc + FMC_CE_CTRL); ++ h2fmi_device_reset(b, on, if_ctrl); ++ if (ret) { ++ b->failed = true; ++ dev_err(fmi->dev, "PPN program row %08x: status %02x, error %d\n", ++ row, status, ret); ++ } ++ return ret; ++} ++ ++static void h2fmi_join_page(u8 *raw, const u8 *data, const u8 *oob) ++{ ++ unsigned int chunk; ++ ++ for (chunk = 0; chunk < H2FMI_CHUNKS; chunk++) { ++ u8 *dst = raw + chunk * H2FMI_CHUNK_RAW; ++ const u8 *src = data + chunk * 4096; ++ ++ memcpy(dst, src, H2FMI_CHUNK_LO); ++ memcpy(dst + H2FMI_CHUNK_LO, oob + chunk * H2FMI_CHUNK_META, ++ H2FMI_CHUNK_META); ++ memcpy(dst + H2FMI_CHUNK_LO + H2FMI_CHUNK_META, ++ src + H2FMI_CHUNK_LO, H2FMI_CHUNK_HI); ++ } ++} ++ ++/* Canonical data and metadata form one complete PPN program unit. */ ++static int h2fmi_program_page(struct h2fmi *fmi, struct h2fmi_bus *b, ++ unsigned int cau, unsigned int block, ++ unsigned int page, const u8 *data, const u8 *oob) ++{ ++ u32 row; ++ u8 status, *expected; ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_MTD_APPLE_H2FMI_WRITE)) ++ return -EROFS; ++ if (!data || !oob) ++ return -EINVAL; ++ ret = h2fmi_page_row(&b->geometry, cau, block, page, &row); ++ if (ret) ++ return ret; ++ /* Preserve the boot block and both package-reserved tail blocks. */ ++ if (!block || block >= b->geometry.blocks - 2) ++ return -EPERM; ++ expected = kmalloc(H2FMI_PAGE_RAW, GFP_KERNEL); ++ if (!expected) ++ return -ENOMEM; ++ h2fmi_join_page(expected, data, oob); ++ mutex_lock(&b->lock); ++ ret = h2fmi_read_transaction(fmi, b, 0x0a, 0x37, row, 4, ++ b->page, H2FMI_PAGE_RAW, &status); ++ if (ret) ++ goto out; ++ if (status != PPN_ST_ERASED || ++ memchr_inv(b->page, 0xff, H2FMI_PAGE_RAW)) { ++ ret = -EEXIST; ++ goto out; ++ } ++ ret = h2fmi_program_transaction(fmi, b, row, expected); ++ if (ret) ++ goto out; ++ ret = h2fmi_read_transaction(fmi, b, 0x0a, 0x37, row, 4, ++ b->page, H2FMI_PAGE_RAW, &status); ++ if (ret || status != PPN_ST_GOOD || ++ memcmp(b->page, expected, H2FMI_PAGE_RAW)) { ++ b->failed = true; ++ ret = ret ? ret : -EIO; ++ dev_err(fmi->dev, "PPN readback row %08x: status %02x, error %d\n", ++ row, status, ret); ++ goto out; ++ } ++out: ++ mutex_unlock(&b->lock); ++ kfree(expected); ++ return ret; ++} ++ ++/* Publish MTD progress after the whole data/OOB unit passes readback. */ ++static int h2fmi_mtd_write_oob(struct mtd_info *mtd, loff_t to, ++ struct mtd_oob_ops *ops) ++{ ++ struct h2fmi_die *die = mtd->priv; ++ struct h2fmi *fmi = die->fmi; ++ struct h2fmi_bus *b = &fmi->bus[die->bus]; ++ u64 page = (u64)to >> H2FMI_PAGE_SHIFT; ++ u32 block = page >> b->geometry.page_bits; ++ u32 pg = page & (b->geometry.pages - 1); ++ int ret; ++ ++ ops->retlen = 0; ++ ops->oobretlen = 0; ++ if (!IS_ENABLED(CONFIG_MTD_APPLE_H2FMI_WRITE)) ++ return -EROFS; ++ if (to < 0 || to >= mtd->size || to & H2FMI_PAGE_MASK || ++ ops->mode != MTD_OPS_PLACE_OOB || ops->len != H2FMI_PAGE_DATA || ++ ops->ooblen != H2FMI_PAGE_OOB || ops->ooboffs || ++ !ops->datbuf || !ops->oobbuf) ++ return -EINVAL; ++ ret = h2fmi_program_page(fmi, b, die->cau, block, pg, ++ ops->datbuf, ops->oobbuf); ++ if (!ret) { ++ ops->retlen = H2FMI_PAGE_DATA; ++ ops->oobretlen = H2FMI_PAGE_OOB; ++ } ++ return ret; ++} ++ + static int h2fmi_read_page_at(struct mtd_info *mtd, loff_t from, + u8 *data, u8 *oob, u8 *stp) + { +@@ -554,6 +787,10 @@ static int h2fmi_setup_mtd(struct h2fmi *fmi, int idx) + mtd->type = MTD_NANDFLASH; + /* PPN page reads expose the package metadata through MTD OOB. */ + mtd->flags = MTD_NO_ERASE; ++ if (IS_ENABLED(CONFIG_MTD_APPLE_H2FMI_WRITE)) { ++ mtd->flags |= MTD_WRITEABLE; ++ mtd->_write_oob = h2fmi_mtd_write_oob; ++ } + mtd->erasesize = g->pages * H2FMI_PAGE_DATA; + mtd->size = (u64)g->blocks * mtd->erasesize; + mtd->writesize = H2FMI_PAGE_DATA; +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi.h b/drivers/mtd/devices/mtd-apple-h2fmi.h +index ad573a5c0..aeb812f33 100644 +--- a/drivers/mtd/devices/mtd-apple-h2fmi.h ++++ b/drivers/mtd/devices/mtd-apple-h2fmi.h +@@ -2,6 +2,7 @@ + #ifndef MTD_APPLE_H2FMI_H + #define MTD_APPLE_H2FMI_H + ++#include + #include + #include + #include +@@ -119,4 +120,10 @@ static inline int h2fmi_page_status(u8 status) + } + } + ++/* CMD 79 reports ready at bit 6 and program failure at bits 0 and 4. */ ++static inline int h2fmi_program_status(u8 status) ++{ ++ return (status & BIT(6)) && !(status & (BIT(0) | BIT(4))) ? 0 : -EIO; ++} ++ + #endif diff --git a/kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch.toml b/kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0002-mtd-apple-h2fmi-Add-verified-complete-page-programmi.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] From 0b4bcc82681d049435cc81b3ddbabb03931bcfae Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:53:52 -0600 Subject: [PATCH 10/16] p105: record bounded native NAND reader evidence Document native VFL/SwissPPNFTL recovery, saved-capture crosschecks, KUnit results and the latest finite provider lifetime. Keep both logical storage capabilities missing until exact-revision device qualification. Record the host and ARM64 test limits separately from hardware evidence. All 34 catalog tests pass. Matrix regeneration leaves capability rows unchanged; the P105 device and A5 chip schemas validate. Assisted-by: LLM --- chips/s5l8942x.toml | 1 + devices/p105/device.toml | 2 + docs/bring-up/p105-sftl-reader.md | 76 +++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 docs/bring-up/p105-sftl-reader.md diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index 566fa06c..da8134d0 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -16,6 +16,7 @@ stages = "Project Cascadia P105 boot chain" reference_version = "Project Cascadia builds its A5 boot chain from 8.4.1 / 12H321" nand_reads = "Measured P105 64 GB PPN geometry has eight CAUs per bus, 2088 blocks per CAU and 128 normal pages per block; exact ARMv7 read-topic qualification is pending (docs/bring-up/p105-nand-reads.md)." nand_program = "The P105 complete-page PPN program candidate defaults off, verifies blank destination and full readback, and exposes no erase operation; exact-revision hardware qualification is pending (docs/bring-up/p105-nand-program.md)." +nand_translation = "The P105 VFL/SwissPPNFTL reader reconstructs the measured 64 GB native map from saved pages and has finite request/time bounds; exact-revision hardware qualification is pending (docs/bring-up/p105-sftl-reader.md)." cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff awaits hardware qualification" # Public Cascadia RAM recovery correlates a guarded host USB identity with the live FDT and boot ID. diff --git a/devices/p105/device.toml b/devices/p105/device.toml index d9fe9d27..ff617f52 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -42,6 +42,8 @@ light_sensor = "missing" touch_id = "missing" # PPN geometry/read topic and native-map provider await exact-build qualification. # See docs/bring-up/p105-nand-reads.md. +# The bounded native VFL/SFTL reader awaits an exact-revision device boot. +# See docs/bring-up/p105-sftl-reader.md; logical storage capabilities stay missing. storage_read = "missing" # The default-off complete-page primitive does not provide a qualified logical writer. # See docs/bring-up/p105-nand-program.md for its checks and pending hardware work. diff --git a/docs/bring-up/p105-sftl-reader.md b/docs/bring-up/p105-sftl-reader.md new file mode 100644 index 00000000..cf8bddc1 --- /dev/null +++ b/docs/bring-up/p105-sftl-reader.md @@ -0,0 +1,76 @@ +# P105 native VFL and SwissPPNFTL reader + +H2FMI exposes physical PPN pages. Apple's VFL and SwissPPNFTL records supply +the logical medium above them. The proposed `apple-sftl` kernel driver reads +those records and publishes a read-only `applesftlN` disk. LwVM partition +mapping belongs above that disk, where encrypted iOS contents retain their +encryption. + +The measured profile has two buses and eight CAUs per bus, each with 2,088 +blocks of 128 normal-view pages. H2FMI supplies each 16 KiB data page with its +64 metadata bytes through a canonical provider. Context reads select the +separate SLC address view. This provider offers no program or erase callback. + +Recovery requires agreeing VFL context replicas and lane tables. It validates +the saved SFTL context fragments, then replays later data and trim records in +sequence order. Completed-block summaries preserve allocation floors; open +blocks are scanned fully so an erased lane cannot hide a later completed +stripe. Malformed or incomplete maps prevent disk registration. Unsupported +geometry returns an error. + +The map contains 15,625,016 entries, including sixteen internal keys. The +exported medium contains 15,625,000 logical 4 KiB pages, or 64,000,000,000 bytes, +and accepts 512-byte sector reads. The frontend serializes access to its +preallocated request buffer and drains requests before releasing the provider. + +## Bounded qualification + +The latest reader candidate is +[`e7d118292d828b9796d85d19872653f7d4a8cb67`](https://github.com/shaynelegault/linux/commit/e7d118292d828b9796d85d19872653f7d4a8cb67). +It follows the canonical H2FMI provider and native-map reader commits +`9b494217444fc61adae2f95bc2ec05e461d45049` and +`e5960c537455b9c40c57deb53823e80ca9b13bb4`. + +`max_page_reads` defaults to 32,768 physical page requests across recovery and +subsequent block reads. `max_io_time_ms` defaults to 600,000 milliseconds from +probe. Both read-only module parameters must be positive. The driver checks +the deadline before and after each provider transaction. One already-bounded +transaction limits the overrun. Exhausting either policy or receiving a +provider error permanently stops that reader instance, including later reads +that require no physical I/O. These limits define a qualification session; +a mounted root cannot depend on the reader beyond its lifetime. + +## Software checks and limits + +The unchanged mapping core reconstructed a saved private capture through +sequence 187831212. An independent Python reference agreed at all 28,830 +checked endpoints of the map union. Both LwVM headers and the System prefix +matched previously captured native iOS logical reads. This comparison used +saved pages and performed zero device operations. Captured pages remain +private; public fixtures generate synthetic media. + +At `e5960c537`, eight tests passed in an ARM64 QEMU KUnit kernel built from +pristine pinned Linux v6.12 plus the exact reader/test sources and required +Kbuild entries. This ran the mapping code without Cascadia or an H2FMI +hardware backend. The host version of those fixtures also passed under +ASan/UBSan with allocation, mutex and tree-API shims. + +The canonical H2FMI provider shares controller code with the imported +two-CAU profile. That profile needs a regression check; native-map recovery +currently accepts only the measured 64 GB geometry. + +The lifetime revision adds five boundary tests for page budgets, deadlines, +provider errors and invalid policies. Its actual core/test fixture passed +again under ASan/UBSan. The changed frontend and test objects compiled against +the current ARMv7 generated-header build with `W=1` and warnings-as-errors, +retaining the existing `vmstat.h` enum-arithmetic warning as a warning. The +earlier core, frontend, fixture and provider object targets also compiled. +These checks establish source behavior and compilation; the new lifetime +tests have no separate kernel KUnit run recorded here. + +The exact candidate still needs a complete qualification image and an +approved P105 boot with working USB recovery, followed by bounded native-map +reads. Host reconstruction and an ARM64 KUnit kernel do not qualify ARMv7 +controller timing. P105 `storage_read` and `storage_write` remain `missing` +while this native logical provider awaits device qualification. Sustained +writes and interruption recovery belong to the dependent writer work. From ffa74e806d894c6b7c154fc53f20221804cf0a54 Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:03:20 -0600 Subject: [PATCH 11/16] mtd: export bounded native VFL and SwissPPNFTL reads Export the canonical PPN provider, native map reader and finite lifetime policy from current ARMv7 source. Preserve the preceding geometry and programming topics in series order, with every MTD patch scoped to ARMv7. The official export reproduces e7d118292 after its read prerequisite. Saved-capture crosschecks, real KUnit and ARMv7 object builds pass. The owner approved draft export before exact device qualification, which remains pending. Logical storage writes stay gated. Assisted-by: LLM --- ...provide-canonical-page-reads-to-SFTL.patch | 173 ++ ...de-canonical-page-reads-to-SFTL.patch.toml | 2 + ...y-Apple-VFL-and-SwissPPNFTL-recovery.patch | 1835 +++++++++++++++++ ...le-VFL-and-SwissPPNFTL-recovery.patch.toml | 2 + ...tl-bound-read-only-provider-lifetime.patch | 315 +++ ...und-read-only-provider-lifetime.patch.toml | 2 + 6 files changed, 2329 insertions(+) create mode 100644 kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch create mode 100644 kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch.toml create mode 100644 kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch create mode 100644 kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch.toml create mode 100644 kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch create mode 100644 kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch.toml diff --git a/kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch b/kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch new file mode 100644 index 00000000..2174d0d2 --- /dev/null +++ b/kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch @@ -0,0 +1,173 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 11:48:07 -0600 +Subject: [PATCH] mtd: apple-h2fmi: provide canonical page reads to SFTL + +Expose normal and SLC-context page reads through one aggregate PPN provider. +Both views keep the complete controller transaction under the bus lock and +return separate data and OOB buffers. Retired pages return their status +without copying stale bytes. + +Register an SFTL child for the supported discovered 64 GB package geometry, +and remove it before releasing the controller. The provider has no write +or erase callback. + +Assisted-by: LLM +--- + drivers/mtd/devices/mtd-apple-h2fmi.c | 78 +++++++++++++++++++++++++++ + include/linux/mtd/apple-ppn.h | 29 ++++++++++ + 2 files changed, 107 insertions(+) + create mode 100644 include/linux/mtd/apple-ppn.h + +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi.c b/drivers/mtd/devices/mtd-apple-h2fmi.c +index c83bc9624..3894e45a4 100644 +--- a/drivers/mtd/devices/mtd-apple-h2fmi.c ++++ b/drivers/mtd/devices/mtd-apple-h2fmi.c +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -102,6 +103,7 @@ struct h2fmi { + struct mtd_info *mtd; + struct h2fmi_die *die; + unsigned int ndies; ++ struct platform_device *sftl; + }; + + /* +@@ -398,6 +400,73 @@ static int h2fmi_read_page_at(struct mtd_info *mtd, loff_t from, + return ret; + } + ++/* Canonical normal/context reads share the same complete transaction lock. */ ++static int h2fmi_ppn_read(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, bool context, ++ u8 *data, u8 *oob, u8 *status) ++{ ++ struct h2fmi *fmi = priv; ++ struct h2fmi_bus *b; ++ u32 row; ++ int ret; ++ ++ if (bus >= H2FMI_BUSES || !data || !oob || !status) ++ return -EINVAL; ++ b = &fmi->bus[bus]; ++ if (context) ++ ret = h2fmi_slc_row(&b->geometry, cau, block, page, &row); ++ else ++ ret = h2fmi_page_row(&b->geometry, cau, block, page, &row); ++ if (ret) ++ return ret; ++ ++ /* Retired pages have no data phase. Never return a stale page buffer. */ ++ memset(data, 0, H2FMI_PAGE_DATA); ++ memset(oob, 0, H2FMI_PAGE_OOB); ++ mutex_lock(&b->lock); ++ ret = h2fmi_read_transaction(fmi, b, 0x0a, 0x37, row, 4, ++ b->page, H2FMI_PAGE_RAW, status); ++ if (!ret && h2fmi_page_status(*status) >= 0) { ++ if (*status == PPN_ST_ERASED && memchr_inv(b->page, 0xff, H2FMI_PAGE_RAW)) ++ ret = -EIO; ++ else ++ h2fmi_split_page(b->page, data, oob); ++ } else if (!ret && *status != PPN_ST_RETIRED) { ++ ret = -EIO; ++ } ++ mutex_unlock(&b->lock); ++ return ret; ++} ++ ++static const struct apple_ppn_ops h2fmi_ppn_ops = { ++ .read_page = h2fmi_ppn_read, ++}; ++ ++static int h2fmi_register_sftl(struct h2fmi *fmi) ++{ ++ const struct h2fmi_geometry *g = &fmi->bus[0].geometry; ++ struct apple_ppn provider = { ++ .dev = fmi->dev, ++ .ops = &h2fmi_ppn_ops, ++ .priv = fmi, ++ .buses = H2FMI_BUSES, ++ .caus = g->caus, ++ .blocks = g->blocks, ++ .pages = g->pages, ++ }; ++ ++ /* This recovery implementation supports the measured 64 GB profile. */ ++ if (!IS_ENABLED(CONFIG_MTD_APPLE_SFTL) || g->caus != 8 || ++ g->blocks != 2088 || g->pages != 128 || g->slc_pages != 64) ++ return 0; ++ fmi->sftl = platform_device_register_data(fmi->dev, "apple-sftl", ++ PLATFORM_DEVID_AUTO, ++ &provider, sizeof(provider)); ++ if (IS_ERR(fmi->sftl)) ++ return PTR_ERR(fmi->sftl); ++ return 0; ++} ++ + /* + * Turn the PPN's status byte into what MTD expects back from _read_oob: a + * count of corrected bitflips, or a negative errno. +@@ -641,6 +710,13 @@ static int h2fmi_probe(struct platform_device *pdev) + } + } + ++ ret = h2fmi_register_sftl(fmi); ++ if (ret) { ++ for (i = 0; i < fmi->ndies; i++) ++ mtd_device_unregister(&fmi->mtd[i]); ++ return ret; ++ } ++ + platform_set_drvdata(pdev, fmi); + dev_info(&pdev->dev, "%u CAUs, %u blocks of %u KiB each\n", + fmi->ndies, fmi->bus[0].geometry.blocks, +@@ -653,6 +729,8 @@ static void h2fmi_remove(struct platform_device *pdev) + struct h2fmi *fmi = platform_get_drvdata(pdev); + int i; + ++ if (fmi->sftl) ++ platform_device_unregister(fmi->sftl); + for (i = 0; i < fmi->ndies; i++) + mtd_device_unregister(&fmi->mtd[i]); + } +diff --git a/include/linux/mtd/apple-ppn.h b/include/linux/mtd/apple-ppn.h +new file mode 100644 +index 000000000..78ee66867 +--- /dev/null ++++ b/include/linux/mtd/apple-ppn.h +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _LINUX_MTD_APPLE_PPN_H ++#define _LINUX_MTD_APPLE_PPN_H ++ ++#include ++ ++struct device; ++ ++#define APPLE_PPN_PAGE_SIZE 16384 ++#define APPLE_PPN_OOB_SIZE 64 ++ ++/* All buffers have canonical data/OOB layout, including context reads. */ ++struct apple_ppn_ops { ++ int (*read_page)(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, bool context, ++ u8 *data, u8 *oob, u8 *status); ++}; ++ ++struct apple_ppn { ++ struct device *dev; ++ const struct apple_ppn_ops *ops; ++ void *priv; ++ u16 buses; ++ u16 caus; ++ u16 blocks; ++ u16 pages; ++}; ++ ++#endif diff --git a/kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch.toml b/kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0003-mtd-apple-h2fmi-provide-canonical-page-reads-to-SFTL.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] diff --git a/kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch b/kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch new file mode 100644 index 00000000..bde46080 --- /dev/null +++ b/kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch @@ -0,0 +1,1835 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 11:48:19 -0600 +Subject: [PATCH] mtd: add read-only Apple VFL and SwissPPNFTL recovery + +Recover the measured 64 GB PPN layout from replicated VFL contexts and +lane tables. Decode the saved SFTL ranges and replay the subsequent native +journal, including completed-block summaries and open stripes with holes. +Reject unsupported formats and incomplete sequences before publishing a disk. + +Expose the 64,000,000,000-byte logical medium as a read-only block device. +The map retains sixteen internal keys outside the disk capacity. Requests +use a preallocated buffer, and a failed storage read stops further reads. + +Port the format rules from Orchard's P105 Python reference. Add synthetic +complete-medium tests for checkpoint recovery, journal and trim ordering, +replica disagreement, torn summaries and runtime failure. The fixtures +contain no captured firmware or device bytes. + +Assisted-by: LLM +--- + Documentation/driver-api/mtd/apple-sftl.rst | 61 ++ + Documentation/driver-api/mtd/index.rst | 2 +- + drivers/mtd/Kconfig | 21 + + drivers/mtd/Makefile | 4 + + drivers/mtd/apple-sftl-block.c | 234 +++++ + drivers/mtd/apple-sftl-core.c | 949 ++++++++++++++++++ + drivers/mtd/apple-sftl-test.c | 436 ++++++++ + drivers/mtd/apple-sftl.h | 23 + + tools/testing/kunit/configs/apple_sftl.config | 4 + + 9 files changed, 1733 insertions(+), 1 deletion(-) + create mode 100644 Documentation/driver-api/mtd/apple-sftl.rst + create mode 100644 drivers/mtd/apple-sftl-block.c + create mode 100644 drivers/mtd/apple-sftl-core.c + create mode 100644 drivers/mtd/apple-sftl-test.c + create mode 100644 drivers/mtd/apple-sftl.h + create mode 100644 tools/testing/kunit/configs/apple_sftl.config + +diff --git a/Documentation/driver-api/mtd/apple-sftl.rst b/Documentation/driver-api/mtd/apple-sftl.rst +new file mode 100644 +index 000000000..8295a5a2a +--- /dev/null ++++ b/Documentation/driver-api/mtd/apple-sftl.rst +@@ -0,0 +1,61 @@ ++.. SPDX-License-Identifier: GPL-2.0-only ++ ++Apple PPN VFL/SwissPPNFTL reader ++============================== ++ ++The ``apple-sftl`` driver recovers the logical medium above Apple's PPN flash ++mapping. H2FMI supplies complete 16 KiB pages and their 64 metadata bytes ++through an aggregate provider. Context reads use the package's separate SLC ++address view. The controller serializes each complete transaction. ++ ++Supported format ++---------------- ++ ++This implementation supports the measured 64 GB Hynix PPN profile with two ++buses and eight CAUs per bus. Each CAU has 2,088 blocks of 128 pages. Profile ++selection uses discovered geometry. Other layouts return ``-EOPNOTSUPP``. ++ ++Recovery selects four agreeing VFL checkpoint copies and checks each ++replicated lane table. A VFL superblock has an 8,192-unit address stride; ++units are 4 KiB. Blocks with fewer active lanes have an unused address gap ++which the reader rejects. ++ ++SFTL subtype-4 context fragments describe the saved logical ranges. Recovery ++validates the complete context sequence, then applies later data and trim ++records in sequence order. Completed blocks carry summaries at their ends. ++Open blocks are scanned completely because an interrupted stripe may leave ++an erased lane before another completed lane. Summary allocation spans also ++preserve the next sequence floor when it exceeds the last visible record. ++ ++The map has 15,625,016 entries, including sixteen internal keys. The block ++device exposes 15,625,000 entries, or 64,000,000,000 bytes, with 512-byte sector ++reads. Linux's disk name is ``applesftlN``. LwVM and filesystem interpretation ++happen above this logical provider. Encrypted partition contents remain ++ciphertext. ++ ++Failure and lifetime rules ++-------------------------- ++ ++The disk is read-only and the provider has no programming or erase operation. ++Malformed context copies and incomplete journal sequences prevent disk ++registration. A refresh status requires a writer and is reported as an error. ++A failed runtime read stops subsequent reads from this map instance. ++ ++H2FMI unregisters the child before releasing its controller state. The block ++frontend freezes and drains requests during removal. Its request buffer is ++allocated before the disk becomes visible, while blocking requests serialize ++access to that buffer and the provider. ++ ++Tests ++----- ++ ++``tools/testing/kunit/configs/apple_sftl.config`` enables the synthetic medium ++tests. They generate VFL replicas and lane tables, plus an SFTL checkpoint ++followed by completed and open data blocks. Tests cover trim ordering and ++allocation floors. Corrupt replicas and sequence gaps must return an error ++with no disk or usable map. Test inputs contain no captured device bytes. ++ ++The format rules were ported from Orchard's P105 Python reference ++(``ftl_map.py``, ``ftl_load.py`` and ``ftl_log.py``), which was checked against ++native iOS logical reads. A kernel object compile or synthetic test result ++does not establish physical-controller timing or qualify a new device layout. +diff --git a/Documentation/driver-api/mtd/index.rst b/Documentation/driver-api/mtd/index.rst +index 6a4278f40..e91da2aa5 100644 +--- a/Documentation/driver-api/mtd/index.rst ++++ b/Documentation/driver-api/mtd/index.rst +@@ -9,4 +9,4 @@ Memory Technology Device (MTD) + + spi-intel + nand_ecc +- spi-nor ++ spi-nor apple-sftl +diff --git a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig +index 796a2eccb..155136a36 100644 +--- a/drivers/mtd/Kconfig ++++ b/drivers/mtd/Kconfig +@@ -206,6 +206,27 @@ config MTD_PARTITIONED_MASTER + the parent of the partition device be the master device, rather than + what lies behind the master. + ++config MTD_APPLE_SFTL ++ tristate "Apple PPN read-only VFL/SwissPPNFTL translation" ++ depends on BLOCK ++ help ++ Recover the logical block medium stored in Apple VFL/SwissPPNFTL ++ metadata on the measured 64 GB Hynix PPN layout. The H2FMI driver ++ supplies canonical page reads through its aggregate PPN provider. ++ Unsupported checkpoints, missing journal sequences and refresh ++ requirements stop recovery before a disk is registered. ++ ++ The disk is read-only. This driver performs no program or erase. ++ ++config MTD_APPLE_SFTL_KUNIT_TEST ++ tristate "KUnit tests for Apple VFL/SwissPPNFTL recovery" if !KUNIT_ALL_TESTS ++ depends on KUNIT && MTD_APPLE_SFTL ++ default KUNIT_ALL_TESTS ++ help ++ Synthesize complete VFL/SFTL media and exercise checkpoint recovery, ++ native block summaries and open-block journal replay. Malformed ++ replicas and unsupported layouts must fail without storage writes. ++ + source "drivers/mtd/chips/Kconfig" + + source "drivers/mtd/maps/Kconfig" +diff --git a/drivers/mtd/Makefile b/drivers/mtd/Makefile +index 593d0593a..fa6180a62 100644 +--- a/drivers/mtd/Makefile ++++ b/drivers/mtd/Makefile +@@ -31,3 +31,7 @@ obj-y += chips/ lpddr/ maps/ devices/ nand/ tests/ + obj-$(CONFIG_MTD_SPI_NOR) += spi-nor/ + obj-$(CONFIG_MTD_UBI) += ubi/ + obj-$(CONFIG_MTD_HYPERBUS) += hyperbus/ ++ ++obj-$(CONFIG_MTD_APPLE_SFTL) += apple-sftl.o ++apple-sftl-y := apple-sftl-core.o apple-sftl-block.o ++obj-$(CONFIG_MTD_APPLE_SFTL_KUNIT_TEST) += apple-sftl-test.o +diff --git a/drivers/mtd/apple-sftl-block.c b/drivers/mtd/apple-sftl-block.c +new file mode 100644 +index 000000000..86d02921b +--- /dev/null ++++ b/drivers/mtd/apple-sftl-block.c +@@ -0,0 +1,234 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Read-only block frontend for Apple's VFL/SwissPPNFTL logical medium. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-sftl.h" ++ ++#define SFTL_BLOCK_NAME "apple-sftl" ++#define SFTL_DISK_MINORS 16 ++#define SFTL_LOGICAL_BYTES 4096U ++ ++struct sftl_disk { ++ struct apple_sftl *ftl; ++ struct blk_mq_tag_set tags; ++ struct gendisk *disk; ++ struct mutex io_lock; /* serializes requests using buffer */ ++ u8 *buffer; ++ unsigned int id; ++ bool dead; ++}; ++ ++static DEFINE_IDA(sftl_disk_ids); ++static int sftl_major; ++ ++/* The blocking queue and this lock keep the preallocated buffer exclusive. */ ++static int sftl_read_request(struct sftl_disk *sftl, struct request *request) ++{ ++ struct req_iterator iterator; ++ struct bio_vec bvec; ++ u64 position = (u64)blk_rq_pos(request) << SECTOR_SHIFT; ++ u32 cached = U32_MAX; ++ ++ rq_for_each_segment(bvec, request, iterator) { ++ unsigned int done = 0; ++ ++ while (done < bvec.bv_len) { ++ u32 logical = position >> 12; ++ unsigned int offset = position & (SFTL_LOGICAL_BYTES - 1); ++ unsigned int length = min(bvec.bv_len - done, ++ SFTL_LOGICAL_BYTES - offset); ++ void *mapped; ++ int ret; ++ ++ if (logical != cached) { ++ ret = apple_sftl_read(sftl->ftl, logical, sftl->buffer); ++ if (ret) ++ return ret; ++ cached = logical; ++ } ++ mapped = kmap_local_page(bvec.bv_page); ++ memcpy(mapped + bvec.bv_offset + done, ++ sftl->buffer + offset, length); ++ kunmap_local(mapped); ++ done += length; ++ position += length; ++ } ++ } ++ return 0; ++} ++ ++static blk_status_t sftl_queue_rq(struct blk_mq_hw_ctx *hctx, ++ const struct blk_mq_queue_data *data) ++{ ++ struct request *request = data->rq; ++ struct sftl_disk *sftl = request->q->queuedata; ++ sector_t capacity = get_capacity(sftl->disk); ++ blk_status_t status; ++ int ret = 0; ++ ++ blk_mq_start_request(request); ++ mutex_lock(&sftl->io_lock); ++ if (sftl->dead) { ++ ret = -ENODEV; ++ } else if (req_op(request) == REQ_OP_FLUSH) { ++ /* A read-only device has no buffered writes to flush. */ ++ } else if (req_op(request) != REQ_OP_READ) { ++ ret = -EOPNOTSUPP; ++ } else if (blk_rq_pos(request) > capacity || ++ blk_rq_sectors(request) > capacity - blk_rq_pos(request)) { ++ ret = -EIO; ++ } else { ++ ret = sftl_read_request(sftl, request); ++ } ++ mutex_unlock(&sftl->io_lock); ++ status = errno_to_blk_status(ret); ++ blk_mq_end_request(request, status); ++ return BLK_STS_OK; ++} ++ ++static const struct blk_mq_ops sftl_mq_ops = { ++ .queue_rq = sftl_queue_rq, ++}; ++ ++static const struct block_device_operations sftl_block_ops = { ++ .owner = THIS_MODULE, ++}; ++ ++static int sftl_disk_probe(struct platform_device *pdev) ++{ ++ struct queue_limits limits = { ++ .logical_block_size = SECTOR_SIZE, ++ .physical_block_size = SFTL_LOGICAL_BYTES, ++ .max_hw_sectors = 128, ++ }; ++ struct apple_ppn *provider = dev_get_platdata(&pdev->dev); ++ struct apple_sftl_info info; ++ struct sftl_disk *sftl; ++ struct gendisk *disk; ++ int ret, id; ++ ++ if (!provider) ++ return -EINVAL; ++ sftl = kzalloc(sizeof(*sftl), GFP_KERNEL); ++ if (!sftl) ++ return -ENOMEM; ++ mutex_init(&sftl->io_lock); ++ sftl->buffer = kmalloc(SFTL_LOGICAL_BYTES, GFP_KERNEL); ++ if (!sftl->buffer) { ++ ret = -ENOMEM; ++ goto free_state; ++ } ++ ret = apple_sftl_load(provider, &sftl->ftl); ++ if (ret) ++ goto free_buffer; ++ apple_sftl_get_info(sftl->ftl, &info); ++ if (!info.disk_pages || info.disk_pages > info.logical_pages) { ++ ret = -EINVAL; ++ goto free_ftl; ++ } ++ id = ida_alloc_range(&sftl_disk_ids, 0, MINORMASK / SFTL_DISK_MINORS, ++ GFP_KERNEL); ++ if (id < 0) { ++ ret = id; ++ goto free_ftl; ++ } ++ sftl->id = id; ++ ret = blk_mq_alloc_sq_tag_set(&sftl->tags, &sftl_mq_ops, 8, ++ BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING); ++ if (ret) ++ goto free_id; ++ disk = blk_mq_alloc_disk(&sftl->tags, &limits, sftl); ++ if (IS_ERR(disk)) { ++ ret = PTR_ERR(disk); ++ goto free_tags; ++ } ++ sftl->disk = disk; ++ disk->major = sftl_major; ++ disk->first_minor = id * SFTL_DISK_MINORS; ++ disk->minors = SFTL_DISK_MINORS; ++ disk->fops = &sftl_block_ops; ++ disk->private_data = sftl; ++ snprintf(disk->disk_name, DISK_NAME_LEN, "applesftl%d", id); ++ set_capacity(disk, (sector_t)info.disk_pages * (SFTL_LOGICAL_BYTES / SECTOR_SIZE)); ++ set_disk_ro(disk, true); ++ platform_set_drvdata(pdev, sftl); ++ ret = device_add_disk(&pdev->dev, disk, NULL); ++ if (ret) ++ goto put_disk; ++ dev_info(&pdev->dev, "%s: read-only, %u logical pages, sequence %llu\n", ++ disk->disk_name, info.disk_pages, info.sequence); ++ return 0; ++ ++put_disk: ++ platform_set_drvdata(pdev, NULL); ++ put_disk(disk); ++free_tags: ++ blk_mq_free_tag_set(&sftl->tags); ++free_id: ++ ida_free(&sftl_disk_ids, sftl->id); ++free_ftl: ++ apple_sftl_free(sftl->ftl); ++free_buffer: ++ kfree(sftl->buffer); ++free_state: ++ kfree(sftl); ++ return ret; ++} ++ ++static void sftl_disk_remove(struct platform_device *pdev) ++{ ++ struct sftl_disk *sftl = platform_get_drvdata(pdev); ++ ++ mutex_lock(&sftl->io_lock); ++ sftl->dead = true; ++ mutex_unlock(&sftl->io_lock); ++ /* del_gendisk freezes and drains the queue before the provider is freed. */ ++ del_gendisk(sftl->disk); ++ put_disk(sftl->disk); ++ blk_mq_free_tag_set(&sftl->tags); ++ ida_free(&sftl_disk_ids, sftl->id); ++ apple_sftl_free(sftl->ftl); ++ kfree(sftl->buffer); ++ kfree(sftl); ++} ++ ++static struct platform_driver sftl_driver = { ++ .probe = sftl_disk_probe, ++ .remove = sftl_disk_remove, ++ .driver = { ++ .name = SFTL_BLOCK_NAME, ++ }, ++}; ++ ++static int __init sftl_init(void) ++{ ++ int ret; ++ ++ sftl_major = register_blkdev(0, SFTL_BLOCK_NAME); ++ if (sftl_major < 0) ++ return sftl_major; ++ ret = platform_driver_register(&sftl_driver); ++ if (ret) ++ unregister_blkdev(sftl_major, SFTL_BLOCK_NAME); ++ return ret; ++} ++module_init(sftl_init); ++ ++static void __exit sftl_exit(void) ++{ ++ platform_driver_unregister(&sftl_driver); ++ unregister_blkdev(sftl_major, SFTL_BLOCK_NAME); ++ ida_destroy(&sftl_disk_ids); ++} ++module_exit(sftl_exit); ++ ++MODULE_DESCRIPTION("Apple VFL/SwissPPNFTL read-only block driver"); ++MODULE_LICENSE("GPL"); ++MODULE_ALIAS("platform:" SFTL_BLOCK_NAME); +diff --git a/drivers/mtd/apple-sftl-core.c b/drivers/mtd/apple-sftl-core.c +new file mode 100644 +index 000000000..ba90916f2 +--- /dev/null ++++ b/drivers/mtd/apple-sftl-core.c +@@ -0,0 +1,949 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Apple VFL/SwissPPNFTL read-only recovery for the measured 64 GB PPN layout. ++ * ++ * Format and replay rules are ported from Orchard's P105 Python reference ++ * (ftl_map.py, ftl_load.py and ftl_log.py). That implementation was checked ++ * against native iOS logical reads. This code contains no captured firmware ++ * or device data. Unsupported formats fail before a block device is created. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-sftl.h" ++ ++#define SFTL_LPAGES 15625016U ++#define SFTL_DISK_PAGES 15625000U ++#define SFTL_BLOCKS 2088U ++#define SFTL_SUBBLOCKS 2086U ++#define SFTL_LANES 16U ++#define SFTL_STRIDE 8192U ++#define SFTL_PROVIDER (SFTL_SUBBLOCKS * SFTL_STRIDE) ++#define SFTL_UNMAPPED U32_MAX ++#define SFTL_SEQUENCE GENMASK_ULL(47, 0) ++#define SFTL_MAX_CONTEXTS 16U ++#define SFTL_META 16U ++#define SFTL_CHUNK 4096U ++#define SFTL_TRIM 0xffff0001U ++ ++struct sftl_run { ++ struct rb_node node; ++ u32 start, end, provider; ++}; ++ ++/* stamp packs the native 48-bit sequence with its flags and record type. */ ++struct sftl_record { ++ u64 stamp; ++ u32 key, address; ++}; ++ ++struct sftl_records { ++ struct sftl_record *items; ++ u32 count, capacity; ++}; ++ ++struct sftl_member { ++ u16 block; ++ u8 lane; ++}; ++ ++struct apple_sftl { ++ struct apple_ppn *ppn; ++ struct rb_root runs; ++ u32 nr_runs; ++ u16 lanes[SFTL_BLOCKS]; ++ DECLARE_BITMAP(reserved, SFTL_BLOCKS); ++ u64 checkpoint_sequence, sequence; ++ u8 *page; ++ u8 oob[APPLE_PPN_OOB_SIZE]; ++ struct mutex lock; /* serializes reads and the shared page buffer */ ++ bool failed; ++}; ++ ++static u64 sftl_sequence(const u8 *meta) ++{ ++ return get_unaligned_le64(meta) >> 16; ++} ++ ++static u64 sftl_stamp(const u8 *meta) ++{ ++ return sftl_sequence(meta) | (u64)meta[1] << 48 | (u64)meta[0] << 56; ++} ++ ++static unsigned int sftl_kind(const struct sftl_record *record) ++{ ++ return record->stamp >> 56; ++} ++ ++static unsigned int sftl_flags(const struct sftl_record *record) ++{ ++ return (record->stamp >> 48) & 0xff; ++} ++ ++static bool sftl_erased(const void *data, size_t size) ++{ ++ return !memchr_inv(data, 0xff, size); ++} ++ ++static int sftl_read_page(struct apple_sftl *sftl, unsigned int bus, ++ unsigned int cau, unsigned int block, ++ unsigned int page, bool context, u8 *data, u8 *oob) ++{ ++ u8 status; ++ int ret; ++ ++ ret = sftl->ppn->ops->read_page(sftl->ppn->priv, bus, cau, block, ++ page, context, data, oob, &status); ++ if (ret) ++ return ret; ++ /* Refresh requires a writer. A reader must not hide that requirement. */ ++ if (status != 0x40 && status != 0x49) ++ return -EIO; ++ if (status == 0x49 && (!sftl_erased(data, APPLE_PPN_PAGE_SIZE) || ++ !sftl_erased(oob, APPLE_PPN_OOB_SIZE))) ++ return -EBADMSG; ++ return 0; ++} ++ ++static int sftl_translate(struct apple_sftl *sftl, u32 address, ++ unsigned int *bus, unsigned int *cau, ++ unsigned int *block, unsigned int *page, ++ unsigned int *chunk) ++{ ++ u32 offset; ++ unsigned int lane, wanted, count; ++ u16 bitmap; ++ ++ if (address >= SFTL_PROVIDER) ++ return -ERANGE; ++ *chunk = address & 3; ++ *block = address / SFTL_STRIDE; ++ offset = (address % SFTL_STRIDE) / 4; ++ bitmap = sftl->lanes[*block]; ++ count = hweight16(bitmap); ++ if (!count || offset / count >= 128) ++ return -ERANGE; ++ *page = offset / count; ++ wanted = offset % count; ++ for (lane = 0; lane < SFTL_LANES; lane++) { ++ if (!(bitmap & BIT(lane))) ++ continue; ++ if (!wanted--) ++ break; ++ } ++ *bus = lane & 1; ++ *cau = lane >> 1; ++ return 0; ++} ++ ++static int sftl_read_address(struct apple_sftl *sftl, u32 address, ++ unsigned int *chunk) ++{ ++ unsigned int bus, cau, block, page; ++ int ret; ++ ++ ret = sftl_translate(sftl, address, &bus, &cau, &block, &page, chunk); ++ if (ret) ++ return ret; ++ return sftl_read_page(sftl, bus, cau, block, page, false, ++ sftl->page, sftl->oob); ++} ++ ++static struct sftl_run *sftl_find(struct apple_sftl *sftl, u32 logical) ++{ ++ struct rb_node *node = sftl->runs.rb_node; ++ ++ while (node) { ++ struct sftl_run *run = rb_entry(node, struct sftl_run, node); ++ ++ if (logical < run->start) ++ node = node->rb_left; ++ else if (logical >= run->end) ++ node = node->rb_right; ++ else ++ return run; ++ } ++ return NULL; ++} ++ ++static void sftl_insert(struct apple_sftl *sftl, struct sftl_run *run) ++{ ++ struct rb_node **link = &sftl->runs.rb_node, *parent = NULL; ++ ++ while (*link) { ++ struct sftl_run *entry = rb_entry(*link, struct sftl_run, node); ++ ++ parent = *link; ++ link = run->start < entry->start ? &parent->rb_left : ++ &parent->rb_right; ++ } ++ rb_link_node(&run->node, parent, link); ++ rb_insert_color(&run->node, &sftl->runs); ++ sftl->nr_runs++; ++} ++ ++static int sftl_add_run(struct apple_sftl *sftl, u32 start, u32 end, u32 provider) ++{ ++ struct sftl_run *run; ++ ++ if (start >= end || end > SFTL_LPAGES || ++ (provider != SFTL_UNMAPPED && ++ (provider >= SFTL_PROVIDER || end - start > SFTL_PROVIDER - provider))) ++ return -EBADMSG; ++ run = kmalloc(sizeof(*run), GFP_KERNEL); ++ if (!run) ++ return -ENOMEM; ++ run->start = start; ++ run->end = end; ++ run->provider = provider; ++ sftl_insert(sftl, run); ++ return 0; ++} ++ ++static int sftl_split(struct apple_sftl *sftl, u32 point) ++{ ++ struct sftl_run *left = sftl_find(sftl, point), *right; ++ ++ if (!left) ++ return -EBADMSG; ++ if (left->start == point) ++ return 0; ++ right = kmalloc(sizeof(*right), GFP_KERNEL); ++ if (!right) ++ return -ENOMEM; ++ right->start = point; ++ right->end = left->end; ++ right->provider = left->provider == SFTL_UNMAPPED ? SFTL_UNMAPPED : ++ left->provider + point - left->start; ++ left->end = point; ++ sftl_insert(sftl, right); ++ return 0; ++} ++ ++static bool sftl_contiguous(struct sftl_run *left, struct sftl_run *right) ++{ ++ return left->end == right->start && ++ ((left->provider == SFTL_UNMAPPED && right->provider == SFTL_UNMAPPED) || ++ (left->provider != SFTL_UNMAPPED && right->provider != SFTL_UNMAPPED && ++ left->provider + left->end - left->start == right->provider)); ++} ++ ++static void sftl_merge(struct apple_sftl *sftl, struct sftl_run *run) ++{ ++ struct rb_node *node = rb_prev(&run->node); ++ struct sftl_run *neighbor; ++ ++ if (node) { ++ neighbor = rb_entry(node, struct sftl_run, node); ++ if (sftl_contiguous(neighbor, run)) { ++ neighbor->end = run->end; ++ rb_erase(&run->node, &sftl->runs); ++ kfree(run); ++ sftl->nr_runs--; ++ run = neighbor; ++ } ++ } ++ node = rb_next(&run->node); ++ if (!node) ++ return; ++ neighbor = rb_entry(node, struct sftl_run, node); ++ if (sftl_contiguous(run, neighbor)) { ++ run->end = neighbor->end; ++ rb_erase(node, &sftl->runs); ++ kfree(neighbor); ++ sftl->nr_runs--; ++ } ++} ++ ++static int sftl_replace(struct apple_sftl *sftl, u32 start, u32 length, u32 address) ++{ ++ struct sftl_run *run, *new; ++ struct rb_node *node, *next; ++ u32 end; ++ int ret; ++ ++ if (!length || start >= SFTL_LPAGES || length > SFTL_LPAGES - start || ++ (address != SFTL_UNMAPPED && ++ (address >= SFTL_PROVIDER || length > SFTL_PROVIDER - address))) ++ return -EBADMSG; ++ end = start + length; ++ ret = sftl_split(sftl, start); ++ if (ret) ++ return ret; ++ if (end < SFTL_LPAGES) { ++ ret = sftl_split(sftl, end); ++ if (ret) ++ return ret; ++ } ++ new = kmalloc(sizeof(*new), GFP_KERNEL); ++ if (!new) ++ return -ENOMEM; ++ run = sftl_find(sftl, start); ++ for (node = &run->node; node; node = next) { ++ run = rb_entry(node, struct sftl_run, node); ++ if (run->start >= end) ++ break; ++ next = rb_next(node); ++ rb_erase(node, &sftl->runs); ++ kfree(run); ++ sftl->nr_runs--; ++ } ++ new->start = start; ++ new->end = end; ++ new->provider = address; ++ sftl_insert(sftl, new); ++ sftl_merge(sftl, new); ++ return 0; ++} ++ ++static int sftl_record_add(struct sftl_records *records, u64 stamp, ++ u32 key, u32 address) ++{ ++ struct sftl_record *new; ++ u32 capacity; ++ ++ if (records->count == SFTL_PROVIDER) ++ return -E2BIG; ++ if (records->count == records->capacity) { ++ capacity = min_t(u32, max(records->capacity * 2, 256U), SFTL_PROVIDER); ++ new = kvrealloc(records->items, array_size(capacity, sizeof(*new)), ++ GFP_KERNEL); ++ if (!new) ++ return -ENOMEM; ++ records->items = new; ++ records->capacity = capacity; ++ } ++ records->items[records->count++] = (struct sftl_record) { ++ .stamp = stamp, .key = key, .address = address, ++ }; ++ return 0; ++} ++ ++static int sftl_record_compare(const void *a, const void *b) ++{ ++ u64 left = ((const struct sftl_record *)a)->stamp & SFTL_SEQUENCE; ++ u64 right = ((const struct sftl_record *)b)->stamp & SFTL_SEQUENCE; ++ ++ return left < right ? -1 : left > right; ++} ++ ++static int sftl_vfl(struct apple_sftl *sftl) ++{ ++ struct sftl_member members[8]; ++ u8 list[32], best_oob[64], best_members = 0; ++ u8 *best, *table; ++ u32 best_age = U32_MAX; ++ unsigned int member, page, latest_page = 0, lane, block; ++ int ret; ++ ++ best = kmalloc(APPLE_PPN_PAGE_SIZE, GFP_KERNEL); ++ table = kmalloc(APPLE_PPN_PAGE_SIZE + APPLE_PPN_OOB_SIZE, GFP_KERNEL); ++ if (!best || !table) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ret = sftl_read_page(sftl, 0, 0, 2087, 0, true, sftl->page, sftl->oob); ++ if (ret) ++ goto out; ++ if (get_unaligned_le32(sftl->oob) != 0x00ffff20) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ memcpy(list, sftl->page + 0x100, sizeof(list)); ++ for (member = 0; member < 8; member++) { ++ unsigned int other; ++ ++ members[member].block = get_unaligned_le16(list + member * 4); ++ members[member].lane = list[member * 4 + 2]; ++ if (members[member].block < 2056 || members[member].block >= SFTL_BLOCKS || ++ members[member].lane >= SFTL_LANES || list[member * 4 + 3]) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ for (other = 0; other < member; other++) ++ if (members[other].block == members[member].block && ++ members[other].lane == members[member].lane) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ } ++ for (member = 0; member < 8; member++) { ++ for (page = 0; page < 64; page++) { ++ u32 age; ++ ++ ret = sftl_read_page(sftl, members[member].lane & 1, ++ members[member].lane >> 1, members[member].block, ++ page, true, sftl->page, sftl->oob); ++ if (ret) ++ goto out; ++ if (!page && memcmp(sftl->page + 0x100, list, sizeof(list))) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ if (get_unaligned_le32(sftl->oob) != 0x00ffff20) ++ continue; ++ age = get_unaligned_le32(sftl->oob + 4); ++ if (age < BIT(31)) { ++ ret = -EOPNOTSUPP; ++ goto out; ++ } ++ if (age > best_age) ++ continue; ++ if (!best_members || age < best_age) { ++ best_age = age; ++ latest_page = page; ++ memcpy(best, sftl->page, APPLE_PPN_PAGE_SIZE); ++ memcpy(best_oob, sftl->oob, sizeof(best_oob)); ++ best_members = BIT(member); ++ } else { ++ if (page != latest_page || best_members & BIT(member) || ++ memcmp(best, sftl->page, APPLE_PPN_PAGE_SIZE) || ++ memcmp(best_oob, sftl->oob, sizeof(best_oob))) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ best_members |= BIT(member); ++ } ++ } ++ } ++ if ((best_members != 0x0f && best_members != 0xf0) || ++ get_unaligned_le16(best + 10) != 16 || ++ get_unaligned_le16(best + 12) != SFTL_SUBBLOCKS || ++ get_unaligned_le16(best + 14) != SFTL_SUBBLOCKS) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ for (lane = 0; lane < SFTL_LANES; lane++) { ++ u16 reference = get_unaligned_le16(best + 0x200 + lane * 32); ++ unsigned int group = reference >> 15; ++ ++ page = reference & 0x7fff; ++ if (page >= 64) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ for (member = group * 4; member < group * 4 + 4; member++) { ++ ret = sftl_read_page(sftl, members[member].lane & 1, ++ members[member].lane >> 1, members[member].block, ++ page, true, sftl->page, sftl->oob); ++ if (ret) ++ goto out; ++ if (sftl->oob[0] != 0x20 || sftl->oob[1] != lane || ++ sftl->oob[2] || sftl->oob[3]) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ if (member == group * 4) { ++ memcpy(table, sftl->page, APPLE_PPN_PAGE_SIZE); ++ memcpy(table + APPLE_PPN_PAGE_SIZE, sftl->oob, sizeof(sftl->oob)); ++ } else if (memcmp(table, sftl->page, APPLE_PPN_PAGE_SIZE) || ++ memcmp(table + APPLE_PPN_PAGE_SIZE, ++ sftl->oob, sizeof(sftl->oob))) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ } ++ for (block = 0; block < SFTL_BLOCKS; block++) ++ if (!(get_unaligned_le16(table + block * 2) & BIT(15))) ++ sftl->lanes[block] |= BIT(lane); ++ } ++ ret = 0; ++out: ++ kfree(table); ++ kfree(best); ++ return ret; ++} ++ ++static int sftl_context_names(struct apple_sftl *sftl, const u8 *payload, ++ u16 *pending, u32 *count) ++{ ++ u32 names = get_unaligned_le32(payload), i; ++ ++ if (!names || names > SFTL_MAX_CONTEXTS) ++ return -EBADMSG; ++ for (i = 0; i < names; i++) { ++ u32 name = get_unaligned_le32(payload + 4 + i * 4); ++ ++ if (!name || name >= SFTL_SUBBLOCKS) ++ return -EBADMSG; ++ if (test_bit(name, sftl->reserved)) ++ continue; ++ if (*count == SFTL_MAX_CONTEXTS) ++ return -E2BIG; ++ __set_bit(name, sftl->reserved); ++ pending[(*count)++] = name; ++ } ++ return 0; ++} ++ ++static int sftl_context(struct apple_sftl *sftl, const u8 *heads) ++{ ++ struct sftl_records records = {}; ++ u16 pending[SFTL_MAX_CONTEXTS]; ++ u32 count = 0, block, at, i, first = 0, cursor = 0, chunk; ++ u64 previous = 0, fragment_sequence = 0; ++ int ret = -EBADMSG; ++ ++ for (block = 1; block < SFTL_SUBBLOCKS; block++) { ++ if (heads[block * SFTL_META] != 0x1f) ++ continue; ++ if (count == SFTL_MAX_CONTEXTS) ++ goto out; ++ pending[count++] = block; ++ __set_bit(block, sftl->reserved); ++ } ++ for (at = 0; at < count; at++) { ++ u32 position, pages; ++ ++ block = pending[at]; ++ pages = hweight16(sftl->lanes[block]) * 128; ++ if (!pages) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ for (position = 0; position < pages; position++) { ++ ret = sftl_read_address(sftl, block * SFTL_STRIDE + position * 4, &chunk); ++ if (ret) ++ goto out; ++ if (sftl_erased(sftl->oob, sizeof(sftl->oob))) { ++ ret = sftl_erased(sftl->page, APPLE_PPN_PAGE_SIZE) ? 0 : -EBADMSG; ++ if (ret) ++ goto out; ++ break; ++ } ++ for (chunk = 0; chunk < 4; chunk++) { ++ const u8 *meta = sftl->oob + chunk * SFTL_META; ++ const u8 *payload = sftl->page + chunk * SFTL_CHUNK; ++ ++ if (sftl_erased(meta, SFTL_META)) ++ continue; ++ if (meta[0] != 0x1f || !meta[1] || meta[1] > 6) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ if (meta[1] == 1 && !chunk) { ++ ret = sftl_context_names(sftl, payload, pending, &count); ++ if (ret) ++ goto out; ++ } ++ ret = sftl_record_add(&records, sftl_stamp(meta), 0, ++ block * SFTL_STRIDE + position * 4 + chunk); ++ if (ret) ++ goto out; ++ } ++ } ++ } ++ ret = -EBADMSG; ++ if (!records.count) ++ goto out; ++ sort(records.items, records.count, sizeof(*records.items), sftl_record_compare, NULL); ++ if (sftl_flags(&records.items[0]) != 1) ++ goto out; ++ for (i = 1; i < records.count; i++) ++ if (sftl_flags(&records.items[i]) == 1 && sftl_flags(&records.items[i - 1]) != 1) ++ first = i; ++ if (sftl_flags(&records.items[records.count - 1]) != 4) ++ goto out; ++ for (i = first; i < records.count; i++) { ++ const struct sftl_record *record = &records.items[i]; ++ u64 sequence = record->stamp & SFTL_SEQUENCE; ++ const u8 *payload; ++ u32 offset, value, length, names; ++ ++ if (i != first && sequence != previous + 1) ++ goto out; ++ previous = sequence; ++ ret = sftl_read_address(sftl, record->address, &chunk); ++ if (ret) ++ goto out; ++ ret = -EBADMSG; ++ if (sftl_stamp(sftl->oob + chunk * SFTL_META) != record->stamp) ++ goto out; ++ payload = sftl->page + chunk * SFTL_CHUNK; ++ if (i == first) { ++ names = get_unaligned_le32(payload); ++ if (!names || names > SFTL_MAX_CONTEXTS) ++ goto out; ++ for (offset = 0; offset < names; offset++) { ++ value = get_unaligned_le32(payload + 4 + offset * 4); ++ if (value >= SFTL_BLOCKS || !test_bit(value, sftl->reserved)) ++ goto out; ++ } ++ } ++ if (sftl_flags(record) != 4) ++ continue; ++ if (fragment_sequence && sequence != fragment_sequence + 1) ++ goto out; ++ fragment_sequence = sequence; ++ value = get_unaligned_le32(payload + 4); ++ if (value == U32_MAX) ++ continue; ++ if (value != 0xfffffff0 || get_unaligned_le32(payload) != cursor) ++ goto out; ++ for (offset = 8; offset < SFTL_CHUNK; offset += 8) { ++ value = get_unaligned_le32(payload + offset); ++ length = get_unaligned_le32(payload + offset + 4); ++ if (value == U32_MAX) ++ break; ++ if (!length || length > SFTL_LPAGES - cursor) ++ goto out; ++ if (value == BIT(fls(SFTL_PROVIDER * 2 - 1) - 1) - 1) ++ value = SFTL_UNMAPPED; ++ ret = sftl_add_run(sftl, cursor, cursor + length, value); ++ if (ret) ++ goto out; ++ cursor += length; ++ ret = -EBADMSG; ++ } ++ } ++ if (cursor != SFTL_LPAGES || !fragment_sequence) ++ goto out; ++ sftl->checkpoint_sequence = fragment_sequence; ++ sftl->sequence = fragment_sequence; ++ ret = 0; ++out: ++ kvfree(records.items); ++ return ret; ++} ++ ++static int sftl_add_journal(struct sftl_records *journal, const u8 *meta, ++ u32 address, u64 checkpoint) ++{ ++ u64 sequence = sftl_sequence(meta); ++ ++ if (sequence <= checkpoint || sftl_erased(meta, SFTL_META)) ++ return 0; ++ if ((meta[0] != 1 && meta[0] != 2) || ++ get_unaligned_le32(meta + 12) != sequence >> 16 || ++ (meta[0] == 1 ? meta[1] > 1 : meta[1] != 0)) ++ return -EBADMSG; ++ return sftl_record_add(journal, sftl_stamp(meta), get_unaligned_le32(meta + 8), address); ++} ++ ++static int sftl_footer(struct apple_sftl *sftl, const u8 *head, u32 block, ++ u32 pages, struct sftl_records *journal, u64 *floor) ++{ ++ u8 footer_meta[16]; ++ u8 *footer; ++ u32 chunks, span, index, position = 0, relative_end = 0; ++ u64 base; ++ unsigned int chunk; ++ bool terminated = false; ++ int ret; ++ ++ memcpy(footer_meta, sftl->oob, sizeof(footer_meta)); ++ base = sftl_sequence(footer_meta); ++ chunks = get_unaligned_le16(footer_meta + 12); ++ span = get_unaligned_le32(footer_meta + 8); ++ if (chunks < 4 || chunks > 32 || chunks % 4 || footer_meta[1] >> 1 != chunks || ++ get_unaligned_le16(footer_meta + 14) != 0xffff || ++ base != sftl_sequence(head) || !span || base + span > SFTL_SEQUENCE) ++ return -EBADMSG; ++ *floor = max(*floor, base + span - 1); ++ if (base + span - 1 <= sftl->checkpoint_sequence) ++ return 0; ++ footer = kmalloc(chunks * SFTL_CHUNK, GFP_KERNEL); ++ if (!footer) ++ return -ENOMEM; ++ for (index = 0; index < chunks / 4; index++) { ++ ret = sftl_read_address(sftl, block * SFTL_STRIDE + ++ (pages - chunks / 4 + index) * 4, &chunk); ++ if (ret) ++ goto out; ++ for (chunk = 0; chunk < 4; chunk++) ++ if (memcmp(footer_meta, sftl->oob + chunk * SFTL_META, SFTL_META)) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ memcpy(footer + index * APPLE_PPN_PAGE_SIZE, sftl->page, APPLE_PPN_PAGE_SIZE); ++ } ++ for (index = 0; index < chunks * SFTL_CHUNK; index += 16) { ++ u32 relative = get_unaligned_le32(footer + index); ++ u32 key = get_unaligned_le32(footer + index + 8); ++ u32 length = get_unaligned_le32(footer + index + 12), offset; ++ ++ if (!length) { ++ terminated = true; ++ break; ++ } ++ if (relative < relative_end || length > U32_MAX - relative || ++ base + relative + length > SFTL_SEQUENCE + 1 || ++ length > pages * 4 - chunks - position) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ relative_end = relative + length; ++ for (offset = 0; offset < length; offset++, position++) { ++ u64 sequence = base + relative + offset; ++ u32 actual_key = key >= 0xffff0000 ? key : key + offset; ++ u32 address = block * SFTL_STRIDE + position; ++ u8 meta[16]; ++ ++ if (sequence <= sftl->checkpoint_sequence) ++ continue; ++ if (key < 0xffff0000 && actual_key < key) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ if (actual_key == SFTL_TRIM) { ++ ret = sftl_read_address(sftl, address, &chunk); ++ if (ret) ++ goto out; ++ memcpy(meta, sftl->oob + chunk * SFTL_META, sizeof(meta)); ++ if (meta[0] != 2 || meta[1] || sftl_sequence(meta) != sequence || ++ get_unaligned_le32(meta + 8) != SFTL_TRIM) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ } else { ++ meta[0] = 1; ++ meta[1] = head[1]; ++ put_unaligned_le64(sequence << 16 | get_unaligned_le16(meta), meta); ++ put_unaligned_le32(actual_key, meta + 8); ++ put_unaligned_le32(sequence >> 16, meta + 12); ++ } ++ ret = sftl_add_journal(journal, meta, address, sftl->checkpoint_sequence); ++ if (ret) ++ goto out; ++ } ++ } ++ ret = terminated && position == pages * 4 - chunks ? 0 : -EBADMSG; ++out: ++ kfree(footer); ++ return ret; ++} ++ ++static int sftl_journal(struct apple_sftl *sftl, const u8 *heads) ++{ ++ struct sftl_records journal = {}; ++ u64 floor = sftl->checkpoint_sequence, expected = floor + 1; ++ u32 block, position, i, pages; ++ unsigned int chunk; ++ int ret; ++ ++ for (block = 1; block < SFTL_SUBBLOCKS; block++) { ++ const u8 *head = heads + block * SFTL_META; ++ ++ if (head[0] != 1 && head[0] != 2) ++ continue; ++ pages = hweight16(sftl->lanes[block]) * 128; ++ ret = sftl_read_address(sftl, block * SFTL_STRIDE + (pages - 1) * 4, &chunk); ++ if (ret) ++ goto out; ++ if (sftl->oob[0] == 0x1c) { ++ ret = sftl_footer(sftl, head, block, pages, &journal, &floor); ++ if (ret) ++ goto out; ++ continue; ++ } ++ if (!sftl_erased(sftl->oob, sizeof(sftl->oob)) || ++ !sftl_erased(sftl->page, APPLE_PPN_PAGE_SIZE)) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ for (position = 0; position < pages; position++) { ++ ret = sftl_read_address(sftl, block * SFTL_STRIDE + position * 4, &chunk); ++ if (ret) ++ goto out; ++ if (sftl_erased(sftl->oob, sizeof(sftl->oob)) && ++ !sftl_erased(sftl->page, APPLE_PPN_PAGE_SIZE)) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ for (chunk = 0; chunk < 4; chunk++) { ++ ret = sftl_add_journal(&journal, sftl->oob + chunk * SFTL_META, ++ block * SFTL_STRIDE + position * 4 + chunk, ++ sftl->checkpoint_sequence); ++ if (ret) ++ goto out; ++ } ++ } ++ } ++ sort(journal.items, journal.count, sizeof(*journal.items), sftl_record_compare, NULL); ++ for (i = 0; i < journal.count; i++) { ++ const struct sftl_record *record = &journal.items[i]; ++ u64 sequence = record->stamp & SFTL_SEQUENCE; ++ ++ ret = -EBADMSG; ++ if (sequence != expected++) ++ goto out; ++ if (sftl_kind(record) == 2 && record->key == SFTL_TRIM) { ++ const u8 *payload; ++ u32 count, offset; ++ ++ ret = sftl_read_address(sftl, record->address, &chunk); ++ if (ret) ++ goto out; ++ ret = -EBADMSG; ++ if (sftl_stamp(sftl->oob + chunk * SFTL_META) != record->stamp || ++ get_unaligned_le32(sftl->oob + chunk * SFTL_META + 8) != record->key) ++ goto out; ++ payload = sftl->page + chunk * SFTL_CHUNK; ++ count = get_unaligned_le32(payload); ++ if (count > (SFTL_CHUNK - 4) / 8) ++ goto out; ++ for (offset = 0; offset < count; offset++) { ++ u32 start = get_unaligned_le32(payload + 4 + offset * 8); ++ u32 length = get_unaligned_le32(payload + 8 + offset * 8); ++ ++ if (!length) ++ continue; ++ ret = sftl_replace(sftl, start, length, SFTL_UNMAPPED); ++ if (ret) ++ goto out; ++ } ++ } else { ++ ret = sftl_replace(sftl, record->key, 1, record->address); ++ if (ret) ++ goto out; ++ } ++ sftl->sequence = sequence; ++ } ++ sftl->sequence = max(sftl->sequence, floor); ++ ret = 0; ++out: ++ kvfree(journal.items); ++ return ret; ++} ++ ++int apple_sftl_load(struct apple_ppn *provider, struct apple_sftl **out) ++{ ++ struct apple_sftl *sftl; ++ u8 *heads; ++ u32 block; ++ unsigned int chunk; ++ int ret; ++ ++ if (!out) ++ return -EINVAL; ++ *out = NULL; ++ if (!provider || !provider->ops || !provider->ops->read_page || ++ provider->buses != 2 || provider->caus != 8 || ++ provider->blocks != SFTL_BLOCKS || provider->pages != 128) ++ return -EOPNOTSUPP; ++ sftl = kzalloc(sizeof(*sftl), GFP_KERNEL); ++ if (!sftl) ++ return -ENOMEM; ++ sftl->ppn = provider; ++ sftl->runs = RB_ROOT; ++ mutex_init(&sftl->lock); ++ sftl->page = kmalloc(APPLE_PPN_PAGE_SIZE, GFP_KERNEL); ++ heads = kcalloc(SFTL_BLOCKS, SFTL_META, GFP_KERNEL); ++ if (!sftl->page || !heads) { ++ ret = -ENOMEM; ++ goto failed; ++ } ++ memset(heads, 0xff, SFTL_BLOCKS * SFTL_META); ++ ret = sftl_vfl(sftl); ++ if (ret) ++ goto failed; ++ for (block = 1; block < SFTL_SUBBLOCKS; block++) { ++ if (!sftl->lanes[block]) ++ continue; ++ ret = sftl_read_address(sftl, block * SFTL_STRIDE, &chunk); ++ if (ret) ++ goto failed; ++ if (sftl_erased(sftl->oob, sizeof(sftl->oob))) { ++ if (!sftl_erased(sftl->page, APPLE_PPN_PAGE_SIZE)) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ } else if (sftl->oob[0] == 0x1f || sftl->oob[0] == 1 || sftl->oob[0] == 2) { ++ memcpy(heads + block * SFTL_META, sftl->oob, SFTL_META); ++ } else { ++ ret = -EOPNOTSUPP; ++ goto failed; ++ } ++ } ++ ret = sftl_context(sftl, heads); ++ if (ret) ++ goto failed; ++ ret = sftl_journal(sftl, heads); ++ if (ret) ++ goto failed; ++ kfree(heads); ++ *out = sftl; ++ return 0; ++failed: ++ kfree(heads); ++ apple_sftl_free(sftl); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_load); ++ ++void apple_sftl_free(struct apple_sftl *sftl) ++{ ++ struct rb_node *node; ++ ++ if (!sftl) ++ return; ++ while ((node = rb_first(&sftl->runs))) { ++ rb_erase(node, &sftl->runs); ++ kfree(rb_entry(node, struct sftl_run, node)); ++ } ++ kfree(sftl->page); ++ kfree(sftl); ++} ++EXPORT_SYMBOL_GPL(apple_sftl_free); ++ ++int apple_sftl_read(struct apple_sftl *sftl, u32 logical_page, u8 *data) ++{ ++ struct sftl_run *run; ++ unsigned int chunk; ++ u32 address; ++ const u8 *meta; ++ int ret = 0; ++ ++ if (!sftl || !data || logical_page >= SFTL_LPAGES) ++ return -EINVAL; ++ mutex_lock(&sftl->lock); ++ if (sftl->failed) { ++ ret = -EIO; ++ goto out; ++ } ++ run = sftl_find(sftl, logical_page); ++ if (!run) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ if (run->provider == SFTL_UNMAPPED) { ++ memset(data, 0, SFTL_CHUNK); ++ goto out; ++ } ++ address = run->provider + logical_page - run->start; ++ ret = sftl_read_address(sftl, address, &chunk); ++ if (ret) ++ goto failed; ++ meta = sftl->oob + chunk * SFTL_META; ++ if ((meta[0] != 1 && meta[0] != 2) || get_unaligned_le32(meta + 8) != logical_page) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ memcpy(data, sftl->page + chunk * SFTL_CHUNK, SFTL_CHUNK); ++ goto out; ++failed: ++ sftl->failed = true; ++out: ++ mutex_unlock(&sftl->lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_read); ++ ++void apple_sftl_get_info(struct apple_sftl *sftl, struct apple_sftl_info *info) ++{ ++ info->logical_pages = SFTL_LPAGES; ++ info->disk_pages = SFTL_DISK_PAGES; ++ info->nr_runs = sftl->nr_runs; ++ info->checkpoint_sequence = sftl->checkpoint_sequence; ++ info->sequence = sftl->sequence; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_get_info); +diff --git a/drivers/mtd/apple-sftl-test.c b/drivers/mtd/apple-sftl-test.c +new file mode 100644 +index 000000000..e451704c9 +--- /dev/null ++++ b/drivers/mtd/apple-sftl-test.c +@@ -0,0 +1,436 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Synthetic VFL/SFTL medium. No firmware or captured device bytes. */ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-sftl.h" ++ ++#define TEST_LOGICAL_PAGES 15625016U ++#define TEST_STRIDE 8192U ++#define TEST_CHECKPOINT 101ULL ++#define TEST_CLOSED_FIRST 102ULL ++#define TEST_CLOSED_CHUNKS 1020U ++#define TEST_OPEN_FIRST (TEST_CLOSED_FIRST + TEST_CLOSED_CHUNKS) ++ ++enum sftl_fault { ++ SFTL_NO_FAULT, ++ SFTL_CHECKPOINT_COPY, ++ SFTL_LANE_COPY, ++ SFTL_BOOTSTRAP_DUPLICATE, ++ SFTL_INCOMPLETE_CONTEXT, ++ SFTL_JOURNAL_GAP, ++ SFTL_JOURNAL_DUPLICATE, ++ SFTL_BAD_FOOTER, ++ SFTL_READ_FAILURE, ++ SFTL_REFRESH, ++ SFTL_TRIM, ++ SFTL_ALLOCATION_FLOOR, ++ SFTL_INACTIVE_ADDRESS, ++ SFTL_OPEN_HOLE, ++ SFTL_UNKNOWN_STATUS, ++ SFTL_TORN_FOOTER, ++ SFTL_RUNTIME_FAILURE, ++}; ++ ++struct sftl_fixture { ++ struct apple_ppn provider; ++ enum sftl_fault fault; ++ unsigned int reads; ++}; ++ ++static void sftl_test_meta(u8 *meta, u8 kind, u8 flags, u64 seq, u32 key) ++{ ++ unsigned int i; ++ ++ meta[0] = kind; ++ meta[1] = flags; ++ for (i = 0; i < 6; i++) ++ meta[2 + i] = seq >> (8 * i); ++ put_unaligned_le32(key, meta + 8); ++ put_unaligned_le32(seq >> 16, meta + 12); ++} ++ ++static void sftl_test_members(u8 *data, enum sftl_fault fault) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < 8; i++) { ++ put_unaligned_le16(2087, data + 0x100 + i * 4); ++ data[0x102 + i * 4] = i; ++ data[0x103 + i * 4] = 0; ++ } ++ if (fault == SFTL_BOOTSTRAP_DUPLICATE) ++ data[0x106] = 0; ++} ++ ++static int sftl_test_context(struct sftl_fixture *f, unsigned int lane, ++ unsigned int page, u8 *data, u8 *oob, u8 *status) ++{ ++ unsigned int i; ++ ++ if (lane >= 8) ++ return 0; ++ if (!page) { ++ memset(data, 0, 16384); ++ put_unaligned_le16(16, data + 10); ++ put_unaligned_le16(2086, data + 12); ++ put_unaligned_le16(2086, data + 14); ++ sftl_test_members(data, f->fault); ++ for (i = 0; i < 16; i++) ++ put_unaligned_le16(i + 1, data + 0x200 + i * 32); ++ oob[0] = 0x20; ++ oob[1] = 0xff; ++ oob[2] = 0xff; ++ oob[3] = 0; ++ put_unaligned_le32(lane < 4 ? 0xfffffff0 : 0xfffffff1, oob + 4); ++ if (f->fault == SFTL_CHECKPOINT_COPY && lane == 1) ++ data[0] ^= 1; ++ *status = 0x40; ++ } else if (page <= 16) { ++ /* Lane tables occupy pages 1..16 in each of four replicas. */ ++ if (page <= 2) ++ for (i = 1; i <= 4; i++) ++ put_unaligned_le16(0, data + i * 2); ++ oob[0] = 0x20; ++ oob[1] = page - 1; ++ oob[2] = 0; ++ oob[3] = 0; ++ if (f->fault == SFTL_LANE_COPY && lane == 1 && page == 1) ++ put_unaligned_le16(0x8000, data + 2); ++ *status = 0x40; ++ } ++ return 0; ++} ++ ++static void sftl_test_checkpoint(struct sftl_fixture *f, u8 *data, u8 *oob) ++{ ++ u8 *ranges = data + 4096; ++ u32 address = 4 * TEST_STRIDE; ++ ++ memset(data, 0xff, 16384); ++ memset(data, 0, 4096); ++ put_unaligned_le32(1, data); ++ put_unaligned_le32(1, data + 4); ++ sftl_test_meta(oob, 0x1f, 1, 100, 0); ++ if (f->fault == SFTL_INCOMPLETE_CONTEXT) ++ return; ++ if (f->fault == SFTL_INACTIVE_ADDRESS) ++ address += 1024; ++ put_unaligned_le32(0, ranges); ++ put_unaligned_le32(0xfffffff0, ranges + 4); ++ put_unaligned_le32(address, ranges + 8); ++ put_unaligned_le32(1, ranges + 12); ++ put_unaligned_le32(0x1ffffff, ranges + 16); ++ put_unaligned_le32(TEST_LOGICAL_PAGES - 1, ranges + 20); ++ sftl_test_meta(oob + 16, 0x1f, 4, TEST_CHECKPOINT, 0); ++} ++ ++static void sftl_test_open(struct sftl_fixture *f, u8 *data, u8 *oob) ++{ ++ u64 first = TEST_OPEN_FIRST; ++ unsigned int i; ++ ++ if (f->fault == SFTL_JOURNAL_GAP) ++ first++; ++ if (f->fault == SFTL_JOURNAL_DUPLICATE) ++ first--; ++ for (i = 0; i < 4; i++) { ++ memset(data + i * 4096, 'D' + i, 4096); ++ sftl_test_meta(oob + i * 16, 1, 0, first + i, 2); ++ } ++ if (f->fault == SFTL_TRIM) { ++ memset(data, 0, 4096); ++ put_unaligned_le32(1, data); ++ put_unaligned_le32(0, data + 4); ++ put_unaligned_le32(1, data + 8); ++ sftl_test_meta(oob, 2, 0, first, 0xffff0001); ++ } ++} ++ ++static void sftl_test_closed(struct sftl_fixture *f, unsigned int position, ++ u8 *data, u8 *oob) ++{ ++ unsigned int i; ++ ++ if (position < 255) { ++ for (i = 0; i < 4; i++) { ++ u32 slot = position * 4 + i; ++ ++ memset(data + i * 4096, (slot + 1) & 0xff, 4096); ++ sftl_test_meta(oob + i * 16, 1, 0, ++ TEST_CLOSED_FIRST + slot, slot + 1); ++ } ++ } else { ++ u32 span = TEST_CLOSED_CHUNKS; ++ ++ memset(data, 0, 16384); ++ put_unaligned_le32(0, data); ++ put_unaligned_le32(0, data + 4); ++ put_unaligned_le32(1, data + 8); ++ put_unaligned_le32(TEST_CLOSED_CHUNKS, data + 12); ++ if (f->fault == SFTL_ALLOCATION_FLOOR) ++ span += 8; ++ sftl_test_meta(oob, 0x1c, 8, TEST_CLOSED_FIRST, span); ++ put_unaligned_le16(f->fault == SFTL_BAD_FOOTER ? 2 : 4, oob + 12); ++ put_unaligned_le16(0xffff, oob + 14); ++ for (i = 1; i < 4; i++) ++ memcpy(oob + i * 16, oob, 16); ++ if (f->fault == SFTL_TORN_FOOTER) ++ oob[63] ^= 1; ++ } ++} ++ ++static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, ++ bool context, u8 *data, u8 *oob, u8 *status) ++{ ++ struct sftl_fixture *f = priv; ++ unsigned int lane = cau * 2 + bus; ++ unsigned int position = page * 2 + lane; ++ ++ f->reads++; ++ if (bus >= 2 || cau >= 8 || block >= 2088 || page >= (context ? 64 : 128)) ++ return -EINVAL; ++ memset(data, 0xff, 16384); ++ memset(oob, 0xff, 64); ++ *status = 0x49; ++ if (context) { ++ if (block == 2087) ++ return sftl_test_context(f, lane, page, data, oob, status); ++ return 0; ++ } ++ if (block == 3 && (f->fault == SFTL_RUNTIME_FAILURE || ++ (!position && f->fault == SFTL_READ_FAILURE))) ++ return -EIO; ++ if (lane > 1) ++ return 0; ++ if (block == 1 && !position) { ++ sftl_test_checkpoint(f, data, oob); ++ } else if (block == 2 && !position) { ++ sftl_test_open(f, data, oob); ++ } else if (block == 2 && position == 2 && f->fault == SFTL_OPEN_HOLE) { ++ unsigned int i; ++ ++ for (i = 0; i < 4; i++) { ++ memset(data + i * 4096, 'H' + i, 4096); ++ sftl_test_meta(oob + i * 16, 1, 0, TEST_OPEN_FIRST + 4 + i, 3); ++ } ++ } else if (block == 3) { ++ sftl_test_closed(f, position, data, oob); ++ } else if (block == 4 && !position) { ++ memset(data, 'C', 4096); ++ sftl_test_meta(oob, 1, 0, 10, 0); ++ } else { ++ return 0; ++ } ++ *status = block == 2 && f->fault == SFTL_REFRESH ? 0x42 : 0x40; ++ if (block == 2 && f->fault == SFTL_UNKNOWN_STATUS) ++ *status = 0x61; ++ return 0; ++} ++ ++static const struct apple_ppn_ops sftl_test_ops = { ++ .read_page = sftl_test_read_page, ++}; ++ ++static struct sftl_fixture *sftl_test_fixture(struct kunit *test, ++ enum sftl_fault fault) ++{ ++ struct sftl_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); ++ ++ if (!f) ++ return NULL; ++ f->fault = fault; ++ f->provider.ops = &sftl_test_ops; ++ f->provider.priv = f; ++ f->provider.buses = 2; ++ f->provider.caus = 8; ++ f->provider.blocks = 2088; ++ f->provider.pages = 128; ++ return f; ++} ++ ++static void sftl_test_expect_fill(struct kunit *test, const u8 *data, u8 fill) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < 4096; i++) ++ if (data[i] != fill) { ++ KUNIT_FAIL(test, "logical byte %u: expected %02x, got %02x", ++ i, fill, data[i]); ++ return; ++ } ++} ++ ++static void sftl_test_complete_medium(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl_info info; ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_NOT_NULL(test, sftl); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.logical_pages, TEST_LOGICAL_PAGES); ++ KUNIT_EXPECT_EQ(test, info.checkpoint_sequence, TEST_CHECKPOINT); ++ KUNIT_EXPECT_EQ(test, info.sequence, TEST_OPEN_FIRST + 3); ++ KUNIT_EXPECT_EQ(test, info.nr_runs, 5U); ++ KUNIT_EXPECT_EQ(test, info.disk_pages, 15625000U); ++ KUNIT_EXPECT_LT(test, f->reads, 12000U); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 0, data), 0); ++ sftl_test_expect_fill(test, data, 'C'); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 2, data), 0); ++ sftl_test_expect_fill(test, data, 'G'); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 6, data), 0); ++ sftl_test_expect_fill(test, data, 6); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 2000, data), 0); ++ sftl_test_expect_fill(test, data, 0); ++ KUNIT_EXPECT_LT(test, apple_sftl_read(sftl, TEST_LOGICAL_PAGES, data), 0); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_reject_incomplete_or_ambiguous(struct kunit *test) ++{ ++ static const enum sftl_fault faults[] = { ++ SFTL_CHECKPOINT_COPY, SFTL_LANE_COPY, SFTL_BOOTSTRAP_DUPLICATE, ++ SFTL_INCOMPLETE_CONTEXT, SFTL_JOURNAL_GAP, ++ SFTL_JOURNAL_DUPLICATE, SFTL_BAD_FOOTER, ++ SFTL_READ_FAILURE, SFTL_REFRESH, SFTL_UNKNOWN_STATUS, ++ SFTL_TORN_FOOTER, ++ }; ++ unsigned int i; ++ ++ for (i = 0; i < ARRAY_SIZE(faults); i++) { ++ struct sftl_fixture *f = sftl_test_fixture(test, faults[i]); ++ struct apple_sftl *sftl = NULL; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_EXPECT_LT_MSG(test, apple_sftl_load(&f->provider, &sftl), 0, ++ "fault %u", faults[i]); ++ KUNIT_EXPECT_PTR_EQ(test, sftl, NULL); ++ apple_sftl_free(sftl); ++ } ++} ++ ++static void sftl_test_trim_then_overwrite(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_TRIM); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 0, data), 0); ++ sftl_test_expect_fill(test, data, 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 2, data), 0); ++ sftl_test_expect_fill(test, data, 'G'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_preserve_footer_allocation_floor(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_ALLOCATION_FLOOR); ++ struct apple_sftl_info info; ++ struct apple_sftl *sftl = NULL; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, ++ TEST_CLOSED_FIRST + TEST_CLOSED_CHUNKS + 8 - 1); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_reject_other_geometry(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ f->provider.caus = 2; ++ KUNIT_EXPECT_LT(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_EXPECT_PTR_EQ(test, sftl, NULL); ++ KUNIT_EXPECT_EQ(test, f->reads, 0U); ++} ++ ++static void sftl_test_reject_inactive_lane_gap(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_INACTIVE_ADDRESS); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ int ret; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ ret = apple_sftl_load(&f->provider, &sftl); ++ if (!ret) { ++ KUNIT_EXPECT_LT(test, apple_sftl_read(sftl, 0, data), 0); ++ apple_sftl_free(sftl); ++ } else { ++ KUNIT_EXPECT_PTR_EQ(test, sftl, NULL); ++ } ++} ++ ++static void sftl_test_scan_after_open_lane_hole(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_OPEN_HOLE); ++ struct apple_sftl *sftl = NULL; ++ struct apple_sftl_info info; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, TEST_OPEN_FIRST + 7); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 3, data), 0); ++ sftl_test_expect_fill(test, data, 'K'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_runtime_failure_stops_reads(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ unsigned int reads; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ f->fault = SFTL_RUNTIME_FAILURE; ++ KUNIT_EXPECT_LT(test, apple_sftl_read(sftl, 6, data), 0); ++ reads = f->reads; ++ KUNIT_EXPECT_LT(test, apple_sftl_read(sftl, 0, data), 0); ++ KUNIT_EXPECT_EQ(test, f->reads, reads); ++ apple_sftl_free(sftl); ++} ++ ++static struct kunit_case apple_sftl_cases[] = { ++ KUNIT_CASE(sftl_test_complete_medium), ++ KUNIT_CASE(sftl_test_scan_after_open_lane_hole), ++ KUNIT_CASE(sftl_test_runtime_failure_stops_reads), ++ KUNIT_CASE(sftl_test_reject_incomplete_or_ambiguous), ++ KUNIT_CASE(sftl_test_trim_then_overwrite), ++ KUNIT_CASE(sftl_test_preserve_footer_allocation_floor), ++ KUNIT_CASE(sftl_test_reject_other_geometry), ++ KUNIT_CASE(sftl_test_reject_inactive_lane_gap), ++ {} ++}; ++ ++static struct kunit_suite apple_sftl_suite = { ++ .name = "apple-sftl", ++ .test_cases = apple_sftl_cases, ++}; ++ ++kunit_test_suite(apple_sftl_suite); ++ ++MODULE_LICENSE("GPL"); +diff --git a/drivers/mtd/apple-sftl.h b/drivers/mtd/apple-sftl.h +new file mode 100644 +index 000000000..7a05b95d1 +--- /dev/null ++++ b/drivers/mtd/apple-sftl.h +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _APPLE_SFTL_H ++#define _APPLE_SFTL_H ++ ++#include ++#include ++ ++struct apple_sftl; ++ ++struct apple_sftl_info { ++ u32 logical_pages; ++ u32 disk_pages; ++ u32 nr_runs; ++ u64 checkpoint_sequence; ++ u64 sequence; ++}; ++ ++int apple_sftl_load(struct apple_ppn *provider, struct apple_sftl **out); ++void apple_sftl_free(struct apple_sftl *sftl); ++int apple_sftl_read(struct apple_sftl *sftl, u32 logical_page, u8 *data); ++void apple_sftl_get_info(struct apple_sftl *sftl, struct apple_sftl_info *info); ++ ++#endif +diff --git a/tools/testing/kunit/configs/apple_sftl.config b/tools/testing/kunit/configs/apple_sftl.config +new file mode 100644 +index 000000000..5ee7e49bb +--- /dev/null ++++ b/tools/testing/kunit/configs/apple_sftl.config +@@ -0,0 +1,4 @@ ++CONFIG_KUNIT=y ++CONFIG_MTD=y ++CONFIG_MTD_APPLE_SFTL=y ++CONFIG_MTD_APPLE_SFTL_KUNIT_TEST=y diff --git a/kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch.toml b/kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0004-mtd-add-read-only-Apple-VFL-and-SwissPPNFTL-recovery.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] diff --git a/kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch b/kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch new file mode 100644 index 00000000..c5153b62 --- /dev/null +++ b/kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch @@ -0,0 +1,315 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 12:26:04 -0600 +Subject: [PATCH] mtd: apple-sftl: bound read-only provider lifetime + +Count physical page requests across map recovery and block reads. Apply a finite deadline before each provider call and check it again when the transaction completes, so one controller transaction bounds any overrun. Keep the reader failed after a provider error or exhausted policy, including logical reads that do not need flash I/O. + +Expose positive read-only policy parameters with defaults of 32768 requests and 600000 milliseconds for manual qualification sessions. These limits describe software lifetime rather than hardware behaviour. Add KUnit boundary cases for both limits and provider failures. + +Assisted-by: LLM +--- + Documentation/driver-api/mtd/apple-sftl.rst | 10 +++ + drivers/mtd/apple-sftl-block.c | 55 ++++++++++++++++- + drivers/mtd/apple-sftl-limits.h | 67 +++++++++++++++++++++ + drivers/mtd/apple-sftl-test.c | 62 +++++++++++++++++++ + 4 files changed, 193 insertions(+), 1 deletion(-) + create mode 100644 drivers/mtd/apple-sftl-limits.h + +diff --git a/Documentation/driver-api/mtd/apple-sftl.rst b/Documentation/driver-api/mtd/apple-sftl.rst +index 8295a5a2a..ade96fde7 100644 +--- a/Documentation/driver-api/mtd/apple-sftl.rst ++++ b/Documentation/driver-api/mtd/apple-sftl.rst +@@ -46,6 +46,16 @@ frontend freezes and drains requests during removal. Its request buffer is + allocated before the disk becomes visible, while blocking requests serialize + access to that buffer and the provider. + ++The reader has finite operating limits. ``max_page_reads`` defaults to ++32,768 physical page requests and ``max_io_time_ms`` defaults to ten minutes ++from probe. Both are read-only module parameters and must be positive. ++Recovery and subsequent block reads consume the same budget. A limit check ++runs before each provider call; completion also checks the deadline, so an ++existing bounded provider transaction is the maximum overrun. Any provider ++failure or exhausted limit permanently stops that reader instance. These ++limits support bounded read-only qualification; a mounted root must not rely ++on an instance after its operating budget expires. ++ + Tests + ----- + +diff --git a/drivers/mtd/apple-sftl-block.c b/drivers/mtd/apple-sftl-block.c +index 86d02921b..bf7a0a108 100644 +--- a/drivers/mtd/apple-sftl-block.c ++++ b/drivers/mtd/apple-sftl-block.c +@@ -4,18 +4,31 @@ + #include + #include + #include ++#include + #include + #include + #include + #include + + #include "apple-sftl.h" ++#include "apple-sftl-limits.h" + + #define SFTL_BLOCK_NAME "apple-sftl" + #define SFTL_DISK_MINORS 16 + #define SFTL_LOGICAL_BYTES 4096U + ++/* Limits bound one manually loaded reader instance, including recovery. */ ++static unsigned int max_page_reads = 32768; ++module_param(max_page_reads, uint, 0400); ++MODULE_PARM_DESC(max_page_reads, "Maximum physical page reads per reader instance"); ++static unsigned int max_io_time_ms = 600000; ++module_param(max_io_time_ms, uint, 0400); ++MODULE_PARM_DESC(max_io_time_ms, "Maximum reader lifetime in milliseconds"); ++ + struct sftl_disk { ++ struct apple_ppn *upstream; ++ struct apple_ppn provider; ++ struct apple_sftl_io_limit limit; + struct apple_sftl *ftl; + struct blk_mq_tag_set tags; + struct gendisk *disk; +@@ -25,6 +38,25 @@ struct sftl_disk { + bool dead; + }; + ++static int sftl_limited_read(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, bool context, ++ u8 *data, u8 *oob, u8 *status) ++{ ++ struct sftl_disk *sftl = priv; ++ int ret; ++ ++ ret = apple_sftl_limit_begin(&sftl->limit, ktime_get_ns()); ++ if (ret) ++ return ret; ++ ret = sftl->upstream->ops->read_page(sftl->upstream->priv, bus, cau, ++ block, page, context, data, oob, status); ++ return apple_sftl_limit_end(&sftl->limit, ret, ktime_get_ns()); ++} ++ ++static const struct apple_ppn_ops sftl_limited_ops = { ++ .read_page = sftl_limited_read, ++}; ++ + static DEFINE_IDA(sftl_disk_ids); + static int sftl_major; + +@@ -77,6 +109,8 @@ static blk_status_t sftl_queue_rq(struct blk_mq_hw_ctx *hctx, + mutex_lock(&sftl->io_lock); + if (sftl->dead) { + ret = -ENODEV; ++ } else if ((ret = apple_sftl_limit_check(&sftl->limit, ktime_get_ns()))) { ++ /* Includes logical pages which need no provider transaction. */ + } else if (req_op(request) == REQ_OP_FLUSH) { + /* A read-only device has no buffered writes to flush. */ + } else if (req_op(request) != REQ_OP_READ) { +@@ -125,9 +159,21 @@ static int sftl_disk_probe(struct platform_device *pdev) + ret = -ENOMEM; + goto free_state; + } +- ret = apple_sftl_load(provider, &sftl->ftl); ++ sftl->upstream = provider; ++ sftl->provider = *provider; ++ sftl->provider.ops = &sftl_limited_ops; ++ sftl->provider.priv = sftl; ++ ret = apple_sftl_limit_start(&sftl->limit, max_page_reads, ++ max_io_time_ms, ktime_get_ns()); + if (ret) + goto free_buffer; ++ dev_info(&pdev->dev, "reader policy: %u page reads, %u ms\n", ++ max_page_reads, max_io_time_ms); ++ ret = apple_sftl_load(&sftl->provider, &sftl->ftl); ++ if (ret) ++ goto free_buffer; ++ dev_info(&pdev->dev, "recovery completed after %u physical page reads\n", ++ sftl->limit.reads); + apple_sftl_get_info(sftl->ftl, &info); + if (!info.disk_pages || info.disk_pages > info.logical_pages) { + ret = -EINVAL; +@@ -176,6 +222,9 @@ static int sftl_disk_probe(struct platform_device *pdev) + free_ftl: + apple_sftl_free(sftl->ftl); + free_buffer: ++ if (sftl->limit.failed) ++ dev_err(&pdev->dev, "reader stopped after %u page requests: %d\n", ++ sftl->limit.reads, ret); + kfree(sftl->buffer); + free_state: + kfree(sftl); +@@ -187,6 +236,8 @@ static void sftl_disk_remove(struct platform_device *pdev) + struct sftl_disk *sftl = platform_get_drvdata(pdev); + + mutex_lock(&sftl->io_lock); ++ dev_info(&pdev->dev, "reader retired after %u physical page reads; failed=%u\n", ++ sftl->limit.reads, sftl->limit.failed); + sftl->dead = true; + mutex_unlock(&sftl->io_lock); + /* del_gendisk freezes and drains the queue before the provider is freed. */ +@@ -211,6 +262,8 @@ static int __init sftl_init(void) + { + int ret; + ++ if (!max_page_reads || !max_io_time_ms) ++ return -EINVAL; + sftl_major = register_blkdev(0, SFTL_BLOCK_NAME); + if (sftl_major < 0) + return sftl_major; +diff --git a/drivers/mtd/apple-sftl-limits.h b/drivers/mtd/apple-sftl-limits.h +new file mode 100644 +index 000000000..315bb8499 +--- /dev/null ++++ b/drivers/mtd/apple-sftl-limits.h +@@ -0,0 +1,67 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _APPLE_SFTL_LIMITS_H ++#define _APPLE_SFTL_LIMITS_H ++ ++#include ++#include ++#include ++ ++/* The caller serializes begin/end with the provider transaction. */ ++struct apple_sftl_io_limit { ++ u64 deadline_ns; ++ u32 maximum_reads; ++ u32 reads; ++ bool failed; ++}; ++ ++static inline int apple_sftl_limit_start(struct apple_sftl_io_limit *limit, ++ u32 reads, u32 milliseconds, u64 now) ++{ ++ u64 duration = (u64)milliseconds * 1000000ULL; ++ ++ if (!reads || !milliseconds || ++ check_add_overflow(now, duration, &limit->deadline_ns)) ++ return -EINVAL; ++ limit->maximum_reads = reads; ++ limit->reads = 0; ++ limit->failed = false; ++ return 0; ++} ++ ++static inline int apple_sftl_limit_check(struct apple_sftl_io_limit *limit, u64 now) ++{ ++ if (limit->failed) ++ return -EIO; ++ if (now < limit->deadline_ns) ++ return 0; ++ limit->failed = true; ++ return -ETIMEDOUT; ++} ++ ++static inline int apple_sftl_limit_begin(struct apple_sftl_io_limit *limit, u64 now) ++{ ++ int ret = apple_sftl_limit_check(limit, now); ++ ++ if (ret) ++ return ret; ++ if (limit->reads >= limit->maximum_reads) ++ ret = -E2BIG; ++ if (ret) { ++ limit->failed = true; ++ return ret; ++ } ++ limit->reads++; ++ return 0; ++} ++ ++static inline int apple_sftl_limit_end(struct apple_sftl_io_limit *limit, ++ int result, u64 now) ++{ ++ if (!result && now >= limit->deadline_ns) ++ result = -ETIMEDOUT; ++ if (result) ++ limit->failed = true; ++ return result; ++} ++ ++#endif +diff --git a/drivers/mtd/apple-sftl-test.c b/drivers/mtd/apple-sftl-test.c +index e451704c9..88308aadb 100644 +--- a/drivers/mtd/apple-sftl-test.c ++++ b/drivers/mtd/apple-sftl-test.c +@@ -8,6 +8,7 @@ + #include + + #include "apple-sftl.h" ++#include "apple-sftl-limits.h" + + #define TEST_LOGICAL_PAGES 15625016U + #define TEST_STRIDE 8192U +@@ -414,7 +415,68 @@ static void sftl_test_runtime_failure_stops_reads(struct kunit *test) + apple_sftl_free(sftl); + } + ++static void sftl_test_page_budget_stops_before_next_read(struct kunit *test) ++{ ++ struct apple_sftl_io_limit limit; ++ ++ KUNIT_ASSERT_EQ(test, apple_sftl_limit_start(&limit, 2, 1, 0), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 0), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_end(&limit, 0, 1), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 2), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_end(&limit, 0, 3), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 4), -E2BIG); ++ KUNIT_EXPECT_EQ(test, limit.reads, 2U); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 5), -EIO); ++ KUNIT_EXPECT_EQ(test, limit.reads, 2U); ++} ++ ++static void sftl_test_deadline_stops_before_next_read(struct kunit *test) ++{ ++ struct apple_sftl_io_limit limit; ++ ++ KUNIT_ASSERT_EQ(test, apple_sftl_limit_start(&limit, 4, 1, 10), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 1000010), -ETIMEDOUT); ++ KUNIT_EXPECT_EQ(test, limit.reads, 0U); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 10), -EIO); ++} ++ ++static void sftl_test_one_transaction_deadline_overrun(struct kunit *test) ++{ ++ struct apple_sftl_io_limit limit; ++ ++ KUNIT_ASSERT_EQ(test, apple_sftl_limit_start(&limit, 4, 1, 0), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 999999), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_end(&limit, 0, 1000000), -ETIMEDOUT); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 1000001), -EIO); ++ KUNIT_EXPECT_EQ(test, limit.reads, 1U); ++} ++ ++static void sftl_test_provider_failure_is_sticky(struct kunit *test) ++{ ++ struct apple_sftl_io_limit limit; ++ ++ KUNIT_ASSERT_EQ(test, apple_sftl_limit_start(&limit, 4, 1, 0), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 0), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_end(&limit, -EBADMSG, 1), -EBADMSG); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_begin(&limit, 2), -EIO); ++ KUNIT_EXPECT_EQ(test, limit.reads, 1U); ++} ++ ++static void sftl_test_invalid_reader_policy(struct kunit *test) ++{ ++ struct apple_sftl_io_limit limit; ++ ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_start(&limit, 0, 1, 0), -EINVAL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_start(&limit, 1, 0, 0), -EINVAL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_limit_start(&limit, 1, 1, ~0ULL), -EINVAL); ++} ++ + static struct kunit_case apple_sftl_cases[] = { ++ KUNIT_CASE(sftl_test_page_budget_stops_before_next_read), ++ KUNIT_CASE(sftl_test_deadline_stops_before_next_read), ++ KUNIT_CASE(sftl_test_one_transaction_deadline_overrun), ++ KUNIT_CASE(sftl_test_provider_failure_is_sticky), ++ KUNIT_CASE(sftl_test_invalid_reader_policy), + KUNIT_CASE(sftl_test_complete_medium), + KUNIT_CASE(sftl_test_scan_after_open_lane_hole), + KUNIT_CASE(sftl_test_runtime_failure_stops_reads), diff --git a/kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch.toml b/kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0005-mtd-apple-sftl-bound-read-only-provider-lifetime.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] From a607325fbe63773814b06c7e2226cbea8140ef38 Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:18:15 -0600 Subject: [PATCH 12/16] ARM: config: build the bounded Apple SFTL reader as a module Resolve the Apple ARMv7 config from kernel source commit 0921ec24248913d2a99ad1a262b2ab1e66f062e3 through kernel-tree/export.py. CONFIG_MTD_APPLE_SFTL=m lets recovery load the read-only frontend after its USB recovery link is ready. Keep CONFIG_MTD_APPLE_H2FMI_WRITE=n so this config exposes no raw page-program path. LLVM 19 olddefconfig and savedefconfig passed. The 23 focused kernel, platform and export tests passed. Physical qualification remains pending. Assisted-by: LLM --- kernel/configs/apple-armv7/config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/configs/apple-armv7/config b/kernel/configs/apple-armv7/config index 6a2ab9d0..31b5d8df 100644 --- a/kernel/configs/apple-armv7/config +++ b/kernel/configs/apple-armv7/config @@ -1450,6 +1450,7 @@ CONFIG_MTD_OF_PARTS=y # CONFIG_MTD_OOPS is not set # CONFIG_MTD_SWAP is not set # CONFIG_MTD_PARTITIONED_MASTER is not set +CONFIG_MTD_APPLE_SFTL=m # # RAM/ROM/Flash chip drivers @@ -1505,6 +1506,7 @@ CONFIG_MTD_SPEAR_SMI=y # # CONFIG_MTD_DOCG3 is not set CONFIG_MTD_APPLE_H2FMI=y +# CONFIG_MTD_APPLE_H2FMI_WRITE is not set # end of Self-contained MTD device drivers # From bb63ec74b0b0e6be5b7719f2417f1c2d61ab39aa Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:59:53 -0600 Subject: [PATCH 13/16] p105: record experimental native append evidence and limits Describe the kernel append, summary replay and logical write policy following native-map recovery. Keep storage_write missing and retain the installation gate while the exact kernel revision awaits physical write qualification. Record separate A5 append evidence and the outstanding recovery, persistence and reclamation work. The source passes both current ARMv7 object variants and 26 real Linux KUnit cases. Host checks cover request updates and bounded provider forwarding. Document their scope together with default-off policy and NOIO request allocation. Kernel patch export remains a separate step. Regenerate the support matrix and run all 34 catalog tests with the checkout's absolute PYTHONPATH; both storage states stay missing. Assisted-by: LLM --- chips/s5l8942x.toml | 1 + devices/p105/README.md | 17 ++++++ devices/p105/device.toml | 3 +- docs/bring-up/p105-sftl-append.md | 89 +++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 docs/bring-up/p105-sftl-append.md diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index da8134d0..e76ef051 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -21,3 +21,4 @@ cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff await # Public Cascadia RAM recovery correlates a guarded host USB identity with the live FDT and boot ID. # Linux SoC ECID access and on-device public recovery identity remain unqualified. +nand_append = "The measured P105 64 GB kernel append path has native data/footer replay, verified programs, default-off write policy and finite operating bounds; synthetic Linux KUnit and ARMv7 object checks passed, while hardware writes remain unqualified (docs/bring-up/p105-sftl-append.md)." diff --git a/devices/p105/README.md b/devices/p105/README.md index f66904bd..fce06bac 100644 --- a/devices/p105/README.md +++ b/devices/p105/README.md @@ -37,3 +37,20 @@ The generated loader checks its configuration before copying the recovery initramfs into the reserved high-memory range. The DTB names that initramfs. This extension needs a hardware boot of the final producer revision; the earlier installed P105 boot used an initramfs compiled into its kernel. + +## Unmerged code + +The native PPN storage reader and append writer are on the kernel source +branch `p105/07-ftl-append`, at +`c63274165870417f01cc3e1264067cef71d9c0b2`. The +[append bring-up note](../../docs/bring-up/p105-sftl-append.md) records its +software checks and write policy. Kernel patch export remains a separate +integration step. Both storage feature states remain `missing`. + +## Open work + +The exact kernel revision needs a bounded device boot with USB recovery, +then approved logical writes and a return to iOS. Reboot persistence, +interruption recovery and sustained writes need physical qualification. +The append allocator stops at `ENOSPC`; reclamation requires further work. +The earlier unplugged Omarchy desktop used the local Python storage backend. diff --git a/devices/p105/device.toml b/devices/p105/device.toml index ff617f52..b5536905 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -43,7 +43,8 @@ touch_id = "missing" # PPN geometry/read topic and native-map provider await exact-build qualification. # See docs/bring-up/p105-nand-reads.md. # The bounded native VFL/SFTL reader awaits an exact-revision device boot. -# See docs/bring-up/p105-sftl-reader.md; logical storage capabilities stay missing. +# Native append, footer replay and write policy also await device qualification. +# See docs/bring-up/p105-sftl-reader.md and p105-sftl-append.md; install stays gated. storage_read = "missing" # The default-off complete-page primitive does not provide a qualified logical writer. # See docs/bring-up/p105-nand-program.md for its checks and pending hardware work. diff --git a/docs/bring-up/p105-sftl-append.md b/docs/bring-up/p105-sftl-append.md new file mode 100644 index 00000000..d209139b --- /dev/null +++ b/docs/bring-up/p105-sftl-append.md @@ -0,0 +1,89 @@ +# P105 native append writes + +The kernel `apple-sftl` writer appends to Apple's shared VFL/SwissPPNFTL +medium on the measured 64 GB P105. Native records carry the updated logical +map across reload. The boot reader must follow VFL and SFTL mapping to find +Linux data. LwVM partitions sit above the recovered `applesftlN` disk. +The [reader note](p105-sftl-reader.md) describes geometry and context recovery. + +## Source and format + +Kernel branch `p105/07-ftl-append` ends at +`c63274165870417f01cc3e1264067cef71d9c0b2`. It contains the verified PPN +program callback (`ab137f3be`), append/replay path (`cb20cc76a`) and request +allocation guard (`c63274165`), following the read-only reader and program +primitive dependencies. Kernel patch export remains pending integration. + +The allocator reserves the boot and package tails, every saved SFTL context, +validated VFL bootstrap members and blocks referenced by the recovered map. +It checks every data and metadata byte across a candidate's active lanes +before programming. An occupied candidate is skipped. A failed storage transaction stops the +instance; exhausted blank space returns `ENOSPC` while reads +remain available. The allocator performs no erase or reclamation. + +A program writes a complete 16 KiB page with 64 canonical metadata bytes. +One through four logical 4 KiB entries occupy its chunks; short requests +repeat the last entry with a distinct sequence for each copy. The provider +waits for completion and verifies exact readback before map publication. +Filled blocks receive eight summary pages containing native type-1c runs. +The span field at metadata offset eight is the next sequence minus the block +base; it reserves no additional eight sequence slots. + +A fresh loader replays open data records or a completed summary. An +interrupted summary can be ignored only after its entire data prefix +validates. Torn or ambiguous metadata stops recovery. Recovery retains its +contiguous journal rule, so an allocation floor beyond the final visible +record prevents writes before mutation. The saved measured map has equal +allocation and journal sequence 187831212. + +The frontend preserves untouched sectors during partial 4 KiB updates and +groups up to four logical pages per physical program. Flush and FUA complete +after synchronous readback. Request processing uses a NOIO allocation scope, +which prevents map or provider allocations from recursing into writeback +while the request lock is held. + +## Explicit operating policy + +Both `MTD_APPLE_H2FMI_WRITE` and `MTD_APPLE_SFTL_WRITE` default off. Logical +writes also require `write_enable=1` when loading `apple-sftl`, or +`apple_sftl.write_enable=1` for a built-in driver. The parameter defaults to +false and is read-only after load. A supported replay state and a program +callback are required. The installer must fence its intended logical range; +this policy controls the logical disk independently of raw MTD permissions. + +`max_page_reads` defaults to 32768 explicit canonical read callbacks across +recovery, candidate scans and runtime reads. Program-internal blank checks +and readback remain inside the program callback and outside that counter. +`max_io_time_ms` defaults to 600000 milliseconds from probe. Programs check +the deadline before and after the synchronous callback. Both limit parameters +are read-only and accept 1 through 4294967295. Exhaustion or a provider error +permanently stops the instance. These defaults bound qualification; an +installed root needs an explicit operating policy and must stay within it. + +## Evidence and qualification + +The exact source passed these software checks: + +- Current ARMv7 core, block, fixture and H2FMI objects compiled in read-only + and write-enabled variants with `W=1` and warnings as errors. The existing + `vmstat.h` enum-arithmetic warning remained a warning. Built-in parameter + prefixes were verified from the objects. +- All 26 tests passed in a Linux KUnit kernel using pinned pristine v6.12, + exact core/block/test sources and a validation-only Kconfig dependency + symbol for the mock program provider. This run excludes H2FMI hardware. + Cases cover all 16 lanes, 8160 separate footer entries and fresh replay. +- Host ASan/UBSan checks passed 26 core cases and 13 read-only cases with + allocation, mutex and tree-API shims. Extracted request helpers passed 25 + cases plus 2048 varied requests. Queue/policy and provider-wrapper checks + cover deadline failures and balanced allocation scope. +- The actual C core reconstructed saved private pages and matched 28830 map + boundary checks against the Python reference, plus native LwVM and System + prefixes. This issued zero device operations. Public fixtures generate + synthetic bytes. + +The exact kernel writer has no physical-device write qualification. The +earlier unplugged Omarchy desktop used the local Python backend. A fresh +approved boot must establish recovery access and bounded reads before any +approved write work. Native iOS recovery after writes, reboot persistence +and interruption handling need physical evidence. P105 `storage_write` +remains `missing`, with installation gated by the current feature states. From e049bb46717e23387d080db9343b9b7eae6c51c0 Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:07:57 -0600 Subject: [PATCH 14/16] mtd: export verified P105 append writes for apple-armv7 Export the optional PPN program callback and append/replay implementation with its NOIO request safeguard. Preserve source authorship and trailers from Linux commits ab137f3be, cb20cc76a and c63274165. The official exporter proves that these three patches reproduce the source head from dependency 161217fd32. No platform config changes are required. Scope all three MTD patches to apple-armv7. Its selected patch count grows from 28 to 31; apple-4k and apple-16k each remain at 450. All 34 catalog checks and four kernel-tree/export tests pass. The exact source also passed both ARMv7 object variants and 26 Linux KUnit cases. Keep P105 storage capabilities missing while the exact writer awaits physical qualification. Update bring-up links to the exported patches. Assisted-by: LLM --- devices/p105/README.md | 4 +- docs/bring-up/p105-sftl-append.md | 3 +- docs/bring-up/p105-sftl-reader.md | 4 +- ...xpose-verified-full-page-programming.patch | 59 + ...-verified-full-page-programming.patch.toml | 2 + ...dd-verified-append-writes-and-fresh-.patch | 1674 +++++++++++++++++ ...rified-append-writes-and-fresh-.patch.toml | 2 + ...revent-recursive-reclaim-during-requ.patch | 73 + ...t-recursive-reclaim-during-requ.patch.toml | 2 + 9 files changed, 1819 insertions(+), 4 deletions(-) create mode 100644 kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch create mode 100644 kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch.toml create mode 100644 kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch create mode 100644 kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch.toml create mode 100644 kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch create mode 100644 kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch.toml diff --git a/devices/p105/README.md b/devices/p105/README.md index fce06bac..52005266 100644 --- a/devices/p105/README.md +++ b/devices/p105/README.md @@ -44,8 +44,8 @@ The native PPN storage reader and append writer are on the kernel source branch `p105/07-ftl-append`, at `c63274165870417f01cc3e1264067cef71d9c0b2`. The [append bring-up note](../../docs/bring-up/p105-sftl-append.md) records its -software checks and write policy. Kernel patch export remains a separate -integration step. Both storage feature states remain `missing`. +software checks and write policy. The exported MTD patches are scoped to +`apple-armv7`. Both storage feature states remain `missing`. ## Open work diff --git a/docs/bring-up/p105-sftl-append.md b/docs/bring-up/p105-sftl-append.md index d209139b..3fb79a7b 100644 --- a/docs/bring-up/p105-sftl-append.md +++ b/docs/bring-up/p105-sftl-append.md @@ -12,7 +12,8 @@ Kernel branch `p105/07-ftl-append` ends at `c63274165870417f01cc3e1264067cef71d9c0b2`. It contains the verified PPN program callback (`ab137f3be`), append/replay path (`cb20cc76a`) and request allocation guard (`c63274165`), following the read-only reader and program -primitive dependencies. Kernel patch export remains pending integration. +primitive dependencies. Exported MTD patches 0006 through 0008 carry these +changes with `apple-armv7` scope and reproduce the source tree exactly. The allocator reserves the boot and package tails, every saved SFTL context, validated VFL bootstrap members and blocks referenced by the recovered map. diff --git a/docs/bring-up/p105-sftl-reader.md b/docs/bring-up/p105-sftl-reader.md index cf8bddc1..80aad88a 100644 --- a/docs/bring-up/p105-sftl-reader.md +++ b/docs/bring-up/p105-sftl-reader.md @@ -9,7 +9,9 @@ encryption. The measured profile has two buses and eight CAUs per bus, each with 2,088 blocks of 128 normal-view pages. H2FMI supplies each 16 KiB data page with its 64 metadata bytes through a canonical provider. Context reads select the -separate SLC address view. This provider offers no program or erase callback. +separate SLC address view. The read-only candidate below exposes canonical +reads. The [append work](p105-sftl-append.md) adds an optional verified program +callback with explicit write policy; the provider has no erase operation. Recovery requires agreeing VFL context replicas and lane tables. It validates the saved SFTL context fragments, then replays later data and trim records in diff --git a/kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch b/kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch new file mode 100644 index 00000000..b8e5f48e --- /dev/null +++ b/kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch @@ -0,0 +1,59 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 12:32:37 -0600 +Subject: [PATCH] mtd: apple-ppn: expose verified full-page programming + +Add an optional program_page operation to the aggregate canonical PPN +provider. Its success contract includes completion and exact data/metadata +readback, so an FTL can publish mappings only after the full unit succeeds. + +Route the H2FMI callback through its existing synchronized page program +engine. The controller retains its geometry, blank-page, boot/tail block +and default-off configuration guards. + +Assisted-by: LLM +--- + drivers/mtd/devices/mtd-apple-h2fmi.c | 12 ++++++++++++ + include/linux/mtd/apple-ppn.h | 4 ++++ + 2 files changed, 16 insertions(+) + +diff --git a/drivers/mtd/devices/mtd-apple-h2fmi.c b/drivers/mtd/devices/mtd-apple-h2fmi.c +index bd7ba9110..ace9d72c7 100644 +--- a/drivers/mtd/devices/mtd-apple-h2fmi.c ++++ b/drivers/mtd/devices/mtd-apple-h2fmi.c +@@ -671,8 +671,20 @@ static int h2fmi_ppn_read(void *priv, unsigned int bus, unsigned int cau, + return ret; + } + ++static int h2fmi_ppn_program(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, ++ const u8 *data, const u8 *oob) ++{ ++ struct h2fmi *fmi = priv; ++ ++ if (bus >= H2FMI_BUSES) ++ return -EINVAL; ++ return h2fmi_program_page(fmi, &fmi->bus[bus], cau, block, page, data, oob); ++} ++ + static const struct apple_ppn_ops h2fmi_ppn_ops = { + .read_page = h2fmi_ppn_read, ++ .program_page = h2fmi_ppn_program, + }; + + static int h2fmi_register_sftl(struct h2fmi *fmi) +diff --git a/include/linux/mtd/apple-ppn.h b/include/linux/mtd/apple-ppn.h +index 78ee66867..73fba467d 100644 +--- a/include/linux/mtd/apple-ppn.h ++++ b/include/linux/mtd/apple-ppn.h +@@ -11,6 +11,10 @@ struct device; + + /* All buffers have canonical data/OOB layout, including context reads. */ + struct apple_ppn_ops { ++ /* Success means the complete page and metadata passed exact readback. */ ++ int (*program_page)(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, ++ const u8 *data, const u8 *oob); + int (*read_page)(void *priv, unsigned int bus, unsigned int cau, + unsigned int block, unsigned int page, bool context, + u8 *data, u8 *oob, u8 *status); diff --git a/kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch.toml b/kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0006-mtd-apple-ppn-expose-verified-full-page-programming.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] diff --git a/kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch b/kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch new file mode 100644 index 00000000..44b350be --- /dev/null +++ b/kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch @@ -0,0 +1,1674 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 12:34:23 -0600 +Subject: [PATCH] mtd: apple-sftl: add verified append writes and fresh replay + +Append complete data pages in fully checked blank VFL blocks while +retaining Apple's shared map, saved contexts and bootstrap members. Use +native metadata and completed-block summaries, and recover a complete +data prefix when summary completion is interrupted. Stop at ENOSPC +without erasing or reclaiming blocks. + +Add block-sector read-modify-write with four-page grouping, synchronous +flush/FUA completion and a default-false logical write_enable policy. +MTD_APPLE_SFTL_WRITE also defaults off and requires H2FMI_WRITE, so the +manual read-only qualification path remains fenced. Preserve the bounded +provider policy when forwarding optional programs. + +Keep the current contiguous replay contract: refuse append before any +program when the allocation floor exceeds the last replayed record. +The saved measured 64 GB map has equal floor/journal sequence 187831212. + +Generate synthetic media, program them, and load a fresh map for open, +closed, interrupted and uncertain writes. Cover all 16 lanes and maximum +footer size, including VFL reservations and sequence overflow. Host +sanitizers pass 26 cases with writes enabled and 13 in read-only mode. +Hardware qualification remains separate from these tests. + +Assisted-by: LLM +--- + Documentation/driver-api/mtd/apple-sftl.rst | 112 +++- + drivers/mtd/Kconfig | 25 +- + drivers/mtd/apple-sftl-block.c | 124 +++- + drivers/mtd/apple-sftl-core.c | 312 ++++++++- + drivers/mtd/apple-sftl-test.c | 597 +++++++++++++++++- + drivers/mtd/apple-sftl.h | 4 + + .../kunit/configs/apple_sftl_write.config | 8 + + 7 files changed, 1120 insertions(+), 62 deletions(-) + create mode 100644 tools/testing/kunit/configs/apple_sftl_write.config + +diff --git a/Documentation/driver-api/mtd/apple-sftl.rst b/Documentation/driver-api/mtd/apple-sftl.rst +index ade96fde7..9d65b2149 100644 +--- a/Documentation/driver-api/mtd/apple-sftl.rst ++++ b/Documentation/driver-api/mtd/apple-sftl.rst +@@ -1,7 +1,7 @@ + .. SPDX-License-Identifier: GPL-2.0-only + +-Apple PPN VFL/SwissPPNFTL reader +-============================== ++Apple PPN VFL/SwissPPNFTL ++======================== + + The ``apple-sftl`` driver recovers the logical medium above Apple's PPN flash + mapping. H2FMI supplies complete 16 KiB pages and their 64 metadata bytes +@@ -23,49 +23,105 @@ which the reader rejects. + SFTL subtype-4 context fragments describe the saved logical ranges. Recovery + validates the complete context sequence, then applies later data and trim + records in sequence order. Completed blocks carry summaries at their ends. +-Open blocks are scanned completely because an interrupted stripe may leave +-an erased lane before another completed lane. Summary allocation spans also ++Open blocks are scanned completely because an interrupted stripe may leave an ++erased lane before another completed lane. Summary allocation spans also + preserve the next sequence floor when it exceeds the last visible record. + + The map has 15,625,016 entries, including sixteen internal keys. The block +-device exposes 15,625,000 entries, or 64,000,000,000 bytes, with 512-byte sector +-reads. Linux's disk name is ``applesftlN``. LwVM and filesystem interpretation +-happen above this logical provider. Encrypted partition contents remain +-ciphertext. ++device exposes 15,625,000 entries, or 64,000,000,000 bytes, with 512-byte ++sector reads and optional read-modify-write sector updates. Linux's disk name ++is ``applesftlN``. LwVM and filesystem interpretation happen above this ++logical provider. Encrypted partition contents remain ciphertext. + + Failure and lifetime rules + -------------------------- + +-The disk is read-only and the provider has no programming or erase operation. +-Malformed context copies and incomplete journal sequences prevent disk +-registration. A refresh status requires a writer and is reported as an error. +-A failed runtime read stops subsequent reads from this map instance. ++The disk is read-only by default. Malformed context copies and incomplete ++journal sequences prevent disk registration. A refresh status is reported as ++an error. A failed runtime read stops subsequent I/O from this map instance. + + H2FMI unregisters the child before releasing its controller state. The block + frontend freezes and drains requests during removal. Its request buffer is + allocated before the disk becomes visible, while blocking requests serialize + access to that buffer and the provider. + +-The reader has finite operating limits. ``max_page_reads`` defaults to +-32,768 physical page requests and ``max_io_time_ms`` defaults to ten minutes +-from probe. Both are read-only module parameters and must be positive. +-Recovery and subsequent block reads consume the same budget. A limit check +-runs before each provider call; completion also checks the deadline, so an +-existing bounded provider transaction is the maximum overrun. Any provider +-failure or exhausted limit permanently stops that reader instance. These +-limits support bounded read-only qualification; a mounted root must not rely +-on an instance after its operating budget expires. ++The reader has finite operating limits. ``max_page_reads`` defaults to 32,768 ++physical page requests and ``max_io_time_ms`` defaults to ten minutes from ++probe. Both are read-only module parameters and must be positive. Recovery, ++candidate blank scans and subsequent block reads consume the same budget. ++Programs check the deadline before and after their synchronous transaction; ++internal readback remains part of that provider operation. A limit check runs ++before each provider call; completion also checks the deadline, so an existing ++bounded provider transaction is the maximum overrun. Any provider failure or ++exhausted limit permanently stops that reader instance. These limits support ++bounded read-only qualification; a mounted root must not rely on an instance ++after its operating budget expires. ++ ++Experimental append writes ++-------------------------- ++ ++``CONFIG_MTD_APPLE_SFTL_WRITE`` defaults off and requires the H2FMI page ++programming option. Logical writes also require the read-only module parameter ++``write_enable=1`` at module load, or ``apple_sftl.write_enable=1`` on the ++kernel command line for a built-in driver. It defaults to false and cannot ++change while the module is loaded. Recovery leaves this policy off; an ++approved installer or installed-root boot path must opt in and fence the ++intended logical partition. The aggregate provider must accept a complete 16 ++KiB page with its 64 metadata bytes, wait for completion, and verify exact ++readback before reporting success. The FTL publishes the new mapping after ++that success. A short logical write repeats its final 4 KiB entry to fill the ++physical program unit; each copy carries its own sequence. ++ ++The allocator reserves every saved SFTL context block, all validated VFL ++bootstrap member blocks, every block referenced by the recovered map, and the ++boot and package tail blocks. It selects a block whose head is erased, then ++checks every physical page and metadata byte on every active lane before ++programming any page. An occupied candidate is skipped permanently within the ++instance. A failed scan stops I/O. Allocation scans have a 500-second limit; ++the block queue allows 600 seconds for the scan and synchronous programs. ++ ++Data records use native type-1 metadata. Once a block's data region is full, ++the driver writes eight summary pages containing native type-1c run entries. ++The old Apple checkpoint and the shared map remain intact. Fresh recovery ++replays completed data records from an open block, or uses a complete summary. ++An interrupted summary can be ignored only when its metadata identifies the ++reserved summary tail and the entire preceding data prefix validates in ++physical and sequence order. Torn or ambiguous metadata stops recovery. ++ ++The recovered allocation floor can exceed the last visible journal record. ++This implementation retains its contiguous journal replay rule, so a mismatch ++keeps the disk read-only and returns ``-EOPNOTSUPP`` before any program. ++Native forward-gap handling requires a separate recovery change with format- ++specific fixtures. A program or map-publication error stops subsequent I/O; a ++fresh loader must determine which complete records reached the medium. ++ ++The block frontend preserves untouched sectors in partial 4 KiB updates and ++groups up to four logical pages per program. Every acknowledged write has ++completed readback. Flush and FUA use that same completion rule. There is no ++erase, discard, checkpoint update, or garbage collection. Allocation returns ++``-ENOSPC`` when verified blank blocks run out, and reads remain available. ++The caller must fence writes to the intended logical partition. This option ++does not qualify sustained writes or all power-interruption cases on hardware. + + Tests + ----- + + ``tools/testing/kunit/configs/apple_sftl.config`` enables the synthetic medium +-tests. They generate VFL replicas and lane tables, plus an SFTL checkpoint +-followed by completed and open data blocks. Tests cover trim ordering and +-allocation floors. Corrupt replicas and sequence gaps must return an error +-with no disk or usable map. Test inputs contain no captured device bytes. ++tests. ``tools/testing/kunit/configs/apple_sftl_write.config`` additionally ++enables the experimental write branches with a synthetic program provider. The ++latter config builds H2FMI through ``COMPILE_TEST`` to satisfy the production ++dependency graph; the fixtures perform no controller I/O. They generate VFL ++replicas and lane tables, plus an SFTL checkpoint followed by completed and ++open data blocks. Tests cover trim ordering and allocation floors. With the ++experimental write option enabled, they program synthetic pages and load a ++fresh map after open and closed blocks, interrupted summaries, and uncertain ++programs. Fixtures also cover complete candidate scans, VFL member ++reservations, space exhaustion, sequence-floor refusal and the final 48-bit ++sequence. Corrupt replicas and sequence gaps must return an error with no disk ++or usable map. Test inputs contain no captured device bytes. + + The format rules were ported from Orchard's P105 Python reference +-(``ftl_map.py``, ``ftl_load.py`` and ``ftl_log.py``), which was checked against +-native iOS logical reads. A kernel object compile or synthetic test result +-does not establish physical-controller timing or qualify a new device layout. ++(``ftl_map.py``, ``ftl_load.py``, ``ftl_log.py`` and ``ftl_append.py``), which ++was checked against native iOS logical reads. A kernel object compile or ++synthetic test result does not establish physical-controller timing or qualify ++a new device layout. +diff --git a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig +index 155136a36..5e4a1e90c 100644 +--- a/drivers/mtd/Kconfig ++++ b/drivers/mtd/Kconfig +@@ -207,7 +207,7 @@ config MTD_PARTITIONED_MASTER + what lies behind the master. + + config MTD_APPLE_SFTL +- tristate "Apple PPN read-only VFL/SwissPPNFTL translation" ++ tristate "Apple PPN VFL/SwissPPNFTL translation" + depends on BLOCK + help + Recover the logical block medium stored in Apple VFL/SwissPPNFTL +@@ -216,16 +216,35 @@ config MTD_APPLE_SFTL + Unsupported checkpoints, missing journal sequences and refresh + requirements stop recovery before a disk is registered. + +- The disk is read-only. This driver performs no program or erase. ++ The disk is read-only unless experimental append writes are enabled. ++ This driver performs no erase. ++ ++config MTD_APPLE_SFTL_WRITE ++ bool "Apple PPN append writes (EXPERIMENTAL)" ++ depends on MTD_APPLE_SFTL && MTD_APPLE_H2FMI_WRITE ++ default n ++ help ++ Append complete physical pages in verified blank blocks while retaining ++ Apple VFL/SFTL metadata and native block summaries. The controller must ++ complete and verify each program before the mapping is published. ++ The shared map and saved-context blocks remain reserved. No erase or ++ reclamation is available; exhausted blank space returns ENOSPC. ++ ++ Recovery and the installer must fence Apple logical partitions. This ++ option is experimental and does not qualify sustained writes or all ++ power-interruption cases. If unsure, say N. + + config MTD_APPLE_SFTL_KUNIT_TEST +- tristate "KUnit tests for Apple VFL/SwissPPNFTL recovery" if !KUNIT_ALL_TESTS ++ tristate "KUnit tests for Apple VFL/SwissPPNFTL" if !KUNIT_ALL_TESTS + depends on KUNIT && MTD_APPLE_SFTL + default KUNIT_ALL_TESTS + help + Synthesize complete VFL/SFTL media and exercise checkpoint recovery, + native block summaries and open-block journal replay. Malformed + replicas and unsupported layouts must fail without storage writes. ++ With experimental append writes enabled, also test complete programs, ++ interrupted summaries, allocation limits and fresh-loader replay. ++ All test media and provider operations are synthetic. + + source "drivers/mtd/chips/Kconfig" + +diff --git a/drivers/mtd/apple-sftl-block.c b/drivers/mtd/apple-sftl-block.c +index bf7a0a108..29a05ae3b 100644 +--- a/drivers/mtd/apple-sftl-block.c ++++ b/drivers/mtd/apple-sftl-block.c +@@ -1,9 +1,10 @@ + // SPDX-License-Identifier: GPL-2.0-only +-/* Read-only block frontend for Apple's VFL/SwissPPNFTL logical medium. */ ++/* Block frontend for Apple's VFL/SwissPPNFTL logical medium. */ + #include + #include + #include + #include ++#include + #include + #include + #include +@@ -28,6 +29,7 @@ MODULE_PARM_DESC(max_io_time_ms, "Maximum reader lifetime in milliseconds"); + struct sftl_disk { + struct apple_ppn *upstream; + struct apple_ppn provider; ++ struct apple_ppn_ops provider_ops; + struct apple_sftl_io_limit limit; + struct apple_sftl *ftl; + struct blk_mq_tag_set tags; +@@ -36,6 +38,7 @@ struct sftl_disk { + u8 *buffer; + unsigned int id; + bool dead; ++ bool writable; + }; + + static int sftl_limited_read(void *priv, unsigned int bus, unsigned int cau, +@@ -53,12 +56,26 @@ static int sftl_limited_read(void *priv, unsigned int bus, unsigned int cau, + return apple_sftl_limit_end(&sftl->limit, ret, ktime_get_ns()); + } + +-static const struct apple_ppn_ops sftl_limited_ops = { +- .read_page = sftl_limited_read, +-}; ++static int sftl_limited_program(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, ++ const u8 *data, const u8 *oob) ++{ ++ struct sftl_disk *sftl = priv; ++ int ret; ++ ++ ret = apple_sftl_limit_check(&sftl->limit, ktime_get_ns()); ++ if (ret) ++ return ret; ++ ret = sftl->upstream->ops->program_page(sftl->upstream->priv, bus, cau, ++ block, page, data, oob); ++ return apple_sftl_limit_end(&sftl->limit, ret, ktime_get_ns()); ++} + + static DEFINE_IDA(sftl_disk_ids); + static int sftl_major; ++static bool write_enable; ++module_param(write_enable, bool, 0444); ++MODULE_PARM_DESC(write_enable, "Allow experimental logical append writes (default: off)"); + + /* The blocking queue and this lock keep the preallocated buffer exclusive. */ + static int sftl_read_request(struct sftl_disk *sftl, struct request *request) +@@ -96,6 +113,69 @@ static int sftl_read_request(struct sftl_disk *sftl, struct request *request) + return 0; + } + ++/* Preserve untouched sectors at both ends and collect up to four 4 KiB ++ * pages per physical program. Request segments can split a logical page. ++ */ ++static int sftl_write_request(struct sftl_disk *sftl, struct request *request) ++{ ++ struct req_iterator iterator; ++ struct bio_vec bvec; ++ u64 position = (u64)blk_rq_pos(request) << SECTOR_SHIFT; ++ u32 cached = U32_MAX, first = 0; ++ unsigned int count = 0; ++ int ret; ++ ++ if (request->cmd_flags & REQ_PREFLUSH) { ++ ret = apple_sftl_flush(sftl->ftl); ++ if (ret) ++ return ret; ++ } ++ rq_for_each_segment(bvec, request, iterator) { ++ unsigned int done = 0; ++ ++ while (done < bvec.bv_len) { ++ u32 logical = position >> 12; ++ unsigned int offset = position & (SFTL_LOGICAL_BYTES - 1); ++ unsigned int length = min(bvec.bv_len - done, ++ SFTL_LOGICAL_BYTES - offset); ++ void *mapped; ++ ++ if (logical != cached) { ++ if (count == 4) { ++ ret = apple_sftl_write(sftl->ftl, first, ++ sftl->buffer, count); ++ if (ret) ++ return ret; ++ count = 0; ++ } ++ if (!count) ++ first = logical; ++ ret = apple_sftl_read(sftl->ftl, logical, ++ sftl->buffer + count * SFTL_LOGICAL_BYTES); ++ if (ret) ++ return ret; ++ count++; ++ cached = logical; ++ } ++ mapped = kmap_local_page(bvec.bv_page); ++ memcpy(sftl->buffer + (count - 1) * SFTL_LOGICAL_BYTES + offset, ++ mapped + bvec.bv_offset + done, length); ++ kunmap_local(mapped); ++ done += length; ++ position += length; ++ } ++ } ++ if (count) { ++ ret = apple_sftl_write(sftl->ftl, first, sftl->buffer, count); ++ if (ret) ++ return ret; ++ } ++ /* Every completed write is already verified on media. FUA shares that ++ * completion rule and never acknowledges a volatile cached page. ++ */ ++ return request->cmd_flags & REQ_FUA ? apple_sftl_flush(sftl->ftl) : 0; ++} ++ + static blk_status_t sftl_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *data) + { +@@ -105,19 +185,21 @@ static blk_status_t sftl_queue_rq(struct blk_mq_hw_ctx *hctx, + blk_status_t status; + int ret = 0; + +- blk_mq_start_request(request); + mutex_lock(&sftl->io_lock); ++ blk_mq_start_request(request); + if (sftl->dead) { + ret = -ENODEV; + } else if ((ret = apple_sftl_limit_check(&sftl->limit, ktime_get_ns()))) { + /* Includes logical pages which need no provider transaction. */ + } else if (req_op(request) == REQ_OP_FLUSH) { +- /* A read-only device has no buffered writes to flush. */ +- } else if (req_op(request) != REQ_OP_READ) { ++ ret = apple_sftl_flush(sftl->ftl); ++ } else if (req_op(request) != REQ_OP_READ && req_op(request) != REQ_OP_WRITE) { + ret = -EOPNOTSUPP; + } else if (blk_rq_pos(request) > capacity || + blk_rq_sectors(request) > capacity - blk_rq_pos(request)) { + ret = -EIO; ++ } else if (req_op(request) == REQ_OP_WRITE) { ++ ret = sftl->writable ? sftl_write_request(sftl, request) : -EROFS; + } else { + ret = sftl_read_request(sftl, request); + } +@@ -148,20 +230,24 @@ static int sftl_disk_probe(struct platform_device *pdev) + struct gendisk *disk; + int ret, id; + +- if (!provider) ++ if (!provider || !provider->ops || !provider->ops->read_page) + return -EINVAL; + sftl = kzalloc(sizeof(*sftl), GFP_KERNEL); + if (!sftl) + return -ENOMEM; + mutex_init(&sftl->io_lock); +- sftl->buffer = kmalloc(SFTL_LOGICAL_BYTES, GFP_KERNEL); ++ sftl->buffer = kmalloc(APPLE_PPN_PAGE_SIZE, GFP_KERNEL); + if (!sftl->buffer) { + ret = -ENOMEM; + goto free_state; + } + sftl->upstream = provider; + sftl->provider = *provider; +- sftl->provider.ops = &sftl_limited_ops; ++ sftl->provider_ops = *provider->ops; ++ sftl->provider_ops.read_page = sftl_limited_read; ++ if (provider->ops->program_page) ++ sftl->provider_ops.program_page = sftl_limited_program; ++ sftl->provider.ops = &sftl->provider_ops; + sftl->provider.priv = sftl; + ret = apple_sftl_limit_start(&sftl->limit, max_page_reads, + max_io_time_ms, ktime_get_ns()); +@@ -179,6 +265,10 @@ static int sftl_disk_probe(struct platform_device *pdev) + ret = -EINVAL; + goto free_ftl; + } ++ sftl->writable = write_enable && IS_ENABLED(CONFIG_MTD_APPLE_SFTL_WRITE) && ++ provider->ops->program_page && info.sequence == info.journal_sequence; ++ if (sftl->writable) ++ limits.features |= BLK_FEAT_FUA; + id = ida_alloc_range(&sftl_disk_ids, 0, MINORMASK / SFTL_DISK_MINORS, + GFP_KERNEL); + if (id < 0) { +@@ -190,6 +280,11 @@ static int sftl_disk_probe(struct platform_device *pdev) + BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING); + if (ret) + goto free_id; ++ /* First use verifies every page in a candidate block. Leave time for ++ * the bounded 500-second scan and synchronous data/footer programs. ++ */ ++ if (sftl->writable) ++ sftl->tags.timeout = 600U * HZ; + disk = blk_mq_alloc_disk(&sftl->tags, &limits, sftl); + if (IS_ERR(disk)) { + ret = PTR_ERR(disk); +@@ -203,13 +298,14 @@ static int sftl_disk_probe(struct platform_device *pdev) + disk->private_data = sftl; + snprintf(disk->disk_name, DISK_NAME_LEN, "applesftl%d", id); + set_capacity(disk, (sector_t)info.disk_pages * (SFTL_LOGICAL_BYTES / SECTOR_SIZE)); +- set_disk_ro(disk, true); ++ set_disk_ro(disk, !sftl->writable); + platform_set_drvdata(pdev, sftl); + ret = device_add_disk(&pdev->dev, disk, NULL); + if (ret) + goto put_disk; +- dev_info(&pdev->dev, "%s: read-only, %u logical pages, sequence %llu\n", +- disk->disk_name, info.disk_pages, info.sequence); ++ dev_info(&pdev->dev, "%s: %s, %u logical pages, sequence %llu\n", ++ disk->disk_name, sftl->writable ? "experimental append writes" : "read-only", ++ info.disk_pages, info.sequence); + return 0; + + put_disk: +@@ -282,6 +378,6 @@ static void __exit sftl_exit(void) + } + module_exit(sftl_exit); + +-MODULE_DESCRIPTION("Apple VFL/SwissPPNFTL read-only block driver"); ++MODULE_DESCRIPTION("Apple VFL/SwissPPNFTL block driver"); + MODULE_LICENSE("GPL"); + MODULE_ALIAS("platform:" SFTL_BLOCK_NAME); +diff --git a/drivers/mtd/apple-sftl-core.c b/drivers/mtd/apple-sftl-core.c +index ba90916f2..6f16ecd54 100644 +--- a/drivers/mtd/apple-sftl-core.c ++++ b/drivers/mtd/apple-sftl-core.c +@@ -1,14 +1,15 @@ + // SPDX-License-Identifier: GPL-2.0-only + /* +- * Apple VFL/SwissPPNFTL read-only recovery for the measured 64 GB PPN layout. ++ * Apple VFL/SwissPPNFTL recovery and append for the measured 64 GB PPN layout. + * + * Format and replay rules are ported from Orchard's P105 Python reference +- * (ftl_map.py, ftl_load.py and ftl_log.py). That implementation was checked ++ * (ftl_map.py, ftl_load.py, ftl_log.py and ftl_append.py). It was checked + * against native iOS logical reads. This code contains no captured firmware + * or device data. Unsupported formats fail before a block device is created. + */ + #include + #include ++#include + #include + #include + #include +@@ -34,6 +35,9 @@ + #define SFTL_META 16U + #define SFTL_CHUNK 4096U + #define SFTL_TRIM 0xffff0001U ++#define SFTL_FOOTER_CHUNKS 32U ++#define SFTL_CLAIM_TIMEOUT (500U * HZ) ++#define SFTL_FOOTER_BYTES (SFTL_FOOTER_CHUNKS * SFTL_CHUNK) + + struct sftl_run { + struct rb_node node; +@@ -62,7 +66,15 @@ struct apple_sftl { + u32 nr_runs; + u16 lanes[SFTL_BLOCKS]; + DECLARE_BITMAP(reserved, SFTL_BLOCKS); +- u64 checkpoint_sequence, sequence; ++ DECLARE_BITMAP(vfl_reserved, SFTL_BLOCKS); ++ u64 checkpoint_sequence, sequence, journal_sequence; ++ DECLARE_BITMAP(candidates, SFTL_BLOCKS); ++ DECLARE_BITMAP(used, SFTL_BLOCKS); ++ u32 next_candidate, append_block, append_position, append_capacity; ++ u32 footer_runs; ++ u64 append_base; ++ u8 *footer; ++ bool writer_prepared; + u8 *page; + u8 oob[APPLE_PPN_OOB_SIZE]; + struct mutex lock; /* serializes reads and the shared page buffer */ +@@ -375,6 +387,7 @@ static int sftl_vfl(struct apple_sftl *sftl) + ret = -EBADMSG; + goto out; + } ++ __set_bit(members[member].block, sftl->vfl_reserved); + } + for (member = 0; member < 8; member++) { + for (page = 0; page < 64; page++) { +@@ -646,7 +659,7 @@ static int sftl_footer(struct apple_sftl *sftl, const u8 *head, u32 block, + span = get_unaligned_le32(footer_meta + 8); + if (chunks < 4 || chunks > 32 || chunks % 4 || footer_meta[1] >> 1 != chunks || + get_unaligned_le16(footer_meta + 14) != 0xffff || +- base != sftl_sequence(head) || !span || base + span > SFTL_SEQUENCE) ++ base != sftl_sequence(head) || !span || base + span > SFTL_SEQUENCE + 1) + return -EBADMSG; + *floor = max(*floor, base + span - 1); + if (base + span - 1 <= sftl->checkpoint_sequence) +@@ -722,6 +735,25 @@ static int sftl_footer(struct apple_sftl *sftl, const u8 *head, u32 block, + return ret; + } + ++static bool sftl_partial_footer(struct apple_sftl *sftl, const u8 *head, ++ u32 position, u32 pages) ++{ ++ const u8 *meta = sftl->oob; ++ unsigned int chunk; ++ ++ if (position < pages - SFTL_FOOTER_CHUNKS / 4 || ++ meta[0] != 0x1c || meta[1] != SFTL_FOOTER_CHUNKS * 2 || ++ sftl_sequence(meta) != sftl_sequence(head) || ++ get_unaligned_le32(meta + 8) != pages * 4 - SFTL_FOOTER_CHUNKS || ++ get_unaligned_le16(meta + 12) != SFTL_FOOTER_CHUNKS || ++ get_unaligned_le16(meta + 14) != 0xffff) ++ return false; ++ for (chunk = 1; chunk < 4; chunk++) ++ if (memcmp(meta, sftl->oob + chunk * SFTL_META, SFTL_META)) ++ return false; ++ return true; ++} ++ + static int sftl_journal(struct apple_sftl *sftl, const u8 *heads) + { + struct sftl_records journal = {}; +@@ -732,6 +764,7 @@ static int sftl_journal(struct apple_sftl *sftl, const u8 *heads) + + for (block = 1; block < SFTL_SUBBLOCKS; block++) { + const u8 *head = heads + block * SFTL_META; ++ bool complete_prefix = true, partial_footer = false; + + if (head[0] != 1 && head[0] != 2) + continue; +@@ -759,6 +792,33 @@ static int sftl_journal(struct apple_sftl *sftl, const u8 *heads) + ret = -EBADMSG; + goto out; + } ++ if (position < pages - SFTL_FOOTER_CHUNKS / 4) { ++ for (chunk = 0; chunk < 4; chunk++) { ++ const u8 *meta = sftl->oob + chunk * SFTL_META; ++ u64 sequence = sftl_sequence(head) + position * 4 + chunk; ++ ++ if (meta[0] != 1 || meta[1] || ++ sftl_sequence(meta) != sequence || ++ get_unaligned_le32(meta + 12) != sequence >> 16) ++ complete_prefix = false; ++ } ++ } ++ /* An incomplete final summary leaves the last page erased. ++ * Its full contiguous data prefix must still validate directly. ++ * The partially written summary payload is never trusted. ++ */ ++ if (sftl_partial_footer(sftl, head, position, pages)) { ++ if (!complete_prefix) { ++ ret = -EBADMSG; ++ goto out; ++ } ++ partial_footer = true; ++ continue; ++ } ++ if (partial_footer && !sftl_erased(sftl->oob, sizeof(sftl->oob))) { ++ ret = -EBADMSG; ++ goto out; ++ } + for (chunk = 0; chunk < 4; chunk++) { + ret = sftl_add_journal(&journal, sftl->oob + chunk * SFTL_META, + block * SFTL_STRIDE + position * 4 + chunk, +@@ -808,6 +868,7 @@ static int sftl_journal(struct apple_sftl *sftl, const u8 *heads) + } + sftl->sequence = sequence; + } ++ sftl->journal_sequence = expected - 1; + sftl->sequence = max(sftl->sequence, floor); + ret = 0; + out: +@@ -835,6 +896,8 @@ int apple_sftl_load(struct apple_ppn *provider, struct apple_sftl **out) + return -ENOMEM; + sftl->ppn = provider; + sftl->runs = RB_ROOT; ++ sftl->append_block = U32_MAX; ++ sftl->next_candidate = 1; + mutex_init(&sftl->lock); + sftl->page = kmalloc(APPLE_PPN_PAGE_SIZE, GFP_KERNEL); + heads = kcalloc(SFTL_BLOCKS, SFTL_META, GFP_KERNEL); +@@ -857,6 +920,7 @@ int apple_sftl_load(struct apple_ppn *provider, struct apple_sftl **out) + ret = -EBADMSG; + goto failed; + } ++ __set_bit(block, sftl->candidates); + } else if (sftl->oob[0] == 0x1f || sftl->oob[0] == 1 || sftl->oob[0] == 2) { + memcpy(heads + block * SFTL_META, sftl->oob, SFTL_META); + } else { +@@ -890,6 +954,7 @@ void apple_sftl_free(struct apple_sftl *sftl) + rb_erase(node, &sftl->runs); + kfree(rb_entry(node, struct sftl_run, node)); + } ++ kvfree(sftl->footer); + kfree(sftl->page); + kfree(sftl); + } +@@ -938,6 +1003,244 @@ int apple_sftl_read(struct apple_sftl *sftl, u32 logical_page, u8 *data) + } + EXPORT_SYMBOL_GPL(apple_sftl_read); + ++/* Caller holds sftl->lock. Every candidate is checked in full before use. */ ++static int sftl_writer_prepare(struct apple_sftl *sftl) ++{ ++ struct rb_node *node; ++ u32 block; ++ ++ if (sftl->writer_prepared) ++ return 0; ++ if (sftl->journal_sequence != sftl->sequence) ++ return -EOPNOTSUPP; ++ if (!sftl->ppn->ops->program_page) ++ return -EROFS; ++ sftl->footer = kvzalloc(SFTL_FOOTER_BYTES, GFP_KERNEL); ++ if (!sftl->footer) ++ return -ENOMEM; ++ for (block = 0; block < SFTL_BLOCKS; block++) ++ if (test_bit(block, sftl->reserved) || test_bit(block, sftl->vfl_reserved)) ++ __set_bit(block, sftl->used); ++ __set_bit(0, sftl->used); ++ __set_bit(SFTL_SUBBLOCKS, sftl->used); ++ __set_bit(SFTL_SUBBLOCKS + 1, sftl->used); ++ for (node = rb_first(&sftl->runs); node; node = rb_next(node)) { ++ struct sftl_run *run = rb_entry(node, struct sftl_run, node); ++ u32 end; ++ ++ if (run->provider == SFTL_UNMAPPED) ++ continue; ++ end = (run->provider + run->end - run->start - 1) / SFTL_STRIDE; ++ for (block = run->provider / SFTL_STRIDE; block <= end; block++) ++ __set_bit(block, sftl->used); ++ } ++ sftl->writer_prepared = true; ++ return 0; ++} ++ ++static int sftl_claim_block(struct apple_sftl *sftl) ++{ ++ unsigned long deadline = jiffies + SFTL_CLAIM_TIMEOUT; ++ u32 block; ++ ++ while (sftl->next_candidate < SFTL_SUBBLOCKS) { ++ u32 capacity, position; ++ unsigned int chunk; ++ bool blank = true; ++ int ret; ++ ++ block = sftl->next_candidate++; ++ if (!test_bit(block, sftl->candidates) || ++ test_bit(block, sftl->used) || !sftl->lanes[block]) ++ continue; ++ /* A checked or rejected candidate is never reused in this instance. */ ++ __set_bit(block, sftl->used); ++ capacity = hweight16(sftl->lanes[block]) * 128 * 4; ++ for (position = 0; position < capacity; position += 4) { ++ if (time_after_eq(jiffies, deadline)) { ++ sftl->failed = true; ++ return -ETIMEDOUT; ++ } ++ ret = sftl_read_address(sftl, block * SFTL_STRIDE + position, &chunk); ++ if (ret) { ++ sftl->failed = true; ++ return ret; ++ } ++ if (!sftl_erased(sftl->page, APPLE_PPN_PAGE_SIZE) || ++ !sftl_erased(sftl->oob, APPLE_PPN_OOB_SIZE)) { ++ blank = false; ++ break; ++ } ++ } ++ if (!blank) ++ continue; ++ sftl->append_block = block; ++ sftl->append_capacity = capacity - SFTL_FOOTER_CHUNKS; ++ sftl->append_position = 0; ++ sftl->append_base = sftl->sequence + 1; ++ sftl->footer_runs = 0; ++ memset(sftl->footer, 0, SFTL_FOOTER_BYTES); ++ return 0; ++ } ++ return -ENOSPC; ++} ++ ++static int sftl_program_address(struct apple_sftl *sftl, u32 address, ++ const u8 *data, const u8 *oob) ++{ ++ unsigned int bus, cau, block, page, chunk; ++ int ret; ++ ++ ret = sftl_translate(sftl, address, &bus, &cau, &block, &page, &chunk); ++ if (ret) ++ return ret; ++ if (chunk) ++ return -EINVAL; ++ return sftl->ppn->ops->program_page(sftl->ppn->priv, bus, cau, ++ block, page, data, oob); ++} ++ ++/* A short request repeats its final logical page to fill the program unit. */ ++static void sftl_data_meta(u8 *meta, u64 sequence, u32 logical_page) ++{ ++ put_unaligned_le64(sequence << 16 | 1, meta); ++ put_unaligned_le32(logical_page, meta + 8); ++ put_unaligned_le32(sequence >> 16, meta + 12); ++} ++ ++static void sftl_append_summary(struct apple_sftl *sftl, u64 sequence, u32 key) ++{ ++ u32 relative = sequence - sftl->append_base; ++ u32 high = sequence >> 16; ++ u8 *entry; ++ ++ if (sftl->footer_runs) { ++ u32 length; ++ ++ entry = sftl->footer + (sftl->footer_runs - 1) * SFTL_META; ++ length = get_unaligned_le32(entry + 12); ++ if (key < 0xffff0000 && get_unaligned_le32(entry + 4) == high && ++ get_unaligned_le32(entry) + length == relative && ++ get_unaligned_le32(entry + 8) + length == key) { ++ put_unaligned_le32(length + 1, entry + 12); ++ return; ++ } ++ } ++ entry = sftl->footer + sftl->footer_runs++ * SFTL_META; ++ put_unaligned_le32(relative, entry); ++ put_unaligned_le32(high, entry + 4); ++ put_unaligned_le32(key, entry + 8); ++ put_unaligned_le32(1, entry + 12); ++} ++ ++static int sftl_close_block(struct apple_sftl *sftl) ++{ ++ u8 meta[SFTL_META]; ++ unsigned int chunk, index; ++ int ret; ++ ++ put_unaligned_le64(sftl->append_base << 16 | 0x401c, meta); ++ put_unaligned_le32(sftl->append_position, meta + 8); ++ put_unaligned_le16(SFTL_FOOTER_CHUNKS, meta + 12); ++ put_unaligned_le16(0xffff, meta + 14); ++ for (chunk = 0; chunk < 4; chunk++) ++ memcpy(sftl->oob + chunk * SFTL_META, meta, SFTL_META); ++ for (index = 0; index < SFTL_FOOTER_CHUNKS / 4; index++) { ++ u32 address = sftl->append_block * SFTL_STRIDE + ++ sftl->append_capacity + index * 4; ++ ++ ret = sftl_program_address(sftl, address, ++ sftl->footer + index * APPLE_PPN_PAGE_SIZE, ++ sftl->oob); ++ if (ret) ++ return ret; ++ } ++ sftl->append_block = U32_MAX; ++ return 0; ++} ++ ++int apple_sftl_write(struct apple_sftl *sftl, u32 logical_page, ++ const u8 *data, unsigned int count) ++{ ++ u64 first_sequence; ++ u32 address; ++ unsigned int chunk; ++ int ret; ++ ++ if (!IS_ENABLED(CONFIG_MTD_APPLE_SFTL_WRITE)) ++ return -EROFS; ++ if (!sftl || !data || !count || count > 4 || logical_page >= SFTL_DISK_PAGES || ++ count > SFTL_DISK_PAGES - logical_page) ++ return -EINVAL; ++ mutex_lock(&sftl->lock); ++ if (sftl->failed) { ++ ret = -EIO; ++ goto out; ++ } ++ if (sftl->sequence > SFTL_SEQUENCE - 4) { ++ ret = -EOVERFLOW; ++ goto out; ++ } ++ ret = sftl_writer_prepare(sftl); ++ if (ret) ++ goto out; ++ if (sftl->append_block == U32_MAX) { ++ ret = sftl_claim_block(sftl); ++ if (ret) ++ goto out; ++ } ++ address = sftl->append_block * SFTL_STRIDE + sftl->append_position; ++ first_sequence = sftl->sequence + 1; ++ for (chunk = 0; chunk < 4; chunk++) { ++ unsigned int index = min(chunk, count - 1); ++ ++ memcpy(sftl->page + chunk * SFTL_CHUNK, ++ data + index * SFTL_CHUNK, SFTL_CHUNK); ++ sftl_data_meta(sftl->oob + chunk * SFTL_META, ++ first_sequence + chunk, logical_page + index); ++ } ++ ret = sftl_program_address(sftl, address, sftl->page, sftl->oob); ++ if (ret) ++ goto failed; ++ for (chunk = 0; chunk < 4; chunk++) { ++ u32 key = logical_page + min(chunk, count - 1); ++ ++ ret = sftl_replace(sftl, key, 1, address + chunk); ++ if (ret) ++ goto failed; ++ sftl_append_summary(sftl, first_sequence + chunk, key); ++ } ++ sftl->sequence += 4; ++ sftl->journal_sequence = sftl->sequence; ++ sftl->append_position += 4; ++ if (sftl->append_position == sftl->append_capacity) { ++ ret = sftl_close_block(sftl); ++ if (ret) ++ goto failed; ++ } ++ goto out; ++failed: ++ sftl->failed = true; ++out: ++ mutex_unlock(&sftl->lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_write); ++ ++int apple_sftl_flush(struct apple_sftl *sftl) ++{ ++ int ret; ++ ++ if (!sftl) ++ return -EINVAL; ++ mutex_lock(&sftl->lock); ++ /* Each program is complete and verified before apple_sftl_write returns. */ ++ ret = sftl->failed ? -EIO : 0; ++ mutex_unlock(&sftl->lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_flush); ++ + void apple_sftl_get_info(struct apple_sftl *sftl, struct apple_sftl_info *info) + { + info->logical_pages = SFTL_LPAGES; +@@ -945,5 +1248,6 @@ void apple_sftl_get_info(struct apple_sftl *sftl, struct apple_sftl_info *info) + info->nr_runs = sftl->nr_runs; + info->checkpoint_sequence = sftl->checkpoint_sequence; + info->sequence = sftl->sequence; ++ info->journal_sequence = sftl->journal_sequence; + } + EXPORT_SYMBOL_GPL(apple_sftl_get_info); +diff --git a/drivers/mtd/apple-sftl-test.c b/drivers/mtd/apple-sftl-test.c +index 88308aadb..e89eee42d 100644 +--- a/drivers/mtd/apple-sftl-test.c ++++ b/drivers/mtd/apple-sftl-test.c +@@ -37,10 +37,28 @@ enum sftl_fault { + SFTL_RUNTIME_FAILURE, + }; + ++struct sftl_test_page { ++ u8 data[APPLE_PPN_PAGE_SIZE]; ++ u8 oob[APPLE_PPN_OOB_SIZE]; ++}; ++ + struct sftl_fixture { ++ struct kunit *test; + struct apple_ppn provider; + enum sftl_fault fault; + unsigned int reads; ++ unsigned int programs; ++ unsigned int fail_program; ++ unsigned int tear_program; ++ unsigned int program_outside; ++ bool uncertain_program; ++ bool no_candidates; ++ bool dirty_candidate; ++ bool vfl_candidate; ++ bool full_lanes; ++ u16 program_lanes; ++ u64 sequence_bias; ++ struct sftl_test_page **written[5]; + }; + + static void sftl_test_meta(u8 *meta, u8 kind, u8 flags, u64 seq, u32 key) +@@ -55,7 +73,7 @@ static void sftl_test_meta(u8 *meta, u8 kind, u8 flags, u64 seq, u32 key) + put_unaligned_le32(seq >> 16, meta + 12); + } + +-static void sftl_test_members(u8 *data, enum sftl_fault fault) ++static void sftl_test_members(struct sftl_fixture *f, u8 *data) + { + unsigned int i; + +@@ -64,8 +82,10 @@ static void sftl_test_members(u8 *data, enum sftl_fault fault) + data[0x102 + i * 4] = i; + data[0x103 + i * 4] = 0; + } +- if (fault == SFTL_BOOTSTRAP_DUPLICATE) ++ if (f->fault == SFTL_BOOTSTRAP_DUPLICATE) + data[0x106] = 0; ++ if (f->vfl_candidate) ++ put_unaligned_le16(2056, data + 0x100); + } + + static int sftl_test_context(struct sftl_fixture *f, unsigned int lane, +@@ -80,7 +100,7 @@ static int sftl_test_context(struct sftl_fixture *f, unsigned int lane, + put_unaligned_le16(16, data + 10); + put_unaligned_le16(2086, data + 12); + put_unaligned_le16(2086, data + 14); +- sftl_test_members(data, f->fault); ++ sftl_test_members(f, data); + for (i = 0; i < 16; i++) + put_unaligned_le16(i + 1, data + 0x200 + i * 32); + oob[0] = 0x20; +@@ -96,6 +116,16 @@ static int sftl_test_context(struct sftl_fixture *f, unsigned int lane, + if (page <= 2) + for (i = 1; i <= 4; i++) + put_unaligned_le16(0, data + i * 2); ++ /* Three initially blank append candidates, single-lane by default. */ ++ if (page == 1 && !f->no_candidates) ++ for (i = 5; i <= 7; i++) ++ put_unaligned_le16(0, data + i * 2); ++ if (f->full_lanes) ++ put_unaligned_le16(0, data + 5 * 2); ++ if (page == 1 && f->vfl_candidate) { ++ put_unaligned_le16(0, data + 2056 * 2); ++ put_unaligned_le16(0, data + 2057 * 2); ++ } + oob[0] = 0x20; + oob[1] = page - 1; + oob[2] = 0; +@@ -116,7 +146,7 @@ static void sftl_test_checkpoint(struct sftl_fixture *f, u8 *data, u8 *oob) + memset(data, 0, 4096); + put_unaligned_le32(1, data); + put_unaligned_le32(1, data + 4); +- sftl_test_meta(oob, 0x1f, 1, 100, 0); ++ sftl_test_meta(oob, 0x1f, 1, 100 + f->sequence_bias, 0); + if (f->fault == SFTL_INCOMPLETE_CONTEXT) + return; + if (f->fault == SFTL_INACTIVE_ADDRESS) +@@ -127,12 +157,12 @@ static void sftl_test_checkpoint(struct sftl_fixture *f, u8 *data, u8 *oob) + put_unaligned_le32(1, ranges + 12); + put_unaligned_le32(0x1ffffff, ranges + 16); + put_unaligned_le32(TEST_LOGICAL_PAGES - 1, ranges + 20); +- sftl_test_meta(oob + 16, 0x1f, 4, TEST_CHECKPOINT, 0); ++ sftl_test_meta(oob + 16, 0x1f, 4, TEST_CHECKPOINT + f->sequence_bias, 0); + } + + static void sftl_test_open(struct sftl_fixture *f, u8 *data, u8 *oob) + { +- u64 first = TEST_OPEN_FIRST; ++ u64 first = TEST_OPEN_FIRST + f->sequence_bias; + unsigned int i; + + if (f->fault == SFTL_JOURNAL_GAP) +@@ -146,7 +176,7 @@ static void sftl_test_open(struct sftl_fixture *f, u8 *data, u8 *oob) + if (f->fault == SFTL_TRIM) { + memset(data, 0, 4096); + put_unaligned_le32(1, data); +- put_unaligned_le32(0, data + 4); ++ put_unaligned_le32((TEST_CLOSED_FIRST + f->sequence_bias) >> 16, data + 4); + put_unaligned_le32(1, data + 8); + sftl_test_meta(oob, 2, 0, first, 0xffff0001); + } +@@ -163,19 +193,19 @@ static void sftl_test_closed(struct sftl_fixture *f, unsigned int position, + + memset(data + i * 4096, (slot + 1) & 0xff, 4096); + sftl_test_meta(oob + i * 16, 1, 0, +- TEST_CLOSED_FIRST + slot, slot + 1); ++ TEST_CLOSED_FIRST + f->sequence_bias + slot, slot + 1); + } + } else { + u32 span = TEST_CLOSED_CHUNKS; + + memset(data, 0, 16384); + put_unaligned_le32(0, data); +- put_unaligned_le32(0, data + 4); ++ put_unaligned_le32((TEST_CLOSED_FIRST + f->sequence_bias) >> 16, data + 4); + put_unaligned_le32(1, data + 8); + put_unaligned_le32(TEST_CLOSED_CHUNKS, data + 12); + if (f->fault == SFTL_ALLOCATION_FLOOR) + span += 8; +- sftl_test_meta(oob, 0x1c, 8, TEST_CLOSED_FIRST, span); ++ sftl_test_meta(oob, 0x1c, 8, TEST_CLOSED_FIRST + f->sequence_bias, span); + put_unaligned_le16(f->fault == SFTL_BAD_FOOTER ? 2 : 4, oob + 12); + put_unaligned_le16(0xffff, oob + 14); + for (i = 1; i < 4; i++) +@@ -185,6 +215,24 @@ static void sftl_test_closed(struct sftl_fixture *f, unsigned int position, + } + } + ++static int sftl_test_candidate_slot(unsigned int block) ++{ ++ if (block >= 5 && block <= 7) ++ return block - 5; ++ if (block == 2056 || block == 2057) ++ return block - 2053; ++ return -1; ++} ++ ++static struct sftl_test_page *sftl_test_written(struct sftl_fixture *f, ++ unsigned int slot, unsigned int lane, ++ unsigned int page) ++{ ++ if (!f->written[slot]) ++ return NULL; ++ return f->written[slot][lane * 128 + page]; ++} ++ + static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, + unsigned int block, unsigned int page, + bool context, u8 *data, u8 *oob, u8 *status) +@@ -192,6 +240,7 @@ static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, + struct sftl_fixture *f = priv; + unsigned int lane = cau * 2 + bus; + unsigned int position = page * 2 + lane; ++ int slot = sftl_test_candidate_slot(block); + + f->reads++; + if (bus >= 2 || cau >= 8 || block >= 2088 || page >= (context ? 64 : 128)) +@@ -200,13 +249,26 @@ static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, + memset(oob, 0xff, 64); + *status = 0x49; + if (context) { +- if (block == 2087) ++ if (block == 2087 || (f->vfl_candidate && block == 2056 && !lane)) + return sftl_test_context(f, lane, page, data, oob, status); + return 0; + } + if (block == 3 && (f->fault == SFTL_RUNTIME_FAILURE || + (!position && f->fault == SFTL_READ_FAILURE))) + return -EIO; ++ if (f->dirty_candidate && block == 5 && !lane && page == 127) { ++ memset(data, 'D', APPLE_PPN_PAGE_SIZE); ++ *status = 0x40; ++ return 0; ++ } ++ if (slot >= 0 && sftl_test_written(f, slot, lane, page)) { ++ struct sftl_test_page *written = sftl_test_written(f, slot, lane, page); ++ ++ memcpy(data, written->data, APPLE_PPN_PAGE_SIZE); ++ memcpy(oob, written->oob, APPLE_PPN_OOB_SIZE); ++ *status = 0x40; ++ return 0; ++ } + if (lane > 1) + return 0; + if (block == 1 && !position) { +@@ -218,13 +280,14 @@ static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, + + for (i = 0; i < 4; i++) { + memset(data + i * 4096, 'H' + i, 4096); +- sftl_test_meta(oob + i * 16, 1, 0, TEST_OPEN_FIRST + 4 + i, 3); ++ sftl_test_meta(oob + i * 16, 1, 0, ++ TEST_OPEN_FIRST + f->sequence_bias + 4 + i, 3); + } + } else if (block == 3) { + sftl_test_closed(f, position, data, oob); + } else if (block == 4 && !position) { + memset(data, 'C', 4096); +- sftl_test_meta(oob, 1, 0, 10, 0); ++ sftl_test_meta(oob, 1, 0, 10 + f->sequence_bias, 0); + } else { + return 0; + } +@@ -234,8 +297,54 @@ static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, + return 0; + } + ++#if IS_ENABLED(CONFIG_MTD_APPLE_SFTL_WRITE) ++static int sftl_test_program_page(void *priv, unsigned int bus, unsigned int cau, ++ unsigned int block, unsigned int page, ++ const u8 *data, const u8 *oob) ++{ ++ struct sftl_fixture *f = priv; ++ struct sftl_test_page *written; ++ unsigned int lane = cau * 2 + bus; ++ int slot = sftl_test_candidate_slot(block); ++ bool failure; ++ ++ f->programs++; ++ if (bus >= 2 || cau >= 8 || slot < 0 || page >= 128 || ++ (lane && (!f->full_lanes || block != 5))) { ++ f->program_outside++; ++ return -EPERM; ++ } ++ if (sftl_test_written(f, slot, lane, page)) ++ return -EEXIST; ++ failure = f->programs == f->fail_program; ++ if (failure && !f->uncertain_program) ++ return -EIO; ++ if (!f->written[slot]) { ++ f->written[slot] = kunit_kcalloc(f->test, 16 * 128, ++ sizeof(*f->written[slot]), GFP_KERNEL); ++ if (!f->written[slot]) ++ return -ENOMEM; ++ } ++ written = kunit_kmalloc(f->test, sizeof(*written), GFP_KERNEL); ++ if (!written) ++ return -ENOMEM; ++ memcpy(written->data, data, APPLE_PPN_PAGE_SIZE); ++ memcpy(written->oob, oob, APPLE_PPN_OOB_SIZE); ++ f->written[slot][lane * 128 + page] = written; ++ f->program_lanes |= 1U << lane; ++ if (f->programs == f->tear_program) { ++ written->oob[63] ^= 1; ++ return -EIO; ++ } ++ return failure ? -EIO : 0; ++} ++#endif ++ + static const struct apple_ppn_ops sftl_test_ops = { + .read_page = sftl_test_read_page, ++#if IS_ENABLED(CONFIG_MTD_APPLE_SFTL_WRITE) ++ .program_page = sftl_test_program_page, ++#endif + }; + + static struct sftl_fixture *sftl_test_fixture(struct kunit *test, +@@ -245,6 +354,7 @@ static struct sftl_fixture *sftl_test_fixture(struct kunit *test, + + if (!f) + return NULL; ++ f->test = test; + f->fault = fault; + f->provider.ops = &sftl_test_ops; + f->provider.priv = f; +@@ -471,12 +581,473 @@ static void sftl_test_invalid_reader_policy(struct kunit *test) + KUNIT_EXPECT_EQ(test, apple_sftl_limit_start(&limit, 1, 1, ~0ULL), -EINVAL); + } + ++#if IS_ENABLED(CONFIG_MTD_APPLE_SFTL_WRITE) ++#define TEST_APPEND_FIRST (TEST_OPEN_FIRST + 4) ++#define TEST_APPEND_BASE 2000U ++#define TEST_APPEND_PROGRAMS 120U ++ ++static void sftl_test_append_replay_open(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ struct apple_sftl_info info; ++ u8 *data = kunit_kmalloc(test, 16384, GFP_KERNEL); ++ u8 *readback = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ unsigned int chunk; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_NOT_NULL(test, readback); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ memset(data, 'W', 4096); ++ KUNIT_ASSERT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), 0); ++ for (chunk = 0; chunk < 3; chunk++) ++ memset(data + chunk * 4096, 'X' + chunk, 4096); ++ KUNIT_ASSERT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE + 1, data, 3), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_flush(sftl), 0); ++ KUNIT_EXPECT_EQ(test, f->programs, 2U); ++ KUNIT_EXPECT_EQ(test, f->program_outside, 0U); ++ KUNIT_ASSERT_NOT_NULL(test, sftl_test_written(f, 0, 0, 0)); ++ for (chunk = 0; chunk < 4; chunk++) { ++ const u8 *meta = sftl_test_written(f, 0, 0, 0)->oob + chunk * 16; ++ ++ KUNIT_EXPECT_EQ(test, meta[0], (u8)1); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le64(meta) >> 16, ++ TEST_APPEND_FIRST + chunk); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(meta + 8), TEST_APPEND_BASE); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(meta + 12), ++ (u32)((TEST_APPEND_FIRST + chunk) >> 16)); ++ } ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, TEST_APPEND_FIRST + 7); ++ for (chunk = 0; chunk < 4; chunk++) { ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE + chunk, readback), 0); ++ sftl_test_expect_fill(test, readback, 'W' + chunk); ++ } ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, 0, readback), 0); ++ sftl_test_expect_fill(test, readback, 'C'); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, 6, readback), 0); ++ sftl_test_expect_fill(test, readback, 6); ++ apple_sftl_free(sftl); ++} ++ ++static int sftl_test_fill_append(struct apple_sftl *sftl, u8 *data) ++{ ++ unsigned int program; ++ int ret; ++ ++ for (program = 0; program < TEST_APPEND_PROGRAMS; program++) { ++ memset(data, program + 1, 16384); ++ ret = apple_sftl_write(sftl, TEST_APPEND_BASE + program * 4, data, 4); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static void sftl_test_append_replay_closed(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ struct apple_sftl_info info; ++ u8 *data = kunit_kmalloc(test, 16384, GFP_KERNEL); ++ u8 *readback = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ unsigned int page; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_NOT_NULL(test, readback); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, sftl_test_fill_append(sftl, data), 0); ++ KUNIT_EXPECT_EQ(test, f->programs, 128U); ++ KUNIT_EXPECT_EQ(test, f->program_outside, 0U); ++ for (page = 120; page < 128; page++) { ++ const u8 *meta; ++ ++ KUNIT_ASSERT_NOT_NULL(test, sftl_test_written(f, 0, 0, page)); ++ meta = sftl_test_written(f, 0, 0, page)->oob; ++ KUNIT_EXPECT_EQ(test, meta[0], (u8)0x1c); ++ KUNIT_EXPECT_EQ(test, meta[1], (u8)64); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le64(meta) >> 16, TEST_APPEND_FIRST); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(meta + 8), 480U); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le16(meta + 12), (u16)32); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le16(meta + 14), (u16)0xffff); ++ KUNIT_EXPECT_MEMEQ(test, meta, meta + 16, 16); ++ KUNIT_EXPECT_MEMEQ(test, meta, meta + 32, 16); ++ KUNIT_EXPECT_MEMEQ(test, meta, meta + 48, 16); ++ } ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(sftl_test_written(f, 0, 0, 120)->data + 8), ++ TEST_APPEND_BASE); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(sftl_test_written(f, 0, 0, 120)->data + 12), 480U); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, TEST_APPEND_FIRST + 479); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE, readback), 0); ++ sftl_test_expect_fill(test, readback, 1); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE + 479, readback), 0); ++ sftl_test_expect_fill(test, readback, 120); ++ apple_sftl_free(sftl); ++} ++ ++/* Repeating one key prevents summary coalescing: all 8160 data records ++ * need separate entries, including those spanning each footer page. ++ */ ++static void sftl_test_append_replay_full_lanes(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ struct apple_sftl_info info; ++ const unsigned int programs = 16 * 128 - 8; ++ const unsigned int chunks = programs * 4; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ unsigned int program, lane, page, entry; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ f->full_lanes = true; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ for (program = 0; program < programs; program++) { ++ memset(data, program + 1, 4096); ++ KUNIT_ASSERT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), 0); ++ } ++ KUNIT_EXPECT_EQ(test, apple_sftl_flush(sftl), 0); ++ KUNIT_EXPECT_EQ(test, f->programs, 16U * 128); ++ KUNIT_EXPECT_EQ(test, f->program_outside, 0U); ++ KUNIT_EXPECT_EQ(test, f->program_lanes, (u16)0xffff); ++ for (page = 0; page < 128; page++) { ++ for (lane = 0; lane < 16; lane++) { ++ struct sftl_test_page *written = sftl_test_written(f, 0, lane, page); ++ unsigned int position = page * 16 + lane; ++ unsigned int chunk; ++ ++ KUNIT_ASSERT_NOT_NULL(test, written); ++ if (position >= programs) { ++ const u8 *meta = written->oob; ++ ++ KUNIT_EXPECT_EQ(test, meta[0], (u8)0x1c); ++ KUNIT_EXPECT_EQ(test, meta[1], (u8)64); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le64(meta) >> 16, ++ TEST_APPEND_FIRST); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(meta + 8), chunks); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le16(meta + 12), (u16)32); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le16(meta + 14), (u16)0xffff); ++ for (chunk = 1; chunk < 4; chunk++) ++ KUNIT_EXPECT_MEMEQ(test, meta, meta + chunk * 16, 16); ++ continue; ++ } ++ for (chunk = 0; chunk < 4; chunk++) { ++ const u8 *meta = written->oob + chunk * 16; ++ u64 sequence = TEST_APPEND_FIRST + position * 4 + chunk; ++ ++ KUNIT_EXPECT_EQ(test, meta[0], (u8)1); ++ KUNIT_EXPECT_EQ(test, meta[1], (u8)0); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le64(meta) >> 16, sequence); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(meta + 8), ++ TEST_APPEND_BASE); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(meta + 12), ++ (u32)(sequence >> 16)); ++ sftl_test_expect_fill(test, written->data + chunk * 4096, ++ (u8)(position + 1)); ++ } ++ } ++ } ++ for (entry = 0; entry < chunks; entry++) { ++ struct sftl_test_page *footer = sftl_test_written(f, 0, 8 + entry / 1024, 127); ++ const u8 *run = footer->data + (entry % 1024) * 16; ++ ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(run), entry); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(run + 4), ++ (u32)((TEST_APPEND_FIRST + entry) >> 16)); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(run + 8), TEST_APPEND_BASE); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(run + 12), 1U); ++ } ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(sftl_test_written(f, 0, 15, 127)->data + ++ (chunks % 1024) * 16 + 12), 0U); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, TEST_APPEND_FIRST + chunks - 1); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE, data), 0); ++ sftl_test_expect_fill(test, data, (u8)programs); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, 0, data), 0); ++ sftl_test_expect_fill(test, data, 'C'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_failure_stops_io(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ unsigned int programs; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ f->fail_program = 1; ++ memset(data, 'F', 4096); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EIO); ++ programs = f->programs; ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EIO); ++ KUNIT_EXPECT_EQ(test, apple_sftl_flush(sftl), -EIO); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 0, data), -EIO); ++ KUNIT_EXPECT_EQ(test, f->programs, programs); ++ KUNIT_EXPECT_EQ(test, f->program_outside, 0U); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE, data), 0); ++ sftl_test_expect_fill(test, data, 0); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_uncertain_program_replays(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ f->fail_program = 1; ++ f->uncertain_program = true; ++ memset(data, 'U', 4096); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EIO); ++ KUNIT_EXPECT_EQ(test, apple_sftl_flush(sftl), -EIO); ++ KUNIT_EXPECT_EQ(test, f->programs, 1U); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE, data), 0); ++ sftl_test_expect_fill(test, data, 'U'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_interrupted_footer_replays(struct kunit *test) ++{ ++ static const struct { ++ unsigned int program; ++ bool uncertain; ++ } stop[] = {{121, false}, {122, false}, {128, false}, {128, true}}; ++ unsigned int index; ++ ++ for (index = 0; index < ARRAY_SIZE(stop); index++) { ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 16384, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ f->fail_program = stop[index].program; ++ f->uncertain_program = stop[index].uncertain; ++ KUNIT_EXPECT_EQ(test, sftl_test_fill_append(sftl, data), -EIO); ++ KUNIT_EXPECT_EQ(test, f->programs, stop[index].program); ++ KUNIT_EXPECT_EQ(test, apple_sftl_flush(sftl), -EIO); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE + 479, data), 0); ++ sftl_test_expect_fill(test, data, 120); ++ apple_sftl_free(sftl); ++ } ++} ++ ++static void sftl_test_append_torn_metadata_rejected(struct kunit *test) ++{ ++ static const unsigned int torn[] = {1, 121}; ++ unsigned int index; ++ ++ for (index = 0; index < ARRAY_SIZE(torn); index++) { ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 16384, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ f->tear_program = torn[index]; ++ memset(data, 'T', 16384); ++ if (torn[index] == 1) ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EIO); ++ else ++ KUNIT_EXPECT_EQ(test, sftl_test_fill_append(sftl, data), -EIO); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_EXPECT_LT(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_EXPECT_PTR_EQ(test, sftl, NULL); ++ apple_sftl_free(sftl); ++ } ++} ++ ++static void sftl_test_append_scans_entire_candidate(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ f->dirty_candidate = true; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ memset(data, 'B', 4096); ++ KUNIT_ASSERT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), 0); ++ KUNIT_EXPECT_PTR_EQ(test, sftl_test_written(f, 0, 0, 0), NULL); ++ KUNIT_ASSERT_NOT_NULL(test, sftl_test_written(f, 1, 0, 0)); ++ KUNIT_EXPECT_EQ(test, f->programs, 1U); ++ KUNIT_EXPECT_EQ(test, f->program_outside, 0U); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE, data), 0); ++ sftl_test_expect_fill(test, data, 'B'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_reserves_vfl_members(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ f->vfl_candidate = true; ++ f->no_candidates = true; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ memset(data, 'V', 4096); ++ KUNIT_ASSERT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), 0); ++ KUNIT_EXPECT_PTR_EQ(test, sftl_test_written(f, 3, 0, 0), NULL); ++ KUNIT_ASSERT_NOT_NULL(test, sftl_test_written(f, 4, 0, 0)); ++ KUNIT_EXPECT_EQ(test, f->programs, 1U); ++ KUNIT_EXPECT_EQ(test, f->program_outside, 0U); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE, data), 0); ++ sftl_test_expect_fill(test, data, 'V'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_closes_at_sequence_limit(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ struct apple_sftl_info info; ++ const u64 limit = (1ULL << 48) - 1; ++ u8 *data = kunit_kmalloc(test, 16384, GFP_KERNEL); ++ unsigned int programs; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ f->sequence_bias = limit - TEST_APPEND_FIRST - 479; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, sftl_test_fill_append(sftl, data), 0); ++ programs = f->programs; ++ KUNIT_EXPECT_EQ(test, programs, 128U); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, limit); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EOVERFLOW); ++ KUNIT_EXPECT_EQ(test, f->programs, programs); ++ KUNIT_EXPECT_EQ(test, apple_sftl_flush(sftl), 0); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ apple_sftl_get_info(sftl, &info); ++ KUNIT_EXPECT_EQ(test, info.sequence, limit); ++ KUNIT_ASSERT_EQ(test, apple_sftl_read(sftl, TEST_APPEND_BASE + 479, data), 0); ++ sftl_test_expect_fill(test, data, 120); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EOVERFLOW); ++ KUNIT_EXPECT_EQ(test, f->programs, programs); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_no_space_keeps_reads(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ f->no_candidates = true; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ memset(data, 'N', 4096); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -ENOSPC); ++ KUNIT_EXPECT_EQ(test, f->programs, 0U); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 0, data), 0); ++ sftl_test_expect_fill(test, data, 'C'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_refuses_sequence_floor_gap(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_ALLOCATION_FLOOR); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 4096, GFP_KERNEL); ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ memset(data, 'S', 4096); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 1), -EOPNOTSUPP); ++ KUNIT_EXPECT_EQ(test, f->programs, 0U); ++ KUNIT_EXPECT_EQ(test, apple_sftl_read(sftl, 0, data), 0); ++ sftl_test_expect_fill(test, data, 'C'); ++ apple_sftl_free(sftl); ++} ++ ++static void sftl_test_append_rejects_invalid_window(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl *sftl = NULL; ++ u8 *data = kunit_kmalloc(test, 16384, GFP_KERNEL); ++ unsigned int reads; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_NOT_NULL(test, data); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ reads = f->reads; ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 0), -EINVAL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, data, 5), -EINVAL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, 15625000, data, 1), -EINVAL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, 15624999, data, 2), -EINVAL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_write(sftl, TEST_APPEND_BASE, NULL, 1), -EINVAL); ++ KUNIT_EXPECT_EQ(test, f->reads, reads); ++ KUNIT_EXPECT_EQ(test, f->programs, 0U); ++ apple_sftl_free(sftl); ++} ++#endif ++ + static struct kunit_case apple_sftl_cases[] = { + KUNIT_CASE(sftl_test_page_budget_stops_before_next_read), + KUNIT_CASE(sftl_test_deadline_stops_before_next_read), + KUNIT_CASE(sftl_test_one_transaction_deadline_overrun), + KUNIT_CASE(sftl_test_provider_failure_is_sticky), + KUNIT_CASE(sftl_test_invalid_reader_policy), ++#if IS_ENABLED(CONFIG_MTD_APPLE_SFTL_WRITE) ++ KUNIT_CASE(sftl_test_append_replay_open), ++ KUNIT_CASE(sftl_test_append_replay_closed), ++ KUNIT_CASE(sftl_test_append_replay_full_lanes), ++ KUNIT_CASE(sftl_test_append_failure_stops_io), ++ KUNIT_CASE(sftl_test_append_uncertain_program_replays), ++ KUNIT_CASE(sftl_test_append_interrupted_footer_replays), ++ KUNIT_CASE(sftl_test_append_torn_metadata_rejected), ++ KUNIT_CASE(sftl_test_append_scans_entire_candidate), ++ KUNIT_CASE(sftl_test_append_reserves_vfl_members), ++ KUNIT_CASE(sftl_test_append_closes_at_sequence_limit), ++ KUNIT_CASE(sftl_test_append_no_space_keeps_reads), ++ KUNIT_CASE(sftl_test_append_refuses_sequence_floor_gap), ++ KUNIT_CASE(sftl_test_append_rejects_invalid_window), ++#endif + KUNIT_CASE(sftl_test_complete_medium), + KUNIT_CASE(sftl_test_scan_after_open_lane_hole), + KUNIT_CASE(sftl_test_runtime_failure_stops_reads), +diff --git a/drivers/mtd/apple-sftl.h b/drivers/mtd/apple-sftl.h +index 7a05b95d1..0babcebb0 100644 +--- a/drivers/mtd/apple-sftl.h ++++ b/drivers/mtd/apple-sftl.h +@@ -13,11 +13,15 @@ struct apple_sftl_info { + u32 nr_runs; + u64 checkpoint_sequence; + u64 sequence; ++ u64 journal_sequence; + }; + + int apple_sftl_load(struct apple_ppn *provider, struct apple_sftl **out); + void apple_sftl_free(struct apple_sftl *sftl); + int apple_sftl_read(struct apple_sftl *sftl, u32 logical_page, u8 *data); ++int apple_sftl_write(struct apple_sftl *sftl, u32 logical_page, ++ const u8 *data, unsigned int count); ++int apple_sftl_flush(struct apple_sftl *sftl); + void apple_sftl_get_info(struct apple_sftl *sftl, struct apple_sftl_info *info); + + #endif +diff --git a/tools/testing/kunit/configs/apple_sftl_write.config b/tools/testing/kunit/configs/apple_sftl_write.config +new file mode 100644 +index 000000000..76fe23a68 +--- /dev/null ++++ b/tools/testing/kunit/configs/apple_sftl_write.config +@@ -0,0 +1,8 @@ ++CONFIG_KUNIT=y ++CONFIG_MTD=y ++CONFIG_COMPILE_TEST=y ++CONFIG_MTD_APPLE_H2FMI=y ++CONFIG_MTD_APPLE_H2FMI_WRITE=y ++CONFIG_MTD_APPLE_SFTL=y ++CONFIG_MTD_APPLE_SFTL_WRITE=y ++CONFIG_MTD_APPLE_SFTL_KUNIT_TEST=y diff --git a/kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch.toml b/kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0007-mtd-apple-sftl-add-verified-append-writes-and-fresh-.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] diff --git a/kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch b/kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch new file mode 100644 index 00000000..efa278ff --- /dev/null +++ b/kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch @@ -0,0 +1,73 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 12:39:42 -0600 +Subject: [PATCH] mtd: apple-sftl: prevent recursive reclaim during request I/O + +Map splits, summary preparation and H2FMI program buffers allocate while +io_lock serializes requests to the logical root disk. GFP_KERNEL reclaim +can otherwise queue writeback against that same disk and wait for the +lock held by the original request. + +Use a NOIO allocation scope around all request processing and lower +provider calls, restoring the previous scope on the common completion +path. This keeps existing allocation failure handling while preventing +recursive filesystem I/O from reclaim. + +The exact queue-function harness checks balanced scope and completion +on 22 success, policy, deadline and error cases in each write config. + +Assisted-by: LLM +--- + Documentation/driver-api/mtd/apple-sftl.rst | 4 +++- + drivers/mtd/apple-sftl-block.c | 5 +++++ + 2 files changed, 8 insertions(+), 1 deletion(-) + +diff --git a/Documentation/driver-api/mtd/apple-sftl.rst b/Documentation/driver-api/mtd/apple-sftl.rst +index 9d65b2149..6f7a3b229 100644 +--- a/Documentation/driver-api/mtd/apple-sftl.rst ++++ b/Documentation/driver-api/mtd/apple-sftl.rst +@@ -43,7 +43,9 @@ an error. A failed runtime read stops subsequent I/O from this map instance. + H2FMI unregisters the child before releasing its controller state. The block + frontend freezes and drains requests during removal. Its request buffer is + allocated before the disk becomes visible, while blocking requests serialize +-access to that buffer and the provider. ++access to that buffer and the provider. Request processing uses a NOIO ++allocation scope so map and provider buffers cannot trigger recursive ++writeback while the queue lock is held. + + The reader has finite operating limits. ``max_page_reads`` defaults to 32,768 + physical page requests and ``max_io_time_ms`` defaults to ten minutes from +diff --git a/drivers/mtd/apple-sftl-block.c b/drivers/mtd/apple-sftl-block.c +index 29a05ae3b..4471a9176 100644 +--- a/drivers/mtd/apple-sftl-block.c ++++ b/drivers/mtd/apple-sftl-block.c +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include + #include + + #include "apple-sftl.h" +@@ -182,10 +183,13 @@ static blk_status_t sftl_queue_rq(struct blk_mq_hw_ctx *hctx, + struct request *request = data->rq; + struct sftl_disk *sftl = request->q->queuedata; + sector_t capacity = get_capacity(sftl->disk); ++ unsigned int noio_flags; + blk_status_t status; + int ret = 0; + + mutex_lock(&sftl->io_lock); ++ /* Map and provider buffers may allocate while the root disk is busy. */ ++ noio_flags = memalloc_noio_save(); + blk_mq_start_request(request); + if (sftl->dead) { + ret = -ENODEV; +@@ -203,6 +207,7 @@ static blk_status_t sftl_queue_rq(struct blk_mq_hw_ctx *hctx, + } else { + ret = sftl_read_request(sftl, request); + } ++ memalloc_noio_restore(noio_flags); + mutex_unlock(&sftl->io_lock); + status = errno_to_blk_status(ret); + blk_mq_end_request(request, status); diff --git a/kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch.toml b/kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0008-mtd-apple-sftl-prevent-recursive-reclaim-during-requ.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] From 2c10c56c96774b17ad6b4b29ed95df80c546bc11 Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:10:34 -0600 Subject: [PATCH 15/16] p105: document offline SFTL checkpoint encoding and GC planning Record the normalized kernel source, software checks and native format work for the checkpoint encoder and GC transaction planner. Preserve missing storage capability and installation gates while native accounting and interrupted-save recovery remain unqualified. Regenerated and checked the matrix; all 34 catalog tests ran successfully with one expected skip. The exact normalized source passed both ARMv7 object configurations and 35 Linux KUnit tests with synthetic media. Assisted-by: LLM --- chips/s5l8942x.toml | 2 + devices/p105/device.toml | 3 ++ docs/bring-up/p105-sftl-checkpoint-gc.md | 49 ++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 docs/bring-up/p105-sftl-checkpoint-gc.md diff --git a/chips/s5l8942x.toml b/chips/s5l8942x.toml index e76ef051..a7fdb314 100644 --- a/chips/s5l8942x.toml +++ b/chips/s5l8942x.toml @@ -22,3 +22,5 @@ cascadia_profile = "devices/p105/cascadia.toml; external-initramfs handoff await # Public Cascadia RAM recovery correlates a guarded host USB identity with the live FDT and boot ID. # Linux SoC ECID access and on-device public recovery identity remain unqualified. nand_append = "The measured P105 64 GB kernel append path has native data/footer replay, verified programs, default-off write policy and finite operating bounds; synthetic Linux KUnit and ARMv7 object checks passed, while hardware writes remain unqualified (docs/bring-up/p105-sftl-append.md)." + +nand_checkpoint_gc = "The measured P105 64 GB checkpoint encoder and GC transaction planner passed synthetic Linux KUnit and exact ARMv7 object checks; media GC stays disconnected pending native accounting, ownership and interruption recovery qualification (docs/bring-up/p105-sftl-checkpoint-gc.md)." diff --git a/devices/p105/device.toml b/devices/p105/device.toml index b5536905..db6c1d64 100644 --- a/devices/p105/device.toml +++ b/devices/p105/device.toml @@ -48,6 +48,9 @@ touch_id = "missing" storage_read = "missing" # The default-off complete-page primitive does not provide a qualified logical writer. # See docs/bring-up/p105-nand-program.md for its checks and pending hardware work. +# Offline checkpoint encoding and GC planning await media qualification. +# Native accounting and interrupted-save recovery remain pending; install stays gated. +# See docs/bring-up/p105-sftl-checkpoint-gc.md. storage_write = "missing" wifi = "untested" bluetooth_input = "untested" diff --git a/docs/bring-up/p105-sftl-checkpoint-gc.md b/docs/bring-up/p105-sftl-checkpoint-gc.md new file mode 100644 index 00000000..539146b3 --- /dev/null +++ b/docs/bring-up/p105-sftl-checkpoint-gc.md @@ -0,0 +1,49 @@ +# P105 offline checkpoint encoder and GC planner + +The kernel draft on +[`p105/08-ftl-reclamation-integration`](https://github.com/shaynelegault/linux/tree/p105/08-ftl-reclamation-integration) +at `c504eae2a214915dcf5d2c4856c52e7a6ac79d2d` stacks on the append writer at +`c63274165870417f01cc3e1264067cef71d9c0b2`. It supplies an offline checkpoint +encoder and GC transaction planner. Kernel patches follow Orchard's +platform-scoped export workflow; exact-revision hardware qualification remains +pending. + +The snapshot contains all 15,625,016 logical keys, including the sixteen +internal keys beyond the exported disk. The encoder preserves opaque native +accounting payloads, updates the decoded context-block list and serializes +range fragments with 48-bit sequence metadata. The planner budgets its +working buffers and destination capacity, retains footer space, checks copy +and checkpoint readback receipts, then compares the entire cold-recovered +map. Supplied blank, reserved and lane descriptions need authoritative VFL +and context ownership proof before future destructive I/O uses them. + +Media GC remains disconnected. Opaque accounting fields still need native +updates before checkpoint installation. The current reader rejects an +incomplete newest SFTL context and requires contiguous replay; native +previous-checkpoint selection and interrupted-save replay need further work. +The earlier installed desktop used the Python backend and supplies no +qualification for these kernel changes. + +Native format work follows IOFlashStorage-730/13G36 and the pinned +[Cascadia NAND research](https://github.com/rzxvx/Project-Cascadia/blob/c20b85ebb951c704d7e3a664c378dde8276e3928/docs/research/p105-nand.md). +The native metadata formatter is at `0x81092408`. Saved 64 GB parameters and +the native row builder agree on normal erase rows +`(cau << 19) | (block << 7)`. Native VFL rotation erases an unreferenced target +bank, writes an initial checkpoint retaining old table references, copies the +tables and writes a final checkpoint. The source bank survives the healthy +path. Physical erase effects, SLC aliasing and retirement/torn-copy recovery +remain unqualified. Firmware, captures and device identifiers stay private; +public tests generate synthetic media. + +Host ASan/UBSan checks passed 27 writer-enabled recovery cases, 14 read-only +cases and eight checkpoint/planner cases. Twenty-four metadata records +matched the pinned native formatter running in synthetic RAM, and 209 C +checkpoint images round-tripped through the Python map decoder. Exact-source +core/frontend/checkpoint/provider and both fixture objects compiled on the +current ARMv7 tree in read-only and experimental write configurations, with +`W=1` and warnings-as-errors apart from the existing `vmstat.h` enum warning. +An ARM64 QEMU KUnit kernel passed 35/35 tests using pinned pristine Linux +v6.12 plus the exact integrated C sources and a validation-only dependency +symbol for synthetic program callbacks. These checks exercise no controller +timing. P105 storage features remain `missing`, with installation gated +while exact-revision device qualification is pending. From 9e0fd8ec5c0ace17e147647ea754711a1f34939f Mon Sep 17 00:00:00 2001 From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:15:57 -0600 Subject: [PATCH 16/16] mtd: export offline P105 checkpoint encoding and GC planning Export the normalized source on the bounded append writer, preserving its source authors and trailers. The two MTD patches select apple-armv7 and reproduce the source tree when applied to the append-writer base. The encoder and planner operate on snapshots and supplied receipts. Physical GC remains disconnected while native accounting and interrupted checkpoint recovery require further work. Validation includes both current ARMv7 object variants, 35 Linux KUnit cases, host sanitizer fixtures and native metadata comparisons. Platform selection remains unchanged for both ARM64 configurations. Assisted-by: LLM --- docs/bring-up/p105-sftl-checkpoint-gc.md | 5 +- ...FTL-checkpoint-encoding-and-GC-plann.patch | 1302 +++++++++++++++++ ...heckpoint-encoding-and-GC-plann.patch.toml | 2 + ...PPN-erase-addressing-and-VFL-rotatio.patch | 60 + ...rase-addressing-and-VFL-rotatio.patch.toml | 2 + 5 files changed, 1369 insertions(+), 2 deletions(-) create mode 100644 kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch create mode 100644 kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch.toml create mode 100644 kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch create mode 100644 kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch.toml diff --git a/docs/bring-up/p105-sftl-checkpoint-gc.md b/docs/bring-up/p105-sftl-checkpoint-gc.md index 539146b3..cb4c889b 100644 --- a/docs/bring-up/p105-sftl-checkpoint-gc.md +++ b/docs/bring-up/p105-sftl-checkpoint-gc.md @@ -5,8 +5,9 @@ The kernel draft on at `c504eae2a214915dcf5d2c4856c52e7a6ac79d2d` stacks on the append writer at `c63274165870417f01cc3e1264067cef71d9c0b2`. It supplies an offline checkpoint encoder and GC transaction planner. Kernel patches follow Orchard's -platform-scoped export workflow; exact-revision hardware qualification remains -pending. +platform-scoped export workflow. MTD patches 0009 and 0010 select only +`apple-armv7`; applying them to the append-writer source reproduces this +branch exactly. Exact-revision hardware qualification remains pending. The snapshot contains all 15,625,016 logical keys, including the sixteen internal keys beyond the exported disk. The encoder preserves opaque native diff --git a/kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch b/kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch new file mode 100644 index 00000000..5a164ed5 --- /dev/null +++ b/kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch @@ -0,0 +1,1302 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 12:28:05 -0600 +Subject: [PATCH] mtd: add SwissPPNFTL checkpoint encoding and GC planning + +Preserve the recovered whole-device map and opaque checkpoint payloads in bounded snapshots. Encode native context metadata and range fragments, then validate relocation, checkpoint readback and cold-recovery receipts through an offline GC transaction. + +Keep media erase disconnected while native accounting, VFL bank rotation and the 64 GB erase granule remain unqualified. The driver-api note records these limits and the next protocol work. + +Validation: 17 recovery and reclamation cases under host ASan/UBSan, 24 metadata records compared with the pinned native formatter, and 209 checkpoint map roundtrips through the Python decoder. Kernel compilation and physical qualification remain pending. + +Assisted-by: LLM +--- + .../driver-api/mtd/apple-sftl-reclamation.rst | 85 +++ + Documentation/driver-api/mtd/index.rst | 4 +- + drivers/mtd/Kconfig | 3 +- + drivers/mtd/Makefile | 3 +- + drivers/mtd/apple-sftl-core.c | 91 ++++ + drivers/mtd/apple-sftl-reclaim-test.c | 329 ++++++++++++ + drivers/mtd/apple-sftl-reclaim.c | 498 ++++++++++++++++++ + drivers/mtd/apple-sftl-reclaim.h | 88 ++++ + drivers/mtd/apple-sftl-test.c | 46 +- + 9 files changed, 1143 insertions(+), 4 deletions(-) + create mode 100644 Documentation/driver-api/mtd/apple-sftl-reclamation.rst + create mode 100644 drivers/mtd/apple-sftl-reclaim-test.c + create mode 100644 drivers/mtd/apple-sftl-reclaim.c + create mode 100644 drivers/mtd/apple-sftl-reclaim.h + +diff --git a/Documentation/driver-api/mtd/apple-sftl-reclamation.rst b/Documentation/driver-api/mtd/apple-sftl-reclamation.rst +new file mode 100644 +index 000000000..e286c6935 +--- /dev/null ++++ b/Documentation/driver-api/mtd/apple-sftl-reclamation.rst +@@ -0,0 +1,85 @@ ++.. SPDX-License-Identifier: GPL-2.0-only ++ ++Apple SwissPPNFTL checkpoint and reclamation draft ++================================================= ++ ++The measured 64 GB PPN layout shares one FTL across the logical partitions. ++A Linux partition therefore provides no exclusive ownership of a physical ++block. Reclamation must preserve the recovered map for the whole device. ++ ++The code in ``apple-sftl-reclaim.c`` serializes checkpoint images and plans ++transactions from supplied snapshots and completion receipts. It registers ++no media operation. The block device retains its existing access policy. ++ ++Checkpoint images ++----------------- ++ ++``apple_sftl_snapshot()`` takes an owned copy of the whole recovered range ++map. It rereads the selected context's non-map records under the map lock, ++checks their saved stamps and preserves their payloads. The caller supplies ++a memory budget. Changed media stops further reads through that instance. ++ ++``apple_sftl_checkpoint_build()`` emits a bounded array of canonical 4 KiB ++records. The metadata follows the 13G36 IOFlashStorage-730 formatter at ++``0x81092408``: type 0x1f, subtype, a 48-bit sequence, an array index/count, ++and a trailing 0xffffffff. The base record's decoded context-block list is ++updated. Other payload bytes survive unchanged. Mapping records use subtype ++4 with the native cursor/range encoding. Padding brings the image to full ++16 KiB pages and consumes sequence values through subtype-4 padding records. ++The resulting bytes round-trip through the current recovery decoder. ++ ++Opaque statistics, superblock state, user-sequence and read-count payloads ++still describe the source checkpoint. Preserving them prevents accidental ++data loss during format work; a native writer must update the fields whose ++meaning depends on relocation. These images have not been qualified for ++native checkpoint installation. ++ ++Offline transaction ++------------------- ++ ++``apple_sftl_gc_plan()`` consumes a whole-device snapshot plus descriptions ++of a closed victim and a fully checked blank destination. It rejects reserved ++blocks, references into the victim's footer or inactive space, a destination ++still referenced by the map, insufficient destination capacity and a working ++plan buffers exceeding the caller's budget. A source with an allocation ++floor above the last replayed record is excluded under the current recovery ++contract. Copies retain the logical keys while ++moving their provider addresses. Destination space leaves eight pages for ++a future native summary. ++ ++The transaction advances through explicit receipt checks: ++ ++* Each copy receipt must match its planned logical range, source, ++ destination and final sequence. The caller must issue and verify that IO. ++* A complete checkpoint image is built only after the copy receipts arrive. ++ Its readback must match every metadata and payload byte. ++* A fresh recovery snapshot must contain the planned whole-device mapping ++ and the new checkpoint sequence. Equivalent coalescing is accepted. ++ ++A mismatched receipt moves the transaction to its failed state. Even a victim ++with zero current references must pass checkpoint and cold-recovery stages, ++since the current replay contract can still require its obsolete history. ++The final VERIFIED state means that these supplied receipts passed. It does ++not certify native accounting or VFL commit durability. ++ ++Outstanding native protocol work ++------------------------------- ++ ++VFL recovery uses two context groups with four copies per group on this ++profile. The native rotation routine at ``0x810a1b68`` checks references into ++the target group before reuse, copies required lane tables and writes a final ++checkpoint page. Its interruption and replacement paths require further ++tracing before a runtime writer can reproduce them. ++ ++Offline execution of the pinned native iBEC erase queue establishes queued ++67/D7 commands, final-row 6A/D7, 79 polling and 77/7D completion. Linking its ++row construction to the 64 GB erase granule remains open, along with timeout, ++retirement and partial completion across CAUs. The provider exposes no erase ++callback. Enabling physical reclamation requires resolving those semantics ++and qualifying the exact candidate with device recovery and interruption ++checks. ++ ++The source research began with `Cascadia's NAND notes ++`_. ++The private native differential fixtures retain source hashes and execute ++against synthetic RAM. Public tests contain synthetic media only. +diff --git a/Documentation/driver-api/mtd/index.rst b/Documentation/driver-api/mtd/index.rst +index e91da2aa5..64342a68b 100644 +--- a/Documentation/driver-api/mtd/index.rst ++++ b/Documentation/driver-api/mtd/index.rst +@@ -9,4 +9,6 @@ Memory Technology Device (MTD) + + spi-intel + nand_ecc +- spi-nor apple-sftl ++ spi-nor ++ apple-sftl ++ apple-sftl-reclamation +diff --git a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig +index 5e4a1e90c..fa5413f61 100644 +--- a/drivers/mtd/Kconfig ++++ b/drivers/mtd/Kconfig +@@ -240,7 +240,8 @@ config MTD_APPLE_SFTL_KUNIT_TEST + default KUNIT_ALL_TESTS + help + Synthesize complete VFL/SFTL media and exercise checkpoint recovery, +- native block summaries and open-block journal replay. Malformed ++ native block summaries, open-block journal replay, checkpoint encoding ++ and offline reclamation transactions. Malformed + replicas and unsupported layouts must fail without storage writes. + With experimental append writes enabled, also test complete programs, + interrupted summaries, allocation limits and fresh-loader replay. +diff --git a/drivers/mtd/Makefile b/drivers/mtd/Makefile +index fa6180a62..8a05767a1 100644 +--- a/drivers/mtd/Makefile ++++ b/drivers/mtd/Makefile +@@ -33,5 +33,6 @@ obj-$(CONFIG_MTD_UBI) += ubi/ + obj-$(CONFIG_MTD_HYPERBUS) += hyperbus/ + + obj-$(CONFIG_MTD_APPLE_SFTL) += apple-sftl.o +-apple-sftl-y := apple-sftl-core.o apple-sftl-block.o ++apple-sftl-y := apple-sftl-core.o apple-sftl-block.o apple-sftl-reclaim.o + obj-$(CONFIG_MTD_APPLE_SFTL_KUNIT_TEST) += apple-sftl-test.o ++obj-$(CONFIG_MTD_APPLE_SFTL_KUNIT_TEST) += apple-sftl-reclaim-test.o +diff --git a/drivers/mtd/apple-sftl-core.c b/drivers/mtd/apple-sftl-core.c +index 6f16ecd54..c7ef773d6 100644 +--- a/drivers/mtd/apple-sftl-core.c ++++ b/drivers/mtd/apple-sftl-core.c +@@ -21,6 +21,7 @@ + #include + + #include "apple-sftl.h" ++#include "apple-sftl-reclaim.h" + + #define SFTL_LPAGES 15625016U + #define SFTL_DISK_PAGES 15625000U +@@ -63,6 +64,7 @@ struct sftl_member { + struct apple_sftl { + struct apple_ppn *ppn; + struct rb_root runs; ++ struct sftl_records context; + u32 nr_runs; + u16 lanes[SFTL_BLOCKS]; + DECLARE_BITMAP(reserved, SFTL_BLOCKS); +@@ -620,6 +622,17 @@ static int sftl_context(struct apple_sftl *sftl, const u8 *heads) + } + if (cursor != SFTL_LPAGES || !fragment_sequence) + goto out; ++ /* Retain compact references so an explicit snapshot can preserve opaque data. */ ++ for (i = first; i < records.count; i++) { ++ const struct sftl_record *record = &records.items[i]; ++ ++ if (sftl_flags(record) == 4) ++ continue; ++ ret = sftl_record_add(&sftl->context, record->stamp, record->key, ++ record->address); ++ if (ret) ++ goto out; ++ } + sftl->checkpoint_sequence = fragment_sequence; + sftl->sequence = fragment_sequence; + ret = 0; +@@ -955,6 +968,7 @@ void apple_sftl_free(struct apple_sftl *sftl) + kfree(rb_entry(node, struct sftl_run, node)); + } + kvfree(sftl->footer); ++ kvfree(sftl->context.items); + kfree(sftl->page); + kfree(sftl); + } +@@ -1251,3 +1265,80 @@ void apple_sftl_get_info(struct apple_sftl *sftl, struct apple_sftl_info *info) + info->journal_sequence = sftl->journal_sequence; + } + EXPORT_SYMBOL_GPL(apple_sftl_get_info); ++ ++int apple_sftl_snapshot(struct apple_sftl *sftl, size_t max_bytes, ++ struct apple_sftl_snapshot **out) ++{ ++ struct apple_sftl_snapshot *snapshot; ++ struct rb_node *node; ++ size_t map_bytes, context_bytes; ++ u32 i = 0; ++ unsigned int chunk; ++ int ret = 0; ++ ++ if (!out) ++ return -EINVAL; ++ *out = NULL; ++ if (!sftl) ++ return -EINVAL; ++ mutex_lock(&sftl->lock); ++ if (sftl->failed) { ++ ret = -EIO; ++ goto unlock; ++ } ++ map_bytes = array_size(sftl->nr_runs, sizeof(*snapshot->runs)); ++ context_bytes = array_size(sftl->context.count, sizeof(*snapshot->opaque)); ++ if (map_bytes > max_bytes || context_bytes > max_bytes - map_bytes) { ++ ret = -ENOSPC; ++ goto unlock; ++ } ++ snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL); ++ if (!snapshot) { ++ ret = -ENOMEM; ++ goto unlock; ++ } ++ snapshot->runs = kvzalloc(map_bytes, GFP_KERNEL); ++ snapshot->opaque = kvzalloc(context_bytes, GFP_KERNEL); ++ if (!snapshot->runs || !snapshot->opaque) { ++ ret = -ENOMEM; ++ goto failed; ++ } ++ for (node = rb_first(&sftl->runs); node; node = rb_next(node)) { ++ const struct sftl_run *run = rb_entry(node, struct sftl_run, node); ++ ++ snapshot->runs[i++] = (struct apple_sftl_extent) { ++ .logical = run->start, .length = run->end - run->start, ++ .provider = run->provider, ++ }; ++ } ++ for (i = 0; i < sftl->context.count; i++) { ++ const struct sftl_record *reference = &sftl->context.items[i]; ++ struct apple_sftl_context_record *record = &snapshot->opaque[i]; ++ ++ ret = sftl_read_address(sftl, reference->address, &chunk); ++ if (ret) { ++ sftl->failed = true; ++ goto failed; ++ } ++ if (sftl_stamp(sftl->oob + chunk * SFTL_META) != reference->stamp) { ++ ret = -EBADMSG; ++ sftl->failed = true; ++ goto failed; ++ } ++ memcpy(record->meta, sftl->oob + chunk * SFTL_META, SFTL_META); ++ memcpy(record->data, sftl->page + chunk * SFTL_CHUNK, SFTL_CHUNK); ++ } ++ snapshot->nr_runs = sftl->nr_runs; ++ snapshot->nr_opaque = sftl->context.count; ++ snapshot->sequence = sftl->sequence; ++ snapshot->journal_sequence = sftl->journal_sequence; ++ snapshot->checkpoint_sequence = sftl->checkpoint_sequence; ++ *out = snapshot; ++ goto unlock; ++failed: ++ apple_sftl_snapshot_free(snapshot); ++unlock: ++ mutex_unlock(&sftl->lock); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_snapshot); +diff --git a/drivers/mtd/apple-sftl-reclaim-test.c b/drivers/mtd/apple-sftl-reclaim-test.c +new file mode 100644 +index 000000000..c26d8b43a +--- /dev/null ++++ b/drivers/mtd/apple-sftl-reclaim-test.c +@@ -0,0 +1,329 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Synthetic checkpoints and supplied IO receipts; no media mutations. */ ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-sftl-reclaim.h" ++ ++static void reclaim_source(struct apple_sftl_snapshot *snapshot, ++ struct apple_sftl_context_record *opaque, ++ struct apple_sftl_extent *runs, u32 count) ++{ ++ unsigned int i; ++ ++ memset(opaque, 0x6b, sizeof(*opaque)); ++ apple_sftl_context_meta(opaque->meta, 1, 100, 17, 23); ++ put_unaligned_le32(1, opaque->data); ++ put_unaligned_le32(1, opaque->data + 4); ++ for (i = 0; i < count; i++) ++ runs[i] = (struct apple_sftl_extent) { ++ .logical = i, .length = 1, ++ .provider = i & 1 ? APPLE_SFTL_NO_ADDRESS : 4 * APPLE_SFTL_STRIDE + i, ++ }; ++ runs[count - 1].length = APPLE_SFTL_LOGICAL_PAGES - count + 1; ++ runs[count - 1].provider = APPLE_SFTL_NO_ADDRESS; ++ *snapshot = (struct apple_sftl_snapshot) { ++ .runs = runs, .nr_runs = count, .opaque = opaque, .nr_opaque = 1, ++ .sequence = 200, .checkpoint_sequence = 101, .journal_sequence = 200, ++ }; ++} ++ ++static void reclaim_metadata_vector(struct kunit *test) ++{ ++ /* Native 0x81092408 output, with low-word sequence carry. */ ++ static const u8 expected[16] = { ++ 0x1f, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, ++ 0x00, 0x02, 0x09, 0x00, 0xff, 0xff, 0xff, 0xff, ++ }; ++ u8 meta[16]; ++ ++ KUNIT_ASSERT_EQ(test, apple_sftl_context_meta(meta, 6, 0x100000000ULL, 512, 9), 0); ++ KUNIT_EXPECT_MEMEQ(test, meta, expected, sizeof(meta)); ++ memset(meta, 0x5a, sizeof(meta)); ++ KUNIT_EXPECT_EQ(test, apple_sftl_context_meta(meta, 0, 1, 0, 0), -EINVAL); ++ KUNIT_EXPECT_EQ(test, meta[0], 0x5a); ++ KUNIT_EXPECT_EQ(test, apple_sftl_context_meta(meta, 4, 1ULL << 48, 0, 0), -EINVAL); ++} ++ ++static void reclaim_checkpoint_fragments(struct kunit *test) ++{ ++ struct apple_sftl_context_record *opaque = kunit_kmalloc(test, sizeof(*opaque), GFP_KERNEL); ++ struct apple_sftl_extent *runs = kunit_kmalloc(test, 514 * sizeof(*runs), GFP_KERNEL); ++ struct apple_sftl_checkpoint *image = NULL; ++ struct apple_sftl_snapshot snapshot; ++ u32 blocks[] = { 5, 6 }; ++ u32 i; ++ ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ KUNIT_ASSERT_NOT_NULL(test, runs); ++ reclaim_source(&snapshot, opaque, runs, 514); ++ KUNIT_ASSERT_EQ(test, apple_sftl_checkpoint_build(&snapshot, blocks, 2, 4, &image), 0); ++ KUNIT_ASSERT_NOT_NULL(test, image); ++ KUNIT_EXPECT_EQ(test, image->count, 4U); ++ KUNIT_EXPECT_EQ(test, image->first_sequence, 201ULL); ++ KUNIT_EXPECT_EQ(test, image->sequence, 204ULL); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[0].data), 2U); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[0].data + 4), 5U); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[0].data + 8), 6U); ++ KUNIT_EXPECT_MEMEQ(test, image->records[0].data + 12, opaque->data + 12, 4084); ++ KUNIT_EXPECT_MEMEQ(test, image->records[0].meta + 8, opaque->meta + 8, 8); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[1].data), 0U); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[2].data), 511U); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[3].data + 4), (u32)~0U); ++ for (i = 1; i < 4; i++) { ++ KUNIT_EXPECT_EQ(test, image->records[i].meta[0], 0x1f); ++ KUNIT_EXPECT_EQ(test, image->records[i].meta[1], 4); ++ KUNIT_EXPECT_EQ(test, get_unaligned_le32(image->records[i].meta + 8), 0U); ++ } ++ apple_sftl_checkpoint_free(image); ++ image = NULL; ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_checkpoint_build(&snapshot, blocks, 2, 3, &image), -ENOSPC); ++ KUNIT_EXPECT_PTR_EQ(test, image, NULL); ++ snapshot.sequence = (1ULL << 48) - 3; ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_checkpoint_build(&snapshot, blocks, 2, 4, &image), -EOVERFLOW); ++} ++ ++static void reclaim_gc_transaction(struct kunit *test) ++{ ++ struct apple_sftl_context_record *opaque = kunit_kmalloc(test, sizeof(*opaque), GFP_KERNEL); ++ struct apple_sftl_extent runs[3]; ++ struct apple_sftl_extent recovered_runs[3]; ++ struct apple_sftl_snapshot snapshot, recovered; ++ struct apple_sftl_gc_block victim = { ++ .block = 4, .lanes = 3, .closed = true, .last_sequence = 199, ++ }; ++ struct apple_sftl_gc_block destination = { .block = 7, .lanes = 3, .blank = true }; ++ const struct apple_sftl_gc_copy *copy; ++ struct apple_sftl_checkpoint *image = NULL; ++ struct apple_sftl_gc *gc = NULL; ++ u32 blocks[] = { 5 }; ++ ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ reclaim_source(&snapshot, opaque, runs, 3); ++ runs[0].length = 2; ++ runs[1] = (struct apple_sftl_extent) { ++ .logical = 2, .length = 1, .provider = 6 * APPLE_SFTL_STRIDE, ++ }; ++ runs[2] = (struct apple_sftl_extent) { ++ .logical = 3, .length = APPLE_SFTL_LOGICAL_PAGES - 3, ++ .provider = APPLE_SFTL_NO_ADDRESS, ++ }; ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, ++ 65536, &gc), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_COPY); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_checkpoint(gc, blocks, 1, &image), -EINVAL); ++ copy = apple_sftl_gc_next_copy(gc); ++ KUNIT_ASSERT_NOT_NULL(test, copy); ++ KUNIT_EXPECT_EQ(test, copy->logical, 0U); ++ KUNIT_EXPECT_EQ(test, copy->length, 2U); ++ KUNIT_EXPECT_EQ(test, copy->destination, 7 * APPLE_SFTL_STRIDE); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_copy_done(gc, copy, 202), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_CHECKPOINT); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_checkpoint(gc, blocks, 1, &image), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_checkpoint_done(gc, image), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_RECOVERY); ++ memcpy(recovered_runs, runs, sizeof(runs)); ++ recovered_runs[0].provider = 7 * APPLE_SFTL_STRIDE; ++ recovered = snapshot; ++ recovered.runs = recovered_runs; ++ recovered.sequence = image->sequence; ++ recovered.journal_sequence = image->sequence; ++ recovered.checkpoint_sequence = image->sequence; ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_recovered(gc, &recovered), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_VERIFIED); ++ /* The caller's original map and opaque accounting bytes remain intact. */ ++ KUNIT_EXPECT_EQ(test, runs[0].provider, 4 * APPLE_SFTL_STRIDE); ++ KUNIT_EXPECT_EQ(test, opaque->data[200], 0x6b); ++ apple_sftl_checkpoint_free(image); ++ apple_sftl_gc_free(gc); ++} ++ ++static void reclaim_gc_fail_stop(struct kunit *test) ++{ ++ struct apple_sftl_context_record *opaque = kunit_kmalloc(test, sizeof(*opaque), GFP_KERNEL); ++ struct apple_sftl_extent runs[2]; ++ struct apple_sftl_snapshot snapshot; ++ struct apple_sftl_gc_block victim = { ++ .block = 4, .lanes = 3, .closed = true, .last_sequence = 199, ++ }; ++ struct apple_sftl_gc_block destination = { .block = 7, .lanes = 3, .blank = true }; ++ struct apple_sftl_gc *gc = NULL; ++ struct apple_sftl_gc_copy receipt; ++ struct apple_sftl_checkpoint *image = NULL; ++ u32 blocks[] = { 5 }; ++ ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ reclaim_source(&snapshot, opaque, runs, 2); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, ++ 65536, &gc), 0); ++ receipt = *apple_sftl_gc_next_copy(gc); ++ receipt.destination++; ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_copy_done(gc, &receipt, 201), -EBADMSG); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_FAILED); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_checkpoint(gc, blocks, 1, &image), -EINVAL); ++ apple_sftl_gc_free(gc); ++ gc = NULL; ++ victim.reserved = true; ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, ++ 65536, &gc), -EINVAL); ++ victim.reserved = false; ++ destination.block = 4; ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, ++ 65536, &gc), -EINVAL); ++ KUNIT_EXPECT_PTR_EQ(test, gc, NULL); ++} ++ ++static void reclaim_obsolete_history_needs_checkpoint(struct kunit *test) ++{ ++ struct apple_sftl_context_record *opaque = kunit_kmalloc(test, sizeof(*opaque), GFP_KERNEL); ++ struct apple_sftl_extent runs[1]; ++ struct apple_sftl_snapshot snapshot; ++ struct apple_sftl_gc_block victim = { ++ .block = 4, .lanes = 3, .closed = true, .last_sequence = 199, ++ }; ++ struct apple_sftl_gc_block destination = { .block = 7, .lanes = 3, .blank = true }; ++ struct apple_sftl_gc *gc = NULL; ++ ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ reclaim_source(&snapshot, opaque, runs, 1); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, ++ 65536, &gc), 0); ++ KUNIT_EXPECT_PTR_EQ(test, apple_sftl_gc_next_copy(gc), NULL); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_CHECKPOINT); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_recovered(gc, &snapshot), -EINVAL); ++ apple_sftl_gc_free(gc); ++} ++ ++static void reclaim_checkpoint_opaque_and_invalid(struct kunit *test) ++{ ++ static const u8 subtypes[] = { 1, 2, 3, 5, 6 }; ++ struct apple_sftl_context_record *opaque; ++ struct apple_sftl_extent runs[1]; ++ struct apple_sftl_snapshot snapshot; ++ struct apple_sftl_checkpoint *image = NULL; ++ u32 blocks[] = { 5, 5 }; ++ u32 i; ++ ++ opaque = kunit_kmalloc(test, 5 * sizeof(*opaque), GFP_KERNEL); ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ reclaim_source(&snapshot, opaque, runs, 1); ++ for (i = 1; i < 5; i++) { ++ memset(&opaque[i], 0x50 + i, sizeof(opaque[i])); ++ apple_sftl_context_meta(opaque[i].meta, subtypes[i], 100 + i, i * 16, 16); ++ } ++ snapshot.nr_opaque = 5; ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_checkpoint_build(&snapshot, blocks, 2, 8, &image), -EINVAL); ++ KUNIT_ASSERT_EQ(test, apple_sftl_checkpoint_build(&snapshot, blocks, 1, 8, &image), 0); ++ for (i = 1; i < 5; i++) { ++ KUNIT_EXPECT_MEMEQ(test, image->records[i].data, opaque[i].data, 4096); ++ KUNIT_EXPECT_MEMEQ(test, image->records[i].meta + 8, opaque[i].meta + 8, 8); ++ } ++ apple_sftl_checkpoint_free(image); ++ image = NULL; ++ opaque[2].meta[1] = 1; ++ KUNIT_EXPECT_EQ(test, ++ apple_sftl_checkpoint_build(&snapshot, blocks, 1, 8, &image), -EBADMSG); ++ KUNIT_EXPECT_PTR_EQ(test, image, NULL); ++} ++ ++static void reclaim_checkpoint_torn_receipts(struct kunit *test) ++{ ++ struct apple_sftl_context_record *opaque = kunit_kmalloc(test, sizeof(*opaque), GFP_KERNEL); ++ struct apple_sftl_extent runs[1]; ++ struct apple_sftl_snapshot snapshot, recovered; ++ struct apple_sftl_gc_block victim = { ++ .block = 4, .lanes = 3, .closed = true, .last_sequence = 199, ++ }; ++ struct apple_sftl_gc_block destination = { .block = 7, .lanes = 3, .blank = true }; ++ struct apple_sftl_checkpoint *image; ++ struct apple_sftl_gc *gc; ++ u32 blocks[] = { 5 }, i; ++ ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ reclaim_source(&snapshot, opaque, runs, 1); ++ for (i = 0; i < 4; i++) { ++ KUNIT_ASSERT_EQ(test, ++ apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, ++ 65536, &gc), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_checkpoint(gc, blocks, 1, &image), 0); ++ image->records[i].meta[2] ^= 1; ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_checkpoint_done(gc, image), -EBADMSG); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_FAILED); ++ apple_sftl_checkpoint_free(image); ++ apple_sftl_gc_free(gc); ++ } ++ KUNIT_ASSERT_EQ(test, ++ apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, 65536, &gc), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_checkpoint(gc, blocks, 1, &image), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_gc_checkpoint_done(gc, image), 0); ++ recovered = snapshot; ++ recovered.sequence = image->sequence; ++ recovered.journal_sequence = image->sequence; ++ recovered.checkpoint_sequence = image->sequence - 1; ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_recovered(gc, &recovered), -EBADMSG); ++ KUNIT_EXPECT_EQ(test, apple_sftl_gc_stage(gc), APPLE_SFTL_GC_FAILED); ++ apple_sftl_checkpoint_free(image); ++ apple_sftl_gc_free(gc); ++} ++ ++static void reclaim_gc_boundaries(struct kunit *test) ++{ ++ struct apple_sftl_context_record *opaque = kunit_kmalloc(test, sizeof(*opaque), GFP_KERNEL); ++ struct apple_sftl_extent runs[2]; ++ struct apple_sftl_snapshot snapshot; ++ struct apple_sftl_gc_block victim = { ++ .block = 4, .lanes = 3, .closed = true, .last_sequence = 199, ++ }; ++ struct apple_sftl_gc_block destination = { .block = 7, .lanes = 1, .blank = true }; ++ struct apple_sftl_gc *gc = NULL; ++ int ret; ++ ++ KUNIT_ASSERT_NOT_NULL(test, opaque); ++ reclaim_source(&snapshot, opaque, runs, 2); ++ snapshot.journal_sequence--; ++ ret = apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, 65536, &gc); ++ KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); ++ snapshot.journal_sequence++; ++ ret = apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, 1, &gc); ++ KUNIT_EXPECT_EQ(test, ret, -ENOSPC); ++ runs[0].length = 481; ++ runs[1].logical = 481; ++ runs[1].length = APPLE_SFTL_LOGICAL_PAGES - 481; ++ ret = apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, 65536, &gc); ++ KUNIT_EXPECT_EQ(test, ret, -ENOSPC); ++ runs[0].length = 1; ++ runs[0].provider += 1000; ++ runs[1].logical = 1; ++ runs[1].length = APPLE_SFTL_LOGICAL_PAGES - 1; ++ ret = apple_sftl_gc_plan(&snapshot, &victim, &destination, 4, 65536, &gc); ++ KUNIT_EXPECT_EQ(test, ret, -EBADMSG); ++ KUNIT_EXPECT_PTR_EQ(test, gc, NULL); ++} ++ ++static struct kunit_case reclaim_cases[] = { ++ KUNIT_CASE(reclaim_metadata_vector), ++ KUNIT_CASE(reclaim_checkpoint_opaque_and_invalid), ++ KUNIT_CASE(reclaim_checkpoint_torn_receipts), ++ KUNIT_CASE(reclaim_gc_boundaries), ++ KUNIT_CASE(reclaim_checkpoint_fragments), ++ KUNIT_CASE(reclaim_gc_transaction), ++ KUNIT_CASE(reclaim_gc_fail_stop), ++ KUNIT_CASE(reclaim_obsolete_history_needs_checkpoint), ++ {} ++}; ++ ++static struct kunit_suite reclaim_suite = { ++ .name = "apple-sftl-reclaim", ++ .test_cases = reclaim_cases, ++}; ++ ++kunit_test_suite(reclaim_suite); ++MODULE_LICENSE("GPL"); +diff --git a/drivers/mtd/apple-sftl-reclaim.c b/drivers/mtd/apple-sftl-reclaim.c +new file mode 100644 +index 000000000..100e92b57 +--- /dev/null ++++ b/drivers/mtd/apple-sftl-reclaim.c +@@ -0,0 +1,498 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * SwissPPNFTL checkpoint serialization and offline reclamation transactions. ++ * Native metadata: 13G36 IOFlashStorage-730 formatter at 0x81092408. ++ * Range payload: cxtNextBatch at 0x8108f07c and the recovery decoder. ++ * ++ * Opaque accounting payloads survive byte-for-byte. Updating their native ++ * accounting and committing VFL bank rotation remain separate work. No erase ++ * callback or runtime GC worker is registered by this file. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "apple-sftl-reclaim.h" ++ ++#define SFTL_SEQUENCE GENMASK_ULL(47, 0) ++#define SFTL_PROVIDER (APPLE_SFTL_SUBBLOCKS * APPLE_SFTL_STRIDE) ++#define SFTL_PAIRS ((APPLE_SFTL_CHUNK - 8) / 8) ++#define SFTL_MAX_RECORDS (16U * APPLE_SFTL_STRIDE) ++#define SFTL_UNMAPPED 0x1ffffffU ++ ++struct apple_sftl_gc { ++ struct apple_sftl_snapshot snapshot; ++ struct apple_sftl_gc_copy *copies; ++ struct apple_sftl_checkpoint *checkpoint; ++ u32 nr_copies, copied, max_records, victim, destination; ++ u64 retire_sequence, copied_sequence; ++ enum apple_sftl_gc_stage stage; ++}; ++ ++int apple_sftl_context_meta(u8 *meta, u8 subtype, u64 sequence, ++ u16 index, u16 entries) ++{ ++ if (!meta || !subtype || subtype > 6 || sequence > SFTL_SEQUENCE) ++ return -EINVAL; ++ put_unaligned_le64(sequence << 16 | (u16)subtype << 8 | 0x1f, meta); ++ put_unaligned_le16(index, meta + 8); ++ put_unaligned_le16(entries, meta + 10); ++ put_unaligned_le32(U32_MAX, meta + 12); ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_context_meta); ++ ++static int sftl_check_map(const struct apple_sftl_snapshot *snapshot) ++{ ++ u32 cursor = 0, i; ++ ++ if (!snapshot || !snapshot->runs || !snapshot->nr_runs || ++ snapshot->nr_runs > APPLE_SFTL_LOGICAL_PAGES || ++ snapshot->checkpoint_sequence > snapshot->sequence || ++ snapshot->journal_sequence > snapshot->sequence || ++ snapshot->sequence > SFTL_SEQUENCE) ++ return -EINVAL; ++ for (i = 0; i < snapshot->nr_runs; i++) { ++ const struct apple_sftl_extent *run = &snapshot->runs[i]; ++ ++ if (run->logical != cursor || !run->length || ++ run->length > APPLE_SFTL_LOGICAL_PAGES - cursor || ++ (run->provider != APPLE_SFTL_NO_ADDRESS && ++ (run->provider >= SFTL_PROVIDER || ++ run->length > SFTL_PROVIDER - run->provider))) ++ return -EBADMSG; ++ cursor += run->length; ++ } ++ return cursor == APPLE_SFTL_LOGICAL_PAGES ? 0 : -EBADMSG; ++} ++ ++static int sftl_check_context(const struct apple_sftl_snapshot *snapshot) ++{ ++ u32 i, previous = 0, names; ++ ++ for (i = 0; i < snapshot->nr_opaque; i++) { ++ u8 subtype = snapshot->opaque[i].meta[1]; ++ u32 order = subtype >= 5 ? subtype - 1 : subtype; ++ ++ if (snapshot->opaque[i].meta[0] != 0x1f || !subtype || ++ subtype == 4 || subtype > 6 || (!i && subtype != 1) || ++ order < previous) ++ return -EBADMSG; ++ previous = order; ++ } ++ names = get_unaligned_le32(snapshot->opaque[0].data); ++ if (!names || names > 16) ++ return -EBADMSG; ++ return 0; ++} ++ ++static int sftl_checkpoint_count(const struct apple_sftl_snapshot *snapshot, ++ u32 max_records, u32 *count) ++{ ++ u32 records; ++ int ret = sftl_check_map(snapshot); ++ ++ if (ret) ++ return ret; ++ if (!snapshot->opaque || !snapshot->nr_opaque || ++ snapshot->nr_opaque > SFTL_MAX_RECORDS || ++ !max_records || max_records > SFTL_MAX_RECORDS) ++ return -EINVAL; ++ ret = sftl_check_context(snapshot); ++ if (ret) ++ return ret; ++ records = snapshot->nr_opaque + (snapshot->nr_runs + SFTL_PAIRS - 1) / SFTL_PAIRS; ++ records = (records + 3) & ~3U; ++ if (records > max_records) ++ return -ENOSPC; ++ if (records > SFTL_SEQUENCE - snapshot->sequence) ++ return -EOVERFLOW; ++ *count = records; ++ return 0; ++} ++ ++void apple_sftl_snapshot_free(struct apple_sftl_snapshot *snapshot) ++{ ++ if (!snapshot) ++ return; ++ kvfree(snapshot->runs); ++ kvfree(snapshot->opaque); ++ kfree(snapshot); ++} ++EXPORT_SYMBOL_GPL(apple_sftl_snapshot_free); ++ ++void apple_sftl_checkpoint_free(struct apple_sftl_checkpoint *checkpoint) ++{ ++ if (!checkpoint) ++ return; ++ kvfree(checkpoint->records); ++ kfree(checkpoint); ++} ++EXPORT_SYMBOL_GPL(apple_sftl_checkpoint_free); ++ ++int apple_sftl_checkpoint_build(const struct apple_sftl_snapshot *snapshot, ++ const u32 *blocks, u32 nr_blocks, ++ u32 max_records, struct apple_sftl_checkpoint **out) ++{ ++ struct apple_sftl_checkpoint *checkpoint; ++ u32 count, i, j, cursor = 0, run = 0, old_names; ++ int ret; ++ ++ if (!out) ++ return -EINVAL; ++ *out = NULL; ++ ret = sftl_checkpoint_count(snapshot, max_records, &count); ++ if (ret) ++ return ret; ++ if (!blocks || !nr_blocks || nr_blocks > 16) ++ return -EINVAL; ++ for (i = 0; i < nr_blocks; i++) { ++ if (!blocks[i] || blocks[i] >= APPLE_SFTL_SUBBLOCKS) ++ return -EINVAL; ++ for (j = 0; j < i; j++) ++ if (blocks[j] == blocks[i]) ++ return -EINVAL; ++ } ++ old_names = get_unaligned_le32(snapshot->opaque[0].data); ++ checkpoint = kzalloc(sizeof(*checkpoint), GFP_KERNEL); ++ if (!checkpoint) ++ return -ENOMEM; ++ checkpoint->records = kvzalloc(array_size(count, sizeof(*checkpoint->records)), GFP_KERNEL); ++ if (!checkpoint->records) { ++ kfree(checkpoint); ++ return -ENOMEM; ++ } ++ checkpoint->count = count; ++ checkpoint->first_sequence = snapshot->sequence + 1; ++ checkpoint->sequence = snapshot->sequence + count; ++ for (i = 0; i < snapshot->nr_opaque; i++) { ++ struct apple_sftl_context_record *record = &checkpoint->records[i]; ++ ++ *record = snapshot->opaque[i]; ++ apple_sftl_context_meta(record->meta, record->meta[1], ++ checkpoint->first_sequence + i, ++ get_unaligned_le16(record->meta + 8), ++ get_unaligned_le16(record->meta + 10)); ++ } ++ /* Only the decoded base list changes; its remaining native bytes survive. */ ++ memset(checkpoint->records[0].data + 4, 0, max(old_names, nr_blocks) * sizeof(u32)); ++ put_unaligned_le32(nr_blocks, checkpoint->records[0].data); ++ for (i = 0; i < nr_blocks; i++) ++ put_unaligned_le32(blocks[i], checkpoint->records[0].data + 4 + i * 4); ++ for (i = snapshot->nr_opaque; i < count; i++) { ++ struct apple_sftl_context_record *record = &checkpoint->records[i]; ++ ++ memset(record->data, 0xff, sizeof(record->data)); ++ apple_sftl_context_meta(record->meta, 4, ++ checkpoint->first_sequence + i, 0, 0); ++ if (run == snapshot->nr_runs) ++ continue; ++ put_unaligned_le32(cursor, record->data); ++ put_unaligned_le32(0xfffffff0, record->data + 4); ++ for (j = 0; j < SFTL_PAIRS && run < snapshot->nr_runs; j++, run++) { ++ const struct apple_sftl_extent *extent = &snapshot->runs[run]; ++ u32 address = extent->provider; ++ ++ if (address == APPLE_SFTL_NO_ADDRESS) ++ address = SFTL_UNMAPPED; ++ put_unaligned_le32(address, record->data + 8 + j * 8); ++ put_unaligned_le32(extent->length, record->data + 12 + j * 8); ++ cursor += extent->length; ++ } ++ } ++ *out = checkpoint; ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_checkpoint_build); ++ ++static bool sftl_overlap(u32 first, u32 length, u32 start, u32 end) ++{ ++ return first < end && start < first + length; ++} ++ ++static void sftl_gc_run(struct apple_sftl_snapshot *map, u32 logical, ++ u32 length, u32 provider) ++{ ++ struct apple_sftl_extent *run = &map->runs[map->nr_runs++]; ++ ++ *run = (struct apple_sftl_extent) { ++ .logical = logical, .length = length, .provider = provider, ++ }; ++} ++ ++static bool sftl_gc_block_valid(const struct apple_sftl_gc_block *block) ++{ ++ return block && block->block && block->block < APPLE_SFTL_SUBBLOCKS && ++ block->lanes && !block->reserved && block->last_sequence <= SFTL_SEQUENCE; ++} ++ ++int apple_sftl_gc_plan(const struct apple_sftl_snapshot *snapshot, ++ const struct apple_sftl_gc_block *victim, ++ const struct apple_sftl_gc_block *destination, ++ u32 max_records, size_t max_bytes, struct apple_sftl_gc **out) ++{ ++ struct apple_sftl_gc *gc; ++ u32 start, end, dest, capacity, copied = 0, i, count; ++ size_t map_bytes, copy_bytes, opaque_bytes; ++ int ret; ++ ++ if (!out) ++ return -EINVAL; ++ *out = NULL; ++ ret = sftl_checkpoint_count(snapshot, max_records, &count); ++ if (ret) ++ return ret; ++ if (snapshot->sequence != snapshot->journal_sequence) ++ return -EOPNOTSUPP; ++ if (!sftl_gc_block_valid(victim) || !sftl_gc_block_valid(destination) || ++ victim->blank || !victim->closed || !destination->blank || ++ destination->last_sequence || victim->block == destination->block || ++ victim->last_sequence > snapshot->sequence) ++ return -EINVAL; ++ start = victim->block * APPLE_SFTL_STRIDE; ++ end = start + hweight16(victim->lanes) * 512; ++ dest = destination->block * APPLE_SFTL_STRIDE; ++ /* Eight native footer pages remain available when the destination closes. */ ++ capacity = hweight16(destination->lanes) * 512 - 32; ++ for (i = 0; i < snapshot->nr_runs; i++) { ++ const struct apple_sftl_extent *run = &snapshot->runs[i]; ++ ++ if (run->provider != APPLE_SFTL_NO_ADDRESS && ++ sftl_overlap(run->provider, run->length, dest, dest + APPLE_SFTL_STRIDE)) ++ return -EBADMSG; ++ } ++ map_bytes = array_size(snapshot->nr_runs * 3, sizeof(*gc->snapshot.runs)); ++ copy_bytes = array_size(snapshot->nr_runs, sizeof(*gc->copies)); ++ opaque_bytes = array_size(snapshot->nr_opaque, sizeof(*snapshot->opaque)); ++ if (map_bytes > max_bytes || copy_bytes > max_bytes - map_bytes || ++ opaque_bytes > max_bytes - map_bytes - copy_bytes) ++ return -ENOSPC; ++ gc = kzalloc(sizeof(*gc), GFP_KERNEL); ++ if (!gc) ++ return -ENOMEM; ++ gc->snapshot.sequence = snapshot->sequence; ++ gc->snapshot.checkpoint_sequence = snapshot->checkpoint_sequence; ++ gc->snapshot.journal_sequence = snapshot->journal_sequence; ++ gc->max_records = max_records; ++ gc->victim = victim->block; ++ gc->destination = destination->block; ++ gc->retire_sequence = max(victim->last_sequence, snapshot->sequence); ++ gc->copied_sequence = snapshot->sequence; ++ gc->snapshot.runs = kvzalloc(map_bytes, GFP_KERNEL); ++ gc->copies = kvzalloc(copy_bytes, GFP_KERNEL); ++ gc->snapshot.opaque = kvzalloc(opaque_bytes, GFP_KERNEL); ++ if (!gc->snapshot.runs || !gc->copies || !gc->snapshot.opaque) { ++ ret = -ENOMEM; ++ goto failed; ++ } ++ memcpy(gc->snapshot.opaque, snapshot->opaque, ++ snapshot->nr_opaque * sizeof(*snapshot->opaque)); ++ gc->snapshot.nr_opaque = snapshot->nr_opaque; ++ for (i = 0; i < snapshot->nr_runs; i++) { ++ const struct apple_sftl_extent *run = &snapshot->runs[i]; ++ u32 before, live, offset, address = run->provider; ++ ++ if (address != APPLE_SFTL_NO_ADDRESS && ++ sftl_overlap(address, run->length, start, start + APPLE_SFTL_STRIDE) && ++ min(address + run->length, start + APPLE_SFTL_STRIDE) > end - 32) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ if (address == APPLE_SFTL_NO_ADDRESS || ++ !sftl_overlap(address, run->length, start, end)) { ++ sftl_gc_run(&gc->snapshot, run->logical, run->length, address); ++ continue; ++ } ++ before = start > address ? start - address : 0; ++ live = min(address + run->length, end) - max(address, start); ++ if (live > capacity - copied) { ++ ret = -ENOSPC; ++ goto failed; ++ } ++ if (before) ++ sftl_gc_run(&gc->snapshot, run->logical, before, address); ++ sftl_gc_run(&gc->snapshot, run->logical + before, live, dest + copied); ++ gc->copies[gc->nr_copies++] = (struct apple_sftl_gc_copy) { ++ .logical = run->logical + before, .length = live, ++ .source = address + before, .destination = dest + copied, ++ }; ++ copied += live; ++ offset = before + live; ++ if (offset < run->length) ++ sftl_gc_run(&gc->snapshot, run->logical + offset, ++ run->length - offset, address + offset); ++ } ++ /* Copy records consume sequences before checkpoint serialization. */ ++ if (copied > SFTL_SEQUENCE - snapshot->sequence) { ++ ret = -EOVERFLOW; ++ goto failed; ++ } ++ gc->snapshot.sequence += copied; ++ gc->snapshot.journal_sequence += copied; ++ ret = sftl_checkpoint_count(&gc->snapshot, max_records, &count); ++ if (ret) ++ goto failed; ++ gc->stage = gc->nr_copies ? APPLE_SFTL_GC_COPY : APPLE_SFTL_GC_CHECKPOINT; ++ *out = gc; ++ return 0; ++failed: ++ apple_sftl_gc_free(gc); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_plan); ++ ++void apple_sftl_gc_free(struct apple_sftl_gc *gc) ++{ ++ if (!gc) ++ return; ++ apple_sftl_checkpoint_free(gc->checkpoint); ++ kvfree(gc->snapshot.runs); ++ kvfree(gc->snapshot.opaque); ++ kvfree(gc->copies); ++ kfree(gc); ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_free); ++ ++enum apple_sftl_gc_stage apple_sftl_gc_stage(const struct apple_sftl_gc *gc) ++{ ++ return gc ? gc->stage : APPLE_SFTL_GC_FAILED; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_stage); ++ ++const struct apple_sftl_gc_copy *apple_sftl_gc_next_copy(const struct apple_sftl_gc *gc) ++{ ++ if (!gc || gc->stage != APPLE_SFTL_GC_COPY) ++ return NULL; ++ return &gc->copies[gc->copied]; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_next_copy); ++ ++int apple_sftl_gc_copy_done(struct apple_sftl_gc *gc, ++ const struct apple_sftl_gc_copy *receipt, u64 sequence) ++{ ++ const struct apple_sftl_gc_copy *expected = apple_sftl_gc_next_copy(gc); ++ ++ if (!expected || !receipt) ++ return -EINVAL; ++ if (memcmp(expected, receipt, sizeof(*receipt)) || ++ sequence != gc->copied_sequence + expected->length) { ++ gc->stage = APPLE_SFTL_GC_FAILED; ++ return -EBADMSG; ++ } ++ gc->copied_sequence = sequence; ++ if (++gc->copied == gc->nr_copies) ++ gc->stage = APPLE_SFTL_GC_CHECKPOINT; ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_copy_done); ++ ++int apple_sftl_gc_checkpoint(struct apple_sftl_gc *gc, const u32 *blocks, ++ u32 nr_blocks, struct apple_sftl_checkpoint **out) ++{ ++ struct apple_sftl_checkpoint *image; ++ u32 i; ++ int ret; ++ ++ if (!out) ++ return -EINVAL; ++ *out = NULL; ++ if (!gc || gc->stage != APPLE_SFTL_GC_CHECKPOINT || gc->checkpoint) ++ return -EINVAL; ++ if (!blocks || !nr_blocks || nr_blocks > 16) ++ return -EINVAL; ++ for (i = 0; i < nr_blocks; i++) ++ if (blocks[i] == gc->victim || blocks[i] == gc->destination) ++ return -EINVAL; ++ ret = apple_sftl_checkpoint_build(&gc->snapshot, blocks, nr_blocks, ++ gc->max_records, &gc->checkpoint); ++ if (ret) ++ return ret; ++ ret = apple_sftl_checkpoint_build(&gc->snapshot, blocks, nr_blocks, ++ gc->max_records, &image); ++ if (ret) { ++ apple_sftl_checkpoint_free(gc->checkpoint); ++ gc->checkpoint = NULL; ++ return ret; ++ } ++ *out = image; ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_checkpoint); ++ ++int apple_sftl_gc_checkpoint_done(struct apple_sftl_gc *gc, ++ const struct apple_sftl_checkpoint *readback) ++{ ++ const struct apple_sftl_checkpoint *expected; ++ ++ if (!gc || gc->stage != APPLE_SFTL_GC_CHECKPOINT || !gc->checkpoint || !readback) ++ return -EINVAL; ++ expected = gc->checkpoint; ++ if (!readback->records || readback->count != expected->count || ++ readback->first_sequence != expected->first_sequence || ++ readback->sequence != expected->sequence || ++ memcmp(readback->records, expected->records, ++ expected->count * sizeof(*expected->records))) { ++ gc->stage = APPLE_SFTL_GC_FAILED; ++ return -EBADMSG; ++ } ++ gc->stage = APPLE_SFTL_GC_RECOVERY; ++ return 0; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_checkpoint_done); ++ ++int apple_sftl_gc_recovered(struct apple_sftl_gc *gc, ++ const struct apple_sftl_snapshot *recovered) ++{ ++ u32 wanted = 0, found = 0, offset = 0, length; ++ int ret; ++ ++ if (!gc || gc->stage != APPLE_SFTL_GC_RECOVERY) ++ return -EINVAL; ++ ret = sftl_check_map(recovered); ++ if (ret) ++ goto failed; ++ if (recovered->checkpoint_sequence != gc->checkpoint->sequence || ++ recovered->sequence != gc->checkpoint->sequence || ++ recovered->journal_sequence != recovered->sequence || ++ recovered->checkpoint_sequence <= gc->retire_sequence) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ /* Compare logical mappings even when the cold reader coalesces ranges. */ ++ while (wanted < gc->snapshot.nr_runs && found < recovered->nr_runs) { ++ const struct apple_sftl_extent *a = &gc->snapshot.runs[wanted]; ++ const struct apple_sftl_extent *b = &recovered->runs[found]; ++ u32 a_offset = offset - a->logical, b_offset = offset - b->logical; ++ u32 a_address = a->provider, b_address = b->provider; ++ ++ if (a_address != APPLE_SFTL_NO_ADDRESS) ++ a_address += a_offset; ++ if (b_address != APPLE_SFTL_NO_ADDRESS) ++ b_address += b_offset; ++ if (a_address != b_address) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ length = min(a->length - a_offset, b->length - b_offset); ++ offset += length; ++ if (offset == a->logical + a->length) ++ wanted++; ++ if (offset == b->logical + b->length) ++ found++; ++ } ++ if (offset != APPLE_SFTL_LOGICAL_PAGES) { ++ ret = -EBADMSG; ++ goto failed; ++ } ++ gc->stage = APPLE_SFTL_GC_VERIFIED; ++ return 0; ++failed: ++ gc->stage = APPLE_SFTL_GC_FAILED; ++ return ret; ++} ++EXPORT_SYMBOL_GPL(apple_sftl_gc_recovered); +diff --git a/drivers/mtd/apple-sftl-reclaim.h b/drivers/mtd/apple-sftl-reclaim.h +new file mode 100644 +index 000000000..e7b110ab5 +--- /dev/null ++++ b/drivers/mtd/apple-sftl-reclaim.h +@@ -0,0 +1,88 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++#ifndef _APPLE_SFTL_RECLAIM_H ++#define _APPLE_SFTL_RECLAIM_H ++ ++#include ++ ++#define APPLE_SFTL_CHUNK 4096U ++#define APPLE_SFTL_META 16U ++#define APPLE_SFTL_STRIDE 8192U ++#define APPLE_SFTL_SUBBLOCKS 2086U ++#define APPLE_SFTL_LOGICAL_PAGES 15625016U ++#define APPLE_SFTL_NO_ADDRESS ((u32)~0U) ++ ++struct apple_sftl; ++ ++struct apple_sftl_extent { ++ u32 logical, length, provider; ++}; ++ ++struct apple_sftl_context_record { ++ u8 meta[APPLE_SFTL_META]; ++ u8 data[APPLE_SFTL_CHUNK]; ++}; ++ ++/* Owned whole-device map and unchanged non-map context payloads. */ ++struct apple_sftl_snapshot { ++ struct apple_sftl_extent *runs; ++ struct apple_sftl_context_record *opaque; ++ u32 nr_runs, nr_opaque; ++ u64 sequence, checkpoint_sequence, journal_sequence; ++}; ++ ++struct apple_sftl_checkpoint { ++ struct apple_sftl_context_record *records; ++ u32 count; ++ u64 first_sequence, sequence; ++}; ++ ++int apple_sftl_snapshot(struct apple_sftl *sftl, size_t max_bytes, ++ struct apple_sftl_snapshot **out); ++void apple_sftl_snapshot_free(struct apple_sftl_snapshot *snapshot); ++int apple_sftl_checkpoint_build(const struct apple_sftl_snapshot *snapshot, ++ const u32 *blocks, u32 nr_blocks, ++ u32 max_records, struct apple_sftl_checkpoint **out); ++void apple_sftl_checkpoint_free(struct apple_sftl_checkpoint *checkpoint); ++int apple_sftl_context_meta(u8 *meta, u8 subtype, u64 sequence, ++ u16 index, u16 entries); ++ ++/* A blank receipt describes an extent checked in full by the caller. */ ++struct apple_sftl_gc_block { ++ u32 block; ++ u16 lanes; ++ bool blank, closed, reserved; ++ u64 last_sequence; ++}; ++ ++struct apple_sftl_gc_copy { ++ u32 logical, length, source, destination; ++}; ++ ++enum apple_sftl_gc_stage { ++ APPLE_SFTL_GC_COPY, ++ APPLE_SFTL_GC_CHECKPOINT, ++ APPLE_SFTL_GC_RECOVERY, ++ APPLE_SFTL_GC_VERIFIED, ++ APPLE_SFTL_GC_FAILED, ++}; ++ ++struct apple_sftl_gc; ++ ++/* These routines plan and validate supplied bytes. They perform no media IO. */ ++int apple_sftl_gc_plan(const struct apple_sftl_snapshot *snapshot, ++ const struct apple_sftl_gc_block *victim, ++ const struct apple_sftl_gc_block *destination, ++ u32 max_records, size_t max_bytes, struct apple_sftl_gc **out); ++void apple_sftl_gc_free(struct apple_sftl_gc *gc); ++enum apple_sftl_gc_stage apple_sftl_gc_stage(const struct apple_sftl_gc *gc); ++const struct apple_sftl_gc_copy *apple_sftl_gc_next_copy(const struct apple_sftl_gc *gc); ++int apple_sftl_gc_copy_done(struct apple_sftl_gc *gc, ++ const struct apple_sftl_gc_copy *receipt, u64 sequence); ++int apple_sftl_gc_checkpoint(struct apple_sftl_gc *gc, const u32 *blocks, ++ u32 nr_blocks, struct apple_sftl_checkpoint **out); ++int apple_sftl_gc_checkpoint_done(struct apple_sftl_gc *gc, ++ const struct apple_sftl_checkpoint *readback); ++int apple_sftl_gc_recovered(struct apple_sftl_gc *gc, ++ const struct apple_sftl_snapshot *recovered); ++ ++#endif +diff --git a/drivers/mtd/apple-sftl-test.c b/drivers/mtd/apple-sftl-test.c +index e89eee42d..ec2d76a62 100644 +--- a/drivers/mtd/apple-sftl-test.c ++++ b/drivers/mtd/apple-sftl-test.c +@@ -9,6 +9,7 @@ + + #include "apple-sftl.h" + #include "apple-sftl-limits.h" ++#include "apple-sftl-reclaim.h" + + #define TEST_LOGICAL_PAGES 15625016U + #define TEST_STRIDE 8192U +@@ -59,6 +60,7 @@ struct sftl_fixture { + u16 program_lanes; + u64 sequence_bias; + struct sftl_test_page **written[5]; ++ struct apple_sftl_checkpoint *checkpoint; + }; + + static void sftl_test_meta(u8 *meta, u8 kind, u8 flags, u64 seq, u32 key) +@@ -271,7 +273,18 @@ static int sftl_test_read_page(void *priv, unsigned int bus, unsigned int cau, + } + if (lane > 1) + return 0; +- if (block == 1 && !position) { ++ if (block == 1 && f->checkpoint && position < f->checkpoint->count / 4) { ++ unsigned int i; ++ ++ for (i = 0; i < 4; i++) { ++ const struct apple_sftl_context_record *r; ++ ++ r = &f->checkpoint->records[position * 4 + i]; ++ ++ memcpy(data + i * 4096, r->data, 4096); ++ memcpy(oob + i * 16, r->meta, 16); ++ } ++ } else if (block == 1 && !position) { + sftl_test_checkpoint(f, data, oob); + } else if (block == 2 && !position) { + sftl_test_open(f, data, oob); +@@ -1027,6 +1040,36 @@ static void sftl_test_append_rejects_invalid_window(struct kunit *test) + } + #endif + ++static void sftl_test_checkpoint_cold_roundtrip(struct kunit *test) ++{ ++ struct sftl_fixture *f = sftl_test_fixture(test, SFTL_NO_FAULT); ++ struct apple_sftl_snapshot *before = NULL, *after = NULL; ++ struct apple_sftl_checkpoint *checkpoint = NULL; ++ struct apple_sftl *sftl = NULL; ++ u32 blocks[] = { 1 }; ++ ++ KUNIT_ASSERT_NOT_NULL(test, f); ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_EXPECT_EQ(test, apple_sftl_snapshot(sftl, 1, &before), -ENOSPC); ++ KUNIT_EXPECT_PTR_EQ(test, before, NULL); ++ KUNIT_ASSERT_EQ(test, apple_sftl_snapshot(sftl, 65536, &before), 0); ++ apple_sftl_free(sftl); ++ sftl = NULL; ++ KUNIT_ASSERT_EQ(test, apple_sftl_checkpoint_build(before, blocks, 1, 16, &checkpoint), 0); ++ f->checkpoint = checkpoint; ++ KUNIT_ASSERT_EQ(test, apple_sftl_load(&f->provider, &sftl), 0); ++ KUNIT_ASSERT_EQ(test, apple_sftl_snapshot(sftl, 65536, &after), 0); ++ KUNIT_EXPECT_EQ(test, after->checkpoint_sequence, checkpoint->sequence); ++ KUNIT_EXPECT_EQ(test, after->nr_runs, before->nr_runs); ++ KUNIT_EXPECT_MEMEQ(test, after->runs, before->runs, ++ before->nr_runs * sizeof(*before->runs)); ++ KUNIT_EXPECT_MEMEQ(test, after->opaque[0].data, before->opaque[0].data, 4096); ++ apple_sftl_free(sftl); ++ apple_sftl_snapshot_free(after); ++ apple_sftl_snapshot_free(before); ++ apple_sftl_checkpoint_free(checkpoint); ++} ++ + static struct kunit_case apple_sftl_cases[] = { + KUNIT_CASE(sftl_test_page_budget_stops_before_next_read), + KUNIT_CASE(sftl_test_deadline_stops_before_next_read), +@@ -1049,6 +1092,7 @@ static struct kunit_case apple_sftl_cases[] = { + KUNIT_CASE(sftl_test_append_rejects_invalid_window), + #endif + KUNIT_CASE(sftl_test_complete_medium), ++ KUNIT_CASE(sftl_test_checkpoint_cold_roundtrip), + KUNIT_CASE(sftl_test_scan_after_open_lane_hole), + KUNIT_CASE(sftl_test_runtime_failure_stops_reads), + KUNIT_CASE(sftl_test_reject_incomplete_or_ambiguous), diff --git a/kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch.toml b/kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0009-mtd-add-SwissPPNFTL-checkpoint-encoding-and-GC-plann.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"] diff --git a/kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch b/kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch new file mode 100644 index 00000000..47adc159 --- /dev/null +++ b/kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch @@ -0,0 +1,60 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Shayne Legault <8741557+shaynelegault@users.noreply.github.com> +Date: Fri, 6 Nov 2026 12:34:03 -0600 +Subject: [PATCH] docs: pin native PPN erase addressing and VFL rotation order + +Record the saved 64 GB geometry and native single-block row builder. The healthy VFL rotation path erases its unreferenced target, writes an initial checkpoint retaining old table references, copies the tables and writes the final checkpoint. Preserve the distinction between this source trace and physical interruption qualification. + +Clarify that planner blank, reservation and lane receipts require authoritative VFL ownership proof before future destructive IO. + +Assisted-by: LLM +--- + .../driver-api/mtd/apple-sftl-reclamation.rst | 26 ++++++++++++++----- + 1 file changed, 20 insertions(+), 6 deletions(-) + +diff --git a/Documentation/driver-api/mtd/apple-sftl-reclamation.rst b/Documentation/driver-api/mtd/apple-sftl-reclamation.rst +index e286c6935..31ac3e636 100644 +--- a/Documentation/driver-api/mtd/apple-sftl-reclamation.rst ++++ b/Documentation/driver-api/mtd/apple-sftl-reclamation.rst +@@ -56,6 +56,11 @@ The transaction advances through explicit receipt checks: + * A fresh recovery snapshot must contain the planned whole-device mapping + and the new checkpoint sequence. Equivalent coalescing is accepted. + ++The supplied blank, reserved and lane descriptions are trusted inputs. They ++must come from authoritative VFL and context ownership checks before any ++future destructive IO can use a plan. The present receipt checks cannot ++establish those physical ownership facts. ++ + A mismatched receipt moves the transaction to its failed state. Even a victim + with zero current references must pass checkpoint and cold-recovery stages, + since the current replay contract can still require its obsolete history. +@@ -67,14 +72,23 @@ Outstanding native protocol work + + VFL recovery uses two context groups with four copies per group on this + profile. The native rotation routine at ``0x810a1b68`` checks references into +-the target group before reuse, copies required lane tables and writes a final +-checkpoint page. Its interruption and replacement paths require further +-tracing before a runtime writer can reproduce them. ++the target group before reuse. On its healthy path, it erases the target ++members in SLC view and writes an initial ff/ff checkpoint retaining references ++to the old lane tables. It then copies those tables, updates the references ++and writes a final ff/ff checkpoint. The old source bank survives this path. ++The initial write at 0x810a1e5a and final write at 0x810a1e34 are separate ++persisted steps. Torn-copy selection, retirement and replacement paths still ++require tracing before a runtime writer can reproduce them. + + Offline execution of the pinned native iBEC erase queue establishes queued +-67/D7 commands, final-row 6A/D7, 79 polling and 77/7D completion. Linking its +-row construction to the 64 GB erase granule remains open, along with timeout, +-retirement and partial completion across CAUs. The provider exposes no erase ++67/D7 commands, final-row 6A/D7, 79 polling and 77/7D completion. The native single-block ++wrapper at 0x9ff09a0c shifts the block coordinate by the package's page-address ++bit count. The 13G36 provider builder at 0x8109de94 also supplies CAU and view ++fields. Both saved parameter copies agree: page bits 7, block bits 12 and CAU ++bits 3. A normal block's page-0 row is ``(cau << 19) | (block << 7)``; the SLC ++view adds bit 22. The normal view reports 128 pages per block. Physical erase ++effects, the SLC alias relationship, timeout, retirement and partial completion ++across CAUs remain unqualified. The provider exposes no erase + callback. Enabling physical reclamation requires resolving those semantics + and qualifying the exact candidate with device recovery and interruption + checks. diff --git a/kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch.toml b/kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch.toml new file mode 100644 index 00000000..1248e2bf --- /dev/null +++ b/kernel/patches/drivers/mtd/0010-docs-pin-native-PPN-erase-addressing-and-VFL-rotatio.patch.toml @@ -0,0 +1,2 @@ +schema = 1 +platforms = ["apple-armv7"]