diff --git a/Directory.Build.props b/Directory.Build.props index c10256f..b9ca4ab 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -8,7 +8,7 @@ latest true true - 2.0.3 + 2.1.0 arsenic9408 GPL-3.0-only diff --git a/MIGRATION.md b/MIGRATION.md index 369da23..65e6615 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,67 +1,78 @@ # Moving from Python MkPFS -MkPFS.CSharp is a drop-in replacement for the Python [MkPFS](https://github.com/PSBrew/MkPFS) 1.0.0 command -line. Scripts keep working: the program is still `mkpfs`, the commands and every flag have the same -names, and the console output has the same layout. +MkPFS.CSharp is a **drop-in replacement** for the Python [MkPFS](https://github.com/PSBrew/MkPFS) 1.0.0 +command line. Scripts keep working: the program is still `mkpfs`, every command and flag has the same name +and default, and the console output has the same layout. -## Install +## At a glance -| Python MkPFS | MkPFS.CSharp | -|---|---| -| `pip install mkpfs` or `uv tool install mkpfs` | Unpack `mkpfs--` from the [releases page](https://github.com/danthespal/MkPFS.CSharp/releases) | -| Needs Python 3.9+ with `cryptography` and, on x64 and ARM, `isal` | Single native executable plus `mkpfs_zlib`; no runtime to install | -| `mkpfs-gui` (customtkinter) | `mkpfs-gui--`; on macOS `MkPFS.CSharp.app` | +| | Python MkPFS | MkPFS.CSharp | +|---|---|---| +| **Install** | `pip install mkpfs` or `uv tool install mkpfs` | Unpack `mkpfs--` from the [releases page](https://github.com/danthespal/MkPFS.CSharp/releases) | +| **Needs** | Python 3.9+ with `cryptography`, and `isal` on x64 and ARM | Nothing: one native executable plus `mkpfs_zlib` | +| **Desktop app** | `mkpfs-gui` (customtkinter) | `mkpfs-gui--`; on macOS `MkPFS.CSharp.app` | +| **Compression** | ISA-L by default | Always zlib 1.3.1 | +| **Images** | — | Byte-identical to Python with `--compression-backend zlib` | ## Commands and flags -Every Python command and flag exists with the same name and default: - | Command | Change | |---|---| -| `pack folder`, `pack file`, `pack exfat` | Same flags. `--compression-backend` is accepted but ignored (see below). | +| `pack folder`, `pack file`, `pack exfat` | Same flags. `--compression-backend` is accepted but ignored ([why](#compression-backend)). | | `batch` | Same flags. | | `verify`, `inspect`, `tree`, `unpack` | Same flags. | -| `repair` | New: finds and fixes PFSC blocks the PS5 may decode wrongly. | +| `repair` | **New:** finds and fixes PFSC blocks the PS5 may decode wrongly. | +| `ampr` | **New (experimental):** AMPR Emu LZ4 asset packs. | +| `selftest` | **New:** checks that the bundled native libraries load. | -Exit codes match Python: 0 on success, 1 on errors, 2 on usage errors (missing arguments, unknown -or conflicting options). `repair --scan` adds 3 for "blocks need repair". Usage error messages -are worded differently from Python's argparse. +**Exit codes** match Python: `0` success, `1` error, `2` usage error (missing arguments, unknown or +conflicting options). `repair --scan` adds `3` for "blocks need repair". Usage error messages are worded +differently from Python's argparse. ## Behavior that differs -**Compression backend.** Python's default `auto` backend picks ISA-L, which Python MkPFS installs -on x64 and ARM. ISA-L streams use back-references the PS5 decodes wrongly (corrupted game files -after mounting). MkPFS.CSharp always compresses with zlib 1.3.1 at the same level (7 by default): -`auto` and `zlib` are accepted silently, `isal` and `zlib-ng` print a warning and use zlib. - -**Same images.** With the same source and flags, MkPFS.CSharp writes the same bytes as Python MkPFS -compressing with zlib, except for the build time stored in the image. In Python that takes -`--compression-backend zlib`, plus `--cpu-count 1` for large inputs, because its worker processes -ignore the backend option. Set `SOURCE_DATE_EPOCH` (seconds since 1970) to pin the build time for -reproducible builds. - -**Fixed Python bugs.** These Python results change (details in -[tools/oracle/README.md](tools/oracle/README.md), "Findings"): - -- `--skip-verification` works on its own; Python always rejected it. -- `verify` of signed images larger than 12 blocks no longer crashes. -- `verify --source-dir` reports each missing or extra path once, not twice, and compares an - exFAT-wrapped image with the files inside its exFAT instead of reporting every file missing. -- `unpack` refuses entry names that would escape the output folder. -- exFAT images use the exFAT up-case table for name hashes, so names such as `Straße` stay readable - on the console; `pack exfat` fails if a source file changes size while it is packed. -- `batch` prints the real PS4/PS5 profile, and `batch --verify` checks a folder item against the - files inside its image instead of failing every time. -- Game titles use the `param.json` default language instead of the first language in the file - (often Arabic). - -**Name.** The header line reads `MkPFS.CSharp - https://github.com/danthespal/MkPFS.CSharp`. -Tools that parse the first lines of the output should accept both names. +### Compression backend + +Python's default `auto` backend picks ISA-L, which Python MkPFS installs on x64 and ARM. ISA-L streams use +back-references the PS5 decodes wrongly, which corrupts game files after mounting. MkPFS.CSharp always +compresses with zlib 1.3.1 at the same level (7 by default): `auto` and `zlib` are accepted silently, `isal` +and `zlib-ng` print a warning and use zlib. + +### Same images + +With the same source and flags, MkPFS.CSharp writes the same bytes as Python MkPFS compressing with zlib, +except for the build time stored in the image. In Python that takes `--compression-backend zlib`, plus +`--cpu-count 1` for large inputs, because its worker processes ignore the backend option. Set +`SOURCE_DATE_EPOCH` (seconds since 1970) to pin the build time for reproducible builds. + +### Fixed Python bugs + +These Python results change (details in [tools/oracle/README.md](tools/oracle/README.md), "Findings"): + +| Area | Fix | +|---|---| +| `--skip-verification` | Works on its own; Python always rejected it. | +| `verify` | Signed images larger than 12 blocks no longer crash. | +| `verify --source-dir` | Reports each missing or extra path once, and compares an exFAT-wrapped image with the files inside its exFAT. | +| `unpack` | Refuses entry names that would escape the output folder. | +| exFAT images | Use the exFAT up-case table for name hashes, so names such as `Straße` stay readable on the console; `pack exfat` fails if a source file changes size while it is packed. | +| `batch` | Prints the real PS4/PS5 profile; `batch --verify` checks a folder item against the files inside its image. | +| Game titles | Use the `param.json` default language instead of the first language in the file (often Arabic). | +| `ampr_emu.index` | Hashed the way the console looks paths up, so files with non-ASCII names are found. | + +### Name + +The header line reads `MkPFS.CSharp - https://github.com/danthespal/MkPFS.CSharp`. Tools that parse +the first lines of the output should accept both names. ## New in MkPFS.CSharp -- `mkpfs repair` for single-file `.ffpfsc` images, with `--scan`, `--bad-blocks` (PS5 Game Compressor - `bad_blocks.tsv`), `--recompress`, `--mode`, and `--report-dir`. -- `verify` warns when compressed blocks use back-references the PS5 may decode wrongly and points to - `repair`. -- The GUI has a Repair page with a block map of the image. +- **`mkpfs repair`** for single-file `.ffpfsc` images, with `--scan`, `--bad-blocks` (PS5 Game Compressor + `bad_blocks.tsv`), `--recompress`, `--mode` and `--report-dir`. +- **`verify` warnings** when compressed blocks use back-references the PS5 may decode wrongly, pointing to `repair`. +- **APR Emu:** `pack exfat` and `batch` also write `ampr_emu.index`, and `--ampr-libs` copies AMPR Emu into `fakelib/`. +- **`mkpfs ampr`** (experimental): LZ4 asset packs that AMPR Emu reads on the fly, with rules from a TOML + profile or from traces recorded on the console. +- **Desktop app:** a Repair page with a block map, an AMPR Packs page, tooltips on every tuning option, and a + CPU core picker. The desktop app builds exFAT and FFPFSC images; `pack folder` and `batch` stay on the + command line. diff --git a/README.md b/README.md index 6ff1871..1081d0b 100644 --- a/README.md +++ b/README.md @@ -1,422 +1,578 @@ +
+ # MkPFS.CSharp +**Build, check and repair PS4/PS5 game images from the command line or a desktop app.** + [![CI](https://github.com/danthespal/MkPFS.CSharp/actions/workflows/ci.yml/badge.svg)](https://github.com/danthespal/MkPFS.CSharp/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/danthespal/MkPFS.CSharp?label=release)](https://github.com/danthespal/MkPFS.CSharp/releases) [![License: GPL-3.0](https://img.shields.io/badge/license-GPL--3.0-blue.svg)](LICENSE.md) +![.NET 10](https://img.shields.io/badge/.NET-10-512BD4) +![Platforms](https://img.shields.io/badge/platforms-Windows%20%7C%20Linux%20%7C%20macOS-555) -Build, check, and repair PS4/PS5 PFS game images (`.ffpfs`, `.ffpfsc`, `.exfat`) from the command line -or a desktop app. +The MkPFS.CSharp desktop app packing an exFAT image into a .ffpfsc -MkPFS.CSharp is a .NET 10 port of the Python [MkPFS](https://github.com/PSBrew/MkPFS) by PSBrew. It ships -as a native executable (no Python needed), writes the same images as the original, and adds an -offline PFSC block repair ported from PS5 Game Compressor. +
-## Features +MkPFS.CSharp is a .NET 10 port of the Python [MkPFS](https://github.com/PSBrew/MkPFS) by PSBrew. It is a +single native program (no Python needed), writes **the same images** as the original, and adds an offline +PFSC block repair ported from PS5 Game Compressor and LZ4 asset packs for AMPR Emu. -- **Pack**: a game folder, an exFAT image, or any single file into a compressed `.ffpfsc` (PFSC, zlib), - or a folder into a plain PFS image (`--raw`, with signed, encrypted, 64-bit inode, and PS4 options). -- **exFAT**: build exFAT images from game folders (deterministic, 64 KiB clusters by default). -- **Batch**: pack every game folder and image in a folder in one run. -- **Check**: `verify`, `inspect`, `tree`, and `unpack` for PFS, PFSC, and exFAT images, including - encrypted ones (`--ekpfs-key`). -- **Repair**: find and fix compressed blocks the PS5 may decode wrongly (images made with ISA-L). -- **APR Emu**: copies your AMPR Emu libraries into `fakelib/` of APR titles (`--ampr-libs`) and builds - `ampr_emu.index` when `pack folder`, `pack exfat`, or `batch` packs a game that has them. -- **GUI**: `mkpfs-gui` with a page per command, cover and metadata preview, batch queue, and a PFSC - block map; English, Português (BR), Español, Română, Deutsch, and Français. +> [!WARNING] +> **AMPR packing (`ampr`, the AMPR Packs page) is experimental.** It can shrink a game a lot, but whether +> the packed game runs depends on how that game reads its files, and only a test on the PS5 shows that. +> Keep the original game until the packed one has been played. See [AMPR asset packs](#ampr-asset-packs). -## Download +## Contents -Get the archive for your system from the -[releases page](https://github.com/danthespal/MkPFS.CSharp/releases) and unpack it anywhere: - -| System | Command line | Desktop app | -|---|---|---| -| Windows x64 | `mkpfs--win-x64.zip` | `mkpfs-gui--win-x64.zip` | -| Linux x64 | `mkpfs--linux-x64.tar.gz` | `mkpfs-gui--linux-x64.tar.gz` | -| macOS Apple silicon | `mkpfs--osx-arm64.tar.gz` | `mkpfs-gui--osx-arm64.tar.gz` | +- [Features](#features) +- [Quick start](#quick-start) +- [Download](#download) +- [Desktop app](#desktop-app) +- [Command line](#command-line) +- [APR Emu](#apr-emu) +- [AMPR asset packs](#ampr-asset-packs) +- [Differences from Python MkPFS](#differences-from-python-mkpfs) +- [Build from source](#build-from-source) +- [Credits and license](#credits-and-license) -- Keep each program in its folder with the libraries next to it (`mkpfs_zlib`, plus Skia and - HarfBuzz for the desktop app). -- macOS: the desktop app is `MkPFS.CSharp.app`. It is not notarized, so open it the first time with - right-click > Open. -- Linux: the desktop app needs X11 and fontconfig, which desktop distributions include. -- `SHA256SUMS.txt` on the release page lists the archive checksums. -- Check the command line with `mkpfs selftest`. +## Features -## Usage +| | Feature | What it does | +|---|---|---| +| 💿 | **exFAT images** | Turns a game folder into an `.exfat` image (deterministic, 64 KiB clusters). | +| 🗜️ | **FFPFSC images** | Compresses an exFAT image, a game folder or any single file into a `.ffpfsc` (PFSC, zlib). | +| 📁 | **Plain PFS** | Packs a folder as a `.ffpfs` (`--raw`), signed, encrypted, 64-bit inodes or PS4. | +| 📚 | **Batch** | Packs every game folder and image in a folder in one run. | +| 🔍 | **Check** | `verify`, `inspect`, `tree` and `unpack` for PFS, PFSC and exFAT, encrypted ones too. | +| 🩹 | **Repair** | Finds and fixes compressed blocks the PS5 may decode wrongly (images made with ISA-L). | +| 🎮 | **APR Emu** | Copies AMPR Emu into a game's `fakelib/` and writes its `ampr_emu.index`. | +| 📦 | **AMPR packs** *(experimental)* | Packs game data into LZ4 asset packs that AMPR Emu reads on the fly. | +| 🖥️ | **Desktop app** | Every command in a window, in English, Português (BR), Español, Română, Deutsch and Français. | + +## Quick start + +ShadowMountPlus works best with an **exFAT image wrapped in a compressed `.ffpfsc`**. The desktop app +follows the same two steps. + +```mermaid +flowchart LR + A["📁 Game folder
PPSA12345-app"] -->|"1. Pack exFAT"| B["💿 PPSA12345.exfat"] + B -->|"2. Pack FFPFSC"| C["🗜️ PPSA12345.ffpfsc"] + C -->|"copy"| D["🎮 PS5
ShadowMountPlus"] +``` -All paths may be absolute or relative to the current directory. Replace values in angle brackets -with your own paths; square brackets indicate an optional argument. Run `mkpfs --help` -for the parser's built-in help. +```bash +mkpfs pack exfat PPSA12345-app PPSA12345.exfat # 1. build the exFAT image +mkpfs pack file PPSA12345.exfat PPSA12345.ffpfsc # 2. compress it +mkpfs verify PPSA12345.ffpfsc --source-file PPSA12345.exfat # optional: check it +``` -### Commands at a glance +**Which file should I make?** -| Command | Arguments | Default result | Purpose | +| Output | Size | Made with | Use it for | |---|---|---|---| -| `pack folder ` | game/homebrew folder, output path | exFAT wrapped in a compressed `.ffpfsc` | Package a game folder. | -| `pack file ` | input file, output path | compressed `.ffpfsc` | Package one file in a PFS container. | -| `pack exfat [output]` | game/homebrew folder, optional output path | `.exfat` beside the source | Build an uncompressed exFAT image. | -| `batch ` | directory of folders/images, destination directory | one `.ffpfsc` per discovered item | Package many inputs; existing outputs are skipped. | -| `verify ` | image path | — | Validate an image, optionally against its source. | -| `inspect ` | image path | text report | Show image metadata and integrity information. | -| `tree ` | folder or image path | outer tree | List files and directories. | -| `unpack ` | image path, destination directory | — | Extract an image. | -| `repair ` | single-file `.ffpfsc` path | repairs risky blocks | Repair PFSC blocks that a PS5 may decode incorrectly. | - -### Common examples +| `.exfat` | Same as the game | `pack exfat` | The input of `pack file`, or a mount without compression. | +| `.ffpfsc` | Smaller (zlib) | `pack file`, `pack folder` | The usual image for ShadowMountPlus. | +| `.ffpfs` | Same as the game | `pack folder --raw` | Signed, encrypted or PS4 images. | +| AMPR packs | Much smaller (LZ4) | `ampr game` | Experimental: AMPR titles with a tested profile. | -Pack a game folder into a `.ffpfsc` (wrapped in exFAT and compressed in one pass): - -```bash -mkpfs pack folder PPSA12345-app PPSA12345.ffpfsc -``` +## Download -Compress an existing exFAT image: +Get the archive for your system from the [releases page](https://github.com/danthespal/MkPFS.CSharp/releases) +and unpack it anywhere. -```bash -mkpfs pack file PPSA12345.exfat PPSA12345.ffpfsc -``` +| System | Command line | Desktop app | +|---|---|---| +| Windows x64 | `mkpfs--win-x64.zip` | `mkpfs-gui--win-x64.zip` | +| Linux x64 | `mkpfs--linux-x64.tar.gz` | `mkpfs-gui--linux-x64.tar.gz` | +| macOS Apple silicon | `mkpfs--osx-arm64.tar.gz` | `mkpfs-gui--osx-arm64.tar.gz` | -Pack a folder directly as PFS (`--signed`, `--encrypted`, `--inode-bits 64`, and `--version PS4` apply -here): +> [!TIP] +> - Keep each program in its folder with the libraries next to it (`mkpfs_zlib`, plus Skia and HarfBuzz for +> the desktop app). +> - **macOS:** the desktop app is `MkPFS.CSharp.app`. It is not notarized, so open it the first time with +> right-click > Open. +> - **Linux:** the desktop app needs X11 and fontconfig, which desktop distributions include. +> - `SHA256SUMS.txt` on the release page lists the checksums. `mkpfs selftest` checks the command line. + +## Desktop app + +`mkpfs-gui` runs the same commands from a window and shows their output and progress. + + + + + + + + + + +
Pack exFAT page
Pack exFAT: game preview and APR Emu		Pack FFPFSC page
Pack FFPFSC: compression tuning
AMPR Packs page
AMPR Packs: build a playable packed game		Repair page with a block map
Repair: block map of risky blocks
+ +| Section | Page | What it does | +|---|---|---| +| **Build** | Pack exFAT | Game folder → `.exfat`, with the APR Emu libraries and index. | +| | Pack FFPFSC | `.exfat` (or any file) → compressed `.ffpfsc`. | +| | AMPR Packs | Every `ampr` command; *Build playable game* is the default. | +| **Check** | Verify | Checks an image, optionally against its source. | +| | Repair | Scans an image, draws a block map and repairs risky blocks. | +| **Read** | Inspect, Tree, Unpack | Shows details, lists files and extracts an image. | + +- **Game preview:** pick a game folder or image to see its cover, title, IDs, version, region and APR Emu marker. +- **Compression tuning:** presets (Fast, Balanced, Max, Low RAM), the zlib level, block size and keep rules. + **CPU Cores** offers *Auto*, which uses every physical core (not the logical processors), or any lower count. +- **Tooltips:** hover over any Compression Tuning or Advanced option to see what it does. +- **One progress bar per run:** a run with several steps (pack, then verify and compare) fills one bar, + and the label names the step, for example `verify (2/3)`. +- **Safe closing:** closing the window during a job asks first; *Stop and Close* waits for the job to clean up. +- `pack folder` and `batch` are available on the command line only. + +From a source checkout: `dotnet run --project src/MkPFS.Gui -c Release`. + +## Command line + +Paths may be absolute or relative. `<...>` is your value, `[...]` is optional, and +`mkpfs --help` shows every option. + +| Command | Result | Purpose | +|---|---|---| +| `pack exfat [output]` | `.exfat` | Build an uncompressed exFAT image. | +| `pack file ` | `.ffpfsc` | Compress one file (usually an `.exfat`) into a PFS image. | +| `pack folder ` | `.ffpfsc` | Wrap a folder in exFAT and compress it in one pass. | +| `batch ` | one `.ffpfsc` per item | Pack many folders and images; existing outputs are skipped. | +| `verify ` | report | Validate an image, optionally against its source. | +| `inspect ` | report | Show metadata and integrity information. | +| `tree ` | file list | List files and folders. | +| `unpack ` | files | Extract an image. | +| `repair ` | repaired image | Fix PFSC blocks the PS5 may decode wrongly. | +| `ampr ` | JSON | Build and manage AMPR Emu LZ4 asset packs ([details](#ampr-asset-packs)). | +| `selftest` | — | Check that the bundled native libraries load. | + +### Examples ```bash -mkpfs pack folder PPSA12345-app PPSA12345.ffpfs --raw +mkpfs pack exfat PPSA12345-app PPSA12345.exfat # game folder -> exFAT +mkpfs pack file PPSA12345.exfat PPSA12345.ffpfsc # exFAT -> compressed image +mkpfs pack folder PPSA12345-app PPSA12345.ffpfsc # both steps in one pass +mkpfs pack folder PPSA12345-app PPSA12345.ffpfs --raw # plain PFS (signed, encrypted, PS4...) +mkpfs batch ./games ./output # every game in a folder +mkpfs verify PPSA12345.ffpfsc --source-file PPSA12345.exfat # check against the source +mkpfs inspect PPSA12345.ffpfsc # details (--format json for scripts) +mkpfs tree PPSA12345.ffpfsc --deep # list files inside the wrapped exFAT +mkpfs unpack PPSA12345.ffpfsc out --deep # extract the game files +mkpfs repair PPSA12345.ffpfsc # fix risky blocks ``` -Build an exFAT image from a game folder: +### Exit codes -```bash -mkpfs pack exfat PPSA12345-app PPSA12345.exfat -``` +| Code | Meaning | +|---|---| +| `0` | Success. | +| `1` | The operation failed. | +| `2` | Invalid command line (missing argument, unknown or conflicting options); `ampr` errors also use 2. | +| `3` | `repair --scan` found blocks that need repair. | -Pack every game folder and image file in a folder (existing outputs are skipped): +### Options of `pack folder` and `pack file` -```bash -mkpfs batch ./games ./output -``` +The defaults are PS5, 32-bit inodes, case-insensitive names, zlib level 7, 64 KiB blocks and compression on. +The output extension is changed to `.ffpfsc` when needed. Single-file and `--raw` builds run a quick +structure check afterwards; the default exFAT-wrapped `pack folder` checks only with `--verify`. -Verify an image against its source: +For the default exFAT-wrapped `pack folder`, compression stays on with 64 KiB blocks. Use `--raw` to make +`--no-compress`, `--block-size`, `--inode-bits`, `--max-compressed-ratio`, `--min-compress-size` and +`--skip-executable-compression` take effect. -```bash -mkpfs verify PPSA12345.ffpfsc --source-file PPSA12345.exfat -``` +
+Compression options -Show image details (`--format json` for scripts): +| Option | Default | Meaning | +|---|---|---| +| `--compress` / `--no-compress` | on | Enable or disable PFSC block compression. | +| `--compression-level <0-9>` | `7` | zlib compression level. | +| `--cpu-count ` | `0` (auto) | Compression workers. Auto uses up to 16 and leaves one logical CPU free. | +| `--block-size ` | `auto` (65536) | Power of two from 4096 to 2097152. `auto-fit` reduces padding (folder packing and the spool builder). | +| `--threshold-gain <0-100>` | `0` | Keep a compressed block only when it saves at least this percentage. | +| `--max-compressed-ratio <0-100>` | `100` | Store a file raw when PFSC would exceed this percentage of its size. | +| `--min-compress-size ` | block size | Store smaller files raw without trying. | +| `--skip-executable-compression` | off | Do not compress important executables. | +| `--compression-backend <...>` | `auto` | Accepted for compatibility; this port always uses zlib 1.3.1. | -```bash -mkpfs inspect PPSA12345.ffpfsc -``` +
-List the files, including inside a wrapped exFAT: +
+Image layout and security options -```bash -mkpfs tree PPSA12345.ffpfsc --deep -``` +| Option | Default | Meaning | +|---|---|---| +| `--version ` | `PS5` | PFS profile version. | +| `--inode-bits <32\|64>` | `32` | PFS inode width. | +| `--case-sensitive` / `--case-insensitive` | insensitive | Name comparison mode. | +| `--signed` | off | Signed PFS with a zero EKPFS key and seed (not with the exFAT-wrapped `pack folder`). | +| `--encrypted` | off | Encrypt blocks with AES-XTS. | +| `--ekpfs-key <64-hex>` | all zeros | EKPFS key; needs `--encrypted`. | -Extract the game files: +
-```bash -mkpfs unpack PPSA12345.ffpfsc out --deep -``` +
+Output, checks and other options -Repair blocks the PS5 may decode wrongly in a single-file `.ffpfsc`: +| Option | Default | Meaning | +|---|---|---| +| `--adjust-output-file-extension` / `--no-adjust-output-file-extension` | adjust | Change the extension to match the pack mode, or keep the name as typed. | +| `--temp-folder ` | system temp | Where staged files are written. | +| `--verify` | off | Full verification after packing instead of the quick structure check. | +| `--verify-structure` / `--no-verify-structure` | on | Turn the quick check on or off. | +| `--skip-verification` | off | Skip every post-pack check; not with `--verify`. | +| `--dry-run` | off | Scan and report the layout without writing. | +| `--verbose` | off | Print per-file decisions. | +| `--raw` | off | `pack folder`: write a plain `.ffpfs` instead of the exFAT-wrapped `.ffpfsc`. | +| `--require-game-files` | off | `pack folder`: refuse to pack without `sce_sys/param.json` and `eboot.bin`. | +| `--use-spool` | off | `pack file`: use the older staged builder instead of writing straight into the image. | +| `--rename-inner-image` / `--no-rename-inner-image` | rename | `pack file`: normalize the file name stored inside, or keep it. | -```bash -mkpfs repair PPSA12345.ffpfsc -``` +`pack folder` also takes the [APR Emu options](#apr-emu). -### `pack folder` and `pack file` +
-Both commands require a source path and an output image path. By default, the output extension is -changed to `.ffpfsc` when necessary. The default build is PS5, 32-bit inodes, case-insensitive, -zlib level 7, 64 KiB blocks, and compression enabled. Direct-PFS folder builds and single-file -builds also run a quick structure verification by default. The default exFAT-wrapped folder flow -runs a post-pack check only when `--verify` is supplied. +### Options of `pack exfat` -For the default exFAT-wrapped `pack folder` flow, compression remains enabled and the PFS block -size remains 64 KiB. Use `--raw` to make options that control direct PFS layout or compression -(`--no-compress`, `--block-size`, `--inode-bits`, `--max-compressed-ratio`, -`--min-compress-size`, and `--skip-executable-compression`) take effect. +If `output` is omitted, the image is `.exfat` next to the source folder; an existing folder as +`output` receives that name. | Option | Default | Meaning | |---|---|---| -| `--adjust-output-file-extension` | on | Change the requested output extension to match the selected pack mode. | -| `--no-adjust-output-file-extension` | off | Keep the output filename exactly as supplied. Cannot be combined with `--adjust-output-file-extension`. | -| `--compress` / `--no-compress` | compression on | Enable or disable PFSC block compression. The two flags are mutually exclusive. | -| `--threshold-gain <0-100>` | `0` | Keep a compressed block only when it saves at least this percentage. | -| `--block-size ` | `auto` (`65536`) | PFS block size; it must be a power of two from 4096 through 2097152. `auto-fit` is supported by folder packing and the spool path to reduce file-data padding. | -| `--temp-folder ` | system temporary directory | Where staged pack artifacts are written. | -| `--version ` | `PS5` | PFS profile version. | -| `--inode-bits <32\|64>` | `32` | PFS inode-width mode. | -| `--case-sensitive` / `--case-insensitive` | case-insensitive | Select the PFS name-comparison mode. The two flags are mutually exclusive. | -| `--cpu-count ` | `0` (automatic) | PFSC compression workers. Automatic mode uses up to 16 workers and leaves one logical CPU free; a nonzero value is clamped to at least one. | -| `--compression-level <0-9>` | `7` | zlib compression level. | -| `--compression-backend ` | `auto` | Compatibility option. This port always uses zlib 1.3.1; unsupported values produce a warning. | -| `--max-compressed-ratio <0-100>` | `100` | Do not use PFSC when its stored size exceeds this percentage of the raw file size. | -| `--min-compress-size ` | resolved block size | Store smaller files raw without attempting PFSC compression. | -| `--skip-executable-compression` | off | Do not compress important executable files. | -| `--signed` | off | Build a signed PFS using a zero EKPFS/seed. It is not supported by the default exFAT-wrapped folder mode. | -| `--encrypted` | off | Encrypt filesystem blocks with AES-XTS. | -| `--ekpfs-key <64-hex>` | all-zero key | EKPFS key for an encrypted image; it requires `--encrypted`. | -| `--verbose` | off | Print per-file decisions. | -| `--dry-run` | off | Scan and report the planned layout without writing an image. | -| `--verify` | off | Run full post-pack verification instead of the default structure-only check. | -| `--verify-structure` / `--no-verify-structure` | structure check on | Explicitly enable or disable the default quick post-pack check. The two flags are mutually exclusive. | -| `--skip-verification` | off | Skip all post-pack verification. It cannot be combined with `--verify`. | +| `--cluster-size ` | `auto` (65536) | exFAT cluster size; 64 KiB suits ShadowMountPlus. | +| `--free-space ` | `0` | Free space inside the image, for example `2GiB`. Without it nothing can write to the mounted image. | +| `--overwrite` | off | Replace an existing image. | +| `--verbose` / `--no-progress` | off | More output / no progress bar. | -`pack folder` also accepts the [APR Emu options](#apr-emu) and the following: +It also takes the [APR Emu options](#apr-emu). -| Option | Default | Meaning | -|---|---|---| -| `--raw` | off | Package the source directly as a PFS `.ffpfs`, rather than making the default exFAT-wrapped `.ffpfsc`. Use this mode for `--signed`, `--inode-bits 64`, and other direct-PFS settings. | -| `--require-game-files` | off | Refuse to pack unless `sce_sys/param.json` and `eboot.bin` are present. | +### Options of `batch` -`pack file` also accepts: +`batch` finds packable folders and image files in `source_dir` and writes one `.ffpfsc` each into +`output_dir`, skipping existing ones. Its compression, profile, naming and encryption options are the same as +`pack` (`auto-fit` block sizes are not accepted). | Option | Default | Meaning | |---|---|---| -| `--use-spool` | off | Force the legacy staged/spool builder instead of direct-to-image streaming. | -| `--rename-inner-image` / `--no-rename-inner-image` | rename on | Normalize the filename stored inside the image (the first flag explicitly selects the default), or preserve the source filename. | +| `--overwrite` | off | Replace existing images. | +| `--dry-run` | off | Report the conversions without writing. | +| `--verify` | off | Fully verify each image. | -### `pack exfat` +Folder items also take the [APR Emu options](#apr-emu). -`mkpfs pack exfat [output]` creates an uncompressed exFAT image. If `output` is omitted, -the program derives `.exfat` beside the source directory; an output directory is also -accepted and receives that derived filename. +### Reading and extracting -| Option | Default | Meaning | -|---|---|---| -| `--cluster-size ` | `auto` (`65536`) | exFAT cluster size. The automatic 64 KiB value is optimized for SMP/LVD. | -| `--overwrite` | off | Replace an existing output image. | -| `--verbose` | off | Print detailed packing output. | -| `--no-progress` | off | Hide the progress bar written to standard error. | +Encrypted images take `--ekpfs-key <64-hex>` (default: all zeros) and `--new-crypt` (alternate key +derivation). `verify`, `tree` and `unpack` take `--format `. -It also accepts the [APR Emu options](#apr-emu). +| Command | Option | Meaning | +|---|---|---| +| `inspect` | `--format ` | Human-readable or JSON report. | +| `tree` | `--deep` | List the files inside a wrapped exFAT. | +| `unpack` | `--deep` | Extract the files inside a wrapped exFAT. | +| | `--only ` | With `--deep`, extract only this file or folder (repeatable). | +| | `--overwrite`, `--no-progress` | Replace existing output; hide progress. | +| `verify` | `--source-dir ` / `--source-file ` | Compare with the source folder or file. | +| | `--expect-crc32 ` | Require this payload CRC32. | +| | `--expect-manifest-sha256 <64-hex>` | Require this manifest SHA-256. | +| | `--require-game-files` | Warn when `sce_sys/param.json`, `eboot.bin` or `pfs-version.dat` is missing. | -### `batch` +### Repair -`mkpfs batch ` discovers packable folders and image files in `source_dir` -and writes `.ffpfsc` images into `output_dir`. It skips existing outputs by default. Its compression, -PFS-profile, naming, and encryption options have the same meanings and defaults as the corresponding -`pack` options: `--compress`/`--no-compress`, `--threshold-gain`, `--block-size` (`auto` = 65536; -`auto-fit` is not accepted), `--version`, `--inode-bits`, `--case-sensitive`/`--case-insensitive`, -`--cpu-count`, `--compression-level`, `--compression-backend`, `--max-compressed-ratio`, -`--min-compress-size`, `--skip-executable-compression`, `--encrypted`, `--ekpfs-key`, and `--verbose`. +`repair` works on an unsigned, unencrypted, single-file `.ffpfsc`. It scans for risky compressed blocks, +stores replacements raw, checks their content and cleans unused bytes in the outer PFS. | Option | Default | Meaning | |---|---|---| -| `--overwrite` | off | Replace images that already exist in the output directory. | -| `--dry-run` | off | Report the conversions without writing images. | -| `--verify` | off | Run full verification for each successful image. | -| `--compress` / `--no-compress` | compression on | Enable or disable compression; these flags are mutually exclusive. | - -Folder items get the [APR Emu options](#apr-emu) too. +| `--scan` | off | Report only; exit code 3 when blocks need repair. | +| `--bad-blocks ` | none | Repair the blocks listed in a PS5 Game Compressor `bad_blocks.tsv`. | +| `--recompress` | off | Re-encode repaired blocks with zlib level 7 instead of storing them raw. | +| `--mode ` | `auto` | `auto` copies when there is 1.2× the image size free, else rewrites in place. An interrupted `in-place` repair can leave the image corrupt. | +| `--report-dir ` | none | Write `summary.json` and `bad_blocks.tsv`. | +| `--no-slack-cleanup` | off | Leave unused outer bytes unchanged. | +| `--cpu-count ` | `0` (all cores) | Workers for scanning and repair. | -### APR Emu +## APR Emu -Some PS5 titles use PlayGo/APR and need Drakmor's APR Emu to run from a mounted image: the emulator -libraries in the game's `fakelib/` folder plus an `ampr_emu.index` that lists every file. MkPFS does -not ship or download the libraries. Download them into one folder and pass it with `--ampr-libs`: +Some PS5 titles use PlayGo/APR and need Drakmor's APR Emu to run from a mounted image: its libraries in the +game's `fakelib/` and an `ampr_emu.index` listing every file. MkPFS does not ship the libraries; download +them into one folder and pass it with `--ampr-libs`. | Library | Download | Required | |---|---|---| -| `libSceAmpr.sprx` | [drakmor/ampr_emu releases](https://github.com/drakmor/ampr_emu/releases) | yes | -| `libScePlayGo.sprx` | [drakmor/pgo_stub releases](https://github.com/drakmor/pgo_stub/releases) | copied when present | +| `libSceAmpr.sprx` | [drakmor/ampr_emu](https://github.com/drakmor/ampr_emu/releases) | yes | +| `libScePlayGo.sprx` | [drakmor/pgo_stub](https://github.com/drakmor/pgo_stub/releases) | copied when present | -Before packing a game folder, `pack folder`, `pack exfat`, and `batch` (folder items): +Before packing a folder, `pack folder`, `pack exfat` and `batch`: 1. With `--ampr-libs`, copy the libraries into `/fakelib/` when the game is an APR title - (`sce_sys/playgo-chunk.dat` exists) or `--ampr-title` is given. Identical files are left alone; - changed ones are replaced. -2. When `fakelib/libSceAmpr.sprx` exists, write `ampr_emu.index` into the game folder. - -Both steps change the source folder, so the image includes them. An APR title without -`fakelib/libSceAmpr.sprx` and without `--ampr-libs` gets a warning and no index. + (`sce_sys/playgo-chunk.dat` exists) or `--ampr-title` is given. Identical files are left alone. +2. When `fakelib/libSceAmpr.sprx` or `fakelib2/libSceAmpr.sprx` exists, write `ampr_emu.index`, hashed and + sorted the way the emulator looks paths up, so names with accents resolve on the console. -A folder that already has `ampr_emu.index` gets it rebuilt by default. To keep an index made by other -tools, pass `--ampr-skip-regen-if-exists`: the index is kept while it lists exactly the folder's files -and sizes (paths compared case-insensitively, modification times ignored) and rebuilt otherwise. The GUI -turns this on when the chosen folder already has an index. `--no-ampr-index` packs the existing index -untouched, even when it no longer matches. +Both steps change the source folder, so the image includes them. | Option | Default | Meaning | |---|---|---| -| `--ampr-libs ` | none | Folder holding `libSceAmpr.sprx` (required) and `libScePlayGo.sprx` (optional) to copy into `fakelib/`. | -| `--ampr-title` | off | With `--ampr-libs`, add the libraries even without `sce_sys/playgo-chunk.dat`. | -| `--no-ampr-index` | off | Do not create `ampr_emu.index` when `fakelib/libSceAmpr.sprx` is present. | -| `--ampr-skip-regen-if-exists` | off | Keep an existing index while it lists exactly the folder's files and sizes; rebuild it otherwise. | -| `--ampr-force-regen` | off | Regenerate an existing AMPR index. | +| `--ampr-libs ` | none | Folder with `libSceAmpr.sprx` (and optionally `libScePlayGo.sprx`). | +| `--ampr-title` | off | With `--ampr-libs`, add the libraries even without `playgo-chunk.dat`. | +| `--no-ampr-index` | off | Do not write `ampr_emu.index`. | +| `--ampr-skip-regen-if-exists` | off | Keep an existing index while it lists exactly the folder's files and sizes. | +| `--ampr-force-regen` | off | Always rebuild the index. | + +> [!NOTE] +> - ShadowMountPlus mounts `fakelib2/` instead of `fakelib/` when both exist, so `--ampr-libs` warns and +> leaves `fakelib2/` for you to update. +> - A folder that also holds AMPR packs (`ampr_assets.index`) keeps its `ampr_emu.index`: the packs address +> files by their row in it. `--ampr-force-regen` still rebuilds it, which breaks the packs. +> - The desktop app keeps an existing index by default. + +## AMPR asset packs + +> [!WARNING] +> Experimental. Keep the original, unpacked game until the packed one has been played on the PS5. + +### How it works + +`ampr` compresses a game's data files (levels, textures, audio, video) with LZ4 into a few pack files +(`ampr_assets-*.pak`) and a manifest (`ampr_assets.index`). On the PS5, Drakmor's +[AMPR Emu](https://github.com/drakmor/ampr_emu) decompresses them on the fly, so the game reads the original +bytes. Executables, modules and `sce_sys` are never packed. + +```mermaid +flowchart LR + G["🎮 Game"] -->|"opens data/level1.pak"| E["AMPR Emu
libSceAmpr.sprx"] + E -->|"reads blocks"| P["📦 ampr_assets-*.pak"] + P -->|"LZ4 decode"| E + E -->|"original bytes"| G +``` -### Reading and extracting images +AMPR Emu only serves files that the game reads in ways it intercepts (AMPR reads and ordinary +`open`/`read`/`stat` calls). If a game memory-maps a packed file, or reads it another way, the game usually +crashes. That is why each game needs its own rules, and why only a test on the console proves that a packed +game works. -Encrypted read commands accept `--ekpfs-key <64-hex>` (default: all-zero key) and `--new-crypt` -(default: off) to select the alternate EKPFS derivation. `verify`, `tree`, and `unpack` also take -`--format `; the default `auto` detects the format from the input. +### Requirements -| Command | Options | Default | Meaning | -|---|---|---|---| -| `inspect ` | `--format ` | `text` | Select a human-readable or JSON metadata report. | -| `tree ` | `--deep` | off | For a PFS that wraps one exFAT image, list the files inside that exFAT. | -| `unpack ` | `--overwrite` | off | Replace an existing output path. | -| | `--deep` | off | Extract files from an inner exFAT image instead of only the outer PFS contents. | -| | `--only ` | none; repeatable | With `--deep`, extract only the named inner exFAT file or directory. | -| | `--no-progress` | off | Hide extraction progress on standard error. | -| `verify ` | `--source-dir ` | none | Compare hierarchy and payloads against a source folder. Cannot be combined with `--source-file`. | -| | `--source-file ` | none | Compare a single-file image to the source file; not supported for exFAT input. | -| | `--expect-crc32 ` | none | Require this cumulative payload CRC32. | -| | `--expect-manifest-sha256 <64-hex>` | none | Require this manifest SHA256 digest. | -| | `--require-game-files` | off | Warn when `sce_sys/param.json`, `eboot.bin`, or `pfs-version.dat` is missing. | - -### `repair` - -`repair` operates on an unsigned, unencrypted, single-file `.ffpfsc`. By default it scans for risky -compressed blocks, stores replacements raw, verifies their decoded content, and cleans unused bytes -in the outer PFS wrapper. +| Need | Details | +|---|---| +| Console | A jailbroken PS5 with [ShadowMountPlus](https://github.com/drakmor/ShadowMountPlus). | +| Game | An AMPR/APR title (loads `libSceAmpr`). | +| Emulator | AMPR Emu **0.4.2.1 or newer**, the first release that reads packs; older builds are refused. | +| Mount | Read-only (`mount_read_only=1` in ShadowMountPlus, the default). | +| Rules | A **TOML profile** for the game, or **traces** you record yourself. | + +### Packing a game + +```mermaid +flowchart LR + A["📁 Game folder"] --> B{"Profile
for the game?"} + B -->|"yes"| D["ampr game
--config game.toml"] + B -->|"no"| C["Record traces
with debug AMPR Emu"] + C --> D2["ampr game
--traces traces"] + D --> E["📦 Packed folder
or .exfat"] + D2 --> E + E --> F["🎮 Test on the PS5"] +``` -| Option | Default | Meaning | -|---|---|---| -| `--scan` | off | Report only; do not modify the image. Returns exit code 3 if blocks need repair. | -| `--bad-blocks ` | none | Repair the block numbers in a PS5 Game Compressor `bad_blocks.tsv` instead of the scan's risky-block selection. | -| `--recompress` | off | Re-encode repaired blocks with zlib level 7 instead of storing them raw. | -| `--mode ` | `auto` | `auto` copy-replaces when free space is at least 1.2× the image, otherwise rewrites in place. `in-place` can leave an interrupted image corrupt; `copy` always uses a replacement copy. | -| `--report-dir ` | none | Write `summary.json` and `bad_blocks.tsv` to this directory. | -| `--no-slack-cleanup` | off | Leave unused bytes in the outer PFS wrapper unchanged. | -| `--cpu-count ` | `0` (all cores) | Worker count for scanning and repair. | -| `--no-progress` | off | Hide progress output. | +1. Check that the unpacked game runs with AMPR Emu in its `fakelib/`. +2. On the **AMPR Packs** page, choose **Build playable game** and fill in the game folder, a new output folder, + the folder with AMPR Emu's `libSceAmpr.sprx`, and the game's profile in **TOML Configuration** (or a + **Trace folder**). Optionally tick **Also build an exFAT image**. On the command line: + + ```bash + mkpfs ampr game --root PPSA12345-app --output PPSA12345-packed --fakelib ampr-emu --config game.toml --exfat . + ``` + +3. Copy the output folder or the `.exfat` image to the PS5. Play past the menu and load a save or a level, + since some files are only read later. +4. Delete the original only after that. + +`ampr game` copies the libraries, writes `ampr_emu.index`, packs, copies the files that stay loose, and checks +that every packed file decodes to the original. The game folder is only read; if a step fails, the output +folder is emptied. -Exit code `0` means success, `1` means an operation failed, `2` means invalid command-line usage, -and `repair --scan` uses `3` when it finds blocks that need repair. +### Making rules from traces -### GUI +The debug build of AMPR Emu (for example the 0.4.2.1 test-debug-pack) records every file the game reads +through AMPR. MkPFS turns those recordings into rules. -`mkpfs-gui` runs the same commands from a window and shows their output and progress. From a -source checkout: +1. Put the debug `libSceAmpr.sprx` into the unpacked game's `fakelib/` and set `mount_read_only=0` for this + run, so the emulator can write into the game folder. +2. Play a session, then quit the game from the PS5 menu. +3. Copy `ampr_commands.bin` and `ampr_emu.index` into a subfolder of a trace folder (for example + `traces/session1/`) and delete them from the game folder. More sessions cover more of the game. +4. Put the normal emulator back and set `mount_read_only=1` again. +5. Pick the trace folder in **Trace folder** (`--traces traces`); the page shows how many sessions it found. + +Only files a session read are packed. To also pack the other files of the same types (extensions), tick +**Also pack files no session read** (`--pack-untraced-types`), then test the parts of the game they belong to. + +To save the rules as a profile you can edit and share: ```bash -dotnet run --project src/MkPFS.Gui -c Release +mkpfs ampr profile generate \ + --trace traces/session1/ampr_commands.bin traces/session1/ampr_emu.index \ + --trace traces/session2/ampr_commands.bin traces/session2/ampr_emu.index \ + --output game.toml --pack-untraced-types ``` -- Pick a game folder or image to see its cover, title, IDs, version, region, and APR Emu marker. -- The Batch page lists every item it will pack before you run it. -- Pack Folder, Pack exFAT, and Batch have an APR Emu section: the libraries folder, download links, - and every [APR Emu option](#apr-emu). -- The packing pages have a collapsed Advanced section with the remaining CLI options: raw PFS, PFS - version, inode size, case sensitivity, encryption and EKPFS key, verification, cluster size, and - verbose output. -- The check pages cover the CLI options too: Verify takes a source file, the image format, and the - game-file checklist; Unpack extracts inside a wrapped exFAT (`--deep`, `--only`); Tree and Unpack - take the image format; Inspect, Tree, Verify, and Unpack take newCrypt keys. -- Closing the window while a job runs asks first; Stop and Close cancels the job and waits for its - cleanup (an in-place repair finishes its rewrite) before the window closes. -- Pack File, Pack Folder, and Batch have compression presets (Fast, Balanced, Max, Low RAM) and - settings for the zlib level, CPU cores, block size, and when to keep blocks uncompressed. -- The Repair page scans an image and draws a block map (zlib, raw, risky); click a cell for its - offset, stored size, and largest back-reference distance. +### When a packed game crashes -## Differences from Python MkPFS +The normal AMPR Emu writes no log. To find the file that fails: -- Same inputs give the same images as Python MkPFS 1.0.0 run with its zlib backend. Set - `SOURCE_DATE_EPOCH` for reproducible timestamps. -- Compression always uses zlib 1.3.1 at level 7. `--compression-backend` is accepted but ignored: - ISA-L output uses back-references the PS5 decodes wrongly. -- `repair` is new. -- `pack exfat` and `batch` also build `ampr_emu.index` (Python only does it in `pack folder`), and - `--ampr-libs`/`--ampr-title` are new. `--ampr-skip-regen-if-exists` checks every path and size, not - just the file count. -- Bugs found in the Python version while porting are listed in - [tools/oracle/README.md](tools/oracle/README.md); some are fixed here. -- Switching from Python MkPFS: see [MIGRATION.md](MIGRATION.md). +1. Build again with **Free space inside the image** set to `2GiB` (`--exfat-free-space 2GiB`). +2. Put the debug `libSceAmpr.sprx` into the packed game's `fakelib/`. +3. Mount the image read-write (`image_rw=` in ShadowMountPlus's `config.ini`) and start the game. +4. Read `ampr_emu.log` in the image: `apr.pack.open` lines are the packed files the game opened, + `apr.pack.open.fail` lines the ones it could not open. Keep those loose in the profile. -## Build from source +When you report a game, include its title, ID and version, the profile or traces used, what happened, and for +a crash the ShadowMountPlus log (`/data/shadowmount/debug.log`) and the console log. -Requirements: +### Profile format -- .NET SDK 10.0.401 or a newer 10.0.4xx (see `global.json`). -- Windows: Visual Studio 2026 with "Desktop development with C++" (native zlib and Native AOT). -- Linux and macOS: CMake and a C compiler; Linux Native AOT also needs `clang` and `zlib1g-dev`. +Profiles use ampr_emu's TOML format (see its `tools/ampr_pack.example.toml`). Rules are checked in order and +**the last match wins**; `*` also matches `/`. -Build and test: +```toml +[pack] +default_action = "loose" # files no rule matches stay as they are -```bash -dotnet build MkPFS.slnx -``` +[[rule]] +action = "compress" # LZ4, 64 KiB blocks by default +include = ["data/*"] -```bash -dotnet test --solution MkPFS.slnx +[[rule]] +action = "loose" # always keep executables, modules and system files loose +include = ["eboot.bin", "*.prx", "*.sprx", "sce_sys/*", "sce_module/*", "fakelib/*"] ``` -The first build compiles the bundled zlib in `native/`. Parity tests skip when the oracle corpus -(`tests/fixtures/generated`) is missing. +### `ampr` reference -Publish native executables: +| Subcommand | Required | Purpose | +|---|---|---| +| `game` | `--root --output`, and `--config` or `--traces` | Build a folder that runs from packs as is. | +| `pack` | `--root --ampr-index --output` | Write only the pack set (manifest, volumes, CRC file). | +| `verify` | `--index` | Decode every chunk; `--root` also compares with the source. | +| `unpack` | `--index --output` | Extract packed files (`--file `, `--overwrite`, `--no-preserve-mtime`). | +| `list` | `--index` | One line per file (`--json` for details). | +| `inspect` | `--index` | Manifest summary, volumes and runtime settings. | +| `runtime-config` | `--index --config` | Replace the runtime settings from a `[runtime]` section without repacking. | +| `remove-sources` | `--index --root` | Show the source files the packs replace; `--confirm` verifies and deletes them. | +| `profile generate` | `--output`, and a session folder or `--trace` | Rules from traces; `--report`, `--metrics`, `--runtime-header`. | +| `profile batch` | ` --output-dir` | One profile per trace session and a `summary.json`. | + +
+Options of ampr game and ampr pack -```bash -dotnet publish src/MkPFS.Cli -r win-x64 -c Release -``` +| Option | Default | Meaning | +|---|---|---| +| `--config ` | none | The game's profile. | +| `--traces ` | none | Rules from traces instead of a profile. | +| `--pack-untraced-types` | off | With `--traces`: also pack untraced files of the traced types. | +| `--workers ` | min(8, cores) | Compression threads (1 to 256). | +| `--self-contained` | off | Never leave selected files loose; store incompressible blocks uncompressed. | +| `--fakelib ` | none | `game`: AMPR Emu and other libraries to add to `fakelib/`. | +| `--exfat ` | none | `game`: also build an exFAT image. | +| `--exfat-free-space ` | 0 | `game`: free space inside the image, for a debug run. | +| `--skip-verify` | off | `game`: skip the final checks. | +| `--include`, `--exclude`, `--include-from`, `--exclude-from` | none | `pack`: narrow the rules; `--exclude` forces files loose. | +| `--require-packed ` | none | `pack`: fail if a matching file would stay loose. | +| `--allow-missing` | off | `pack`: leave selected files missing from `--root` loose. | +| `--no-progress` | off | Hide progress. | + +
+ +> [!NOTE] +> - `ampr pack` and `ampr profile` are ports of ampr_emu's `tools/ampr_pack.py` (4.0) and +> `tools/ampr_pack_profile.py` (4.1), with the same output byte for byte. `game`, `--traces` and +> `--pack-untraced-types` are MkPFS additions. +> - The release emulator loads at most 2,000,000 files, 16,000,000 chunks and 1,024 volumes; `pack` warns +> when a set goes over a limit. +> - Deploy `ampr_emu.index`, the manifest, its `.runtime` file and every volume from the same build. The +> `.crc` file is only used by `verify` and `unpack`. +> - Use an exFAT image or the plain folder, not `.ffpfsc`: zlib on top of LZ4 makes loading slower. + +## Differences from Python MkPFS + +| Area | MkPFS.CSharp | +|---|---| +| Output | The same images as Python MkPFS 1.0.0 with its zlib backend. Set `SOURCE_DATE_EPOCH` for reproducible timestamps. | +| Compression | Always zlib 1.3.1 (level 7 by default). `--compression-backend` is ignored: ISA-L output uses back-references the PS5 decodes wrongly. | +| New commands | `repair` and `ampr`. | +| APR Emu | `pack exfat` and `batch` also build `ampr_emu.index`; `--ampr-libs` and `--ampr-title` are new; `--ampr-skip-regen-if-exists` checks every path and size. | +| `ampr_emu.index` | Matches ampr_emu and the console lookup, so files with non-ASCII names are found; skips the emulator's own trace and log files; `fakelib2/` also counts. | +| Fixed bugs | Listed in [tools/oracle/README.md](tools/oracle/README.md). | + +Switching from Python MkPFS? See [MIGRATION.md](MIGRATION.md). + +## Build from source + +| Requirement | Details | +|---|---| +| .NET SDK | 10.0.401 or a newer 10.0.4xx (see `global.json`). | +| Windows | Visual Studio 2026 with *Desktop development with C++* (native zlib and Native AOT). | +| Linux, macOS | CMake and a C compiler; Linux Native AOT also needs `clang` and `zlib1g-dev`. | ```bash -dotnet publish src/MkPFS.Gui -r win-x64 -c Release +dotnet build MkPFS.slnx # the first build compiles zlib and lz4 in native/ +dotnet test --solution MkPFS.slnx # parity tests skip without the oracle corpus +dotnet publish src/MkPFS.Cli -r win-x64 -c Release # native command line +dotnet publish src/MkPFS.Gui -r win-x64 -c Release # native desktop app ``` -Use `linux-x64` or `osx-arm64` on those systems. On Windows, Native AOT linking needs the Visual -Studio Installer folder (`C:\Program Files (x86)\Microsoft Visual Studio\Installer`) on `PATH`. +Use `linux-x64` or `osx-arm64` on those systems. On Windows, Native AOT linking needs the Visual Studio +Installer folder (`C:\Program Files (x86)\Microsoft Visual Studio\Installer`) on `PATH`. ### Project layout | Path | Content | |---|---| -| `src/MkPFS.Core` | Formats and codecs: PFS, PFSC, exFAT, AMPR, crypto, readers, validators | -| `src/MkPFS.Build` | Image builders, compression planner, batch | -| `src/MkPFS.Repair` | PFSC repair (Game Compressor port) | -| `src/MkPFS.Cli` | `mkpfs` command line | -| `src/MkPFS.Gui` | `mkpfs-gui` desktop app (Avalonia) | -| `native/` | zlib 1.3.1 and the `mkpfs_zlib` shim | -| `tests/MkPFS.Tests` | Unit tests | -| `tests/MkPFS.Parity` | Byte-for-byte tests against the Python oracle corpus | -| `tests/MkPFS.Gui.Tests` | GUI view model and headless UI tests | -| `tools/oracle` | Python oracle scripts ([README](tools/oracle/README.md)) | +| `src/MkPFS.Core` | Formats and codecs: PFS, PFSC, exFAT, AMPR, crypto, readers, validators. | +| `src/MkPFS.Build` | Image builders, compression planner, batch, AMPR packer. | +| `src/MkPFS.Repair` | PFSC repair (Game Compressor port). | +| `src/MkPFS.Cli` | The `mkpfs` command line. | +| `src/MkPFS.Gui` | The `mkpfs-gui` desktop app (Avalonia). | +| `native/` | zlib 1.3.1, lz4 1.9.4 and the `mkpfs_zlib` shim. | +| `tests/` | Unit tests, byte-for-byte parity tests and headless GUI tests. | +| `tools/oracle` | Python oracle scripts ([README](tools/oracle/README.md)). | ### Oracle corpus -The parity tests compare against images and logs made by Python MkPFS. With Python MkPFS checked -out at `../MkPFS` and `uv` installed: +The parity tests compare against images and logs made by the Python tools. See +[tools/oracle/README.md](tools/oracle/README.md) for setup. ```bash -uv run --project ../MkPFS python tools/oracle/build_goldens.py --check +uv run --project ../MkPFS python tools/oracle/build_goldens.py --check # Python MkPFS +python tools/oracle/build_ampr_goldens.py --check # ampr_emu ampr_pack.py +uv run --no-project --python 3.12 --with lz4==4.4.5 python tools/oracle/check_ampr_profile.py ``` ### Releases -Push a version tag to publish a release. The workflow reruns CI, then uploads the archives, -`SHA256SUMS.txt`, and release notes, and marks the release as latest. With a `VT_API_KEY` repository -secret (a VirusTotal API key), it also scans every archive and links the reports in the notes. - -```bash -git tag v2.0.0 -``` - -```bash -git push origin v2.0.0 -``` +Push a version tag (`git tag v2.0.0` then `git push origin v2.0.0`). The release workflow reruns CI, uploads +the archives, `SHA256SUMS.txt` and release notes, and marks the release as latest. With a `VT_API_KEY` +repository secret (a VirusTotal API key) it also scans every archive and links the reports. -## Credits +## Credits and license -- [MkPFS](https://github.com/PSBrew/MkPFS) by PSBrew: the Python original this port follows. -- PS5 Game Compressor by Juma Sayeh: the PFSC repair logic. -- Drakmor's [APR Emu](https://github.com/drakmor/ampr_emu): the `ampr_emu.index` format. -- Drakmor's [PlayGo stub](https://github.com/drakmor/pgo_stub): `libScePlayGo.sprx`. - -Third-party components: - -- [zlib](https://zlib.net) 1.3.1 (zlib license) -- [Avalonia](https://avaloniaui.net), [CommunityToolkit.Mvvm](https://github.com/CommunityToolkit/dotnet), - [System.CommandLine](https://github.com/dotnet/command-line-api), and - [Spectre.Console](https://spectreconsole.net) (MIT) -- [Material Design Icons](https://pictogrammers.com/library/mdi/) (Apache-2.0) - -## License - -GPL-3.0-only, same as MkPFS. See [LICENSE.md](LICENSE.md). - -This project is not affiliated with Sony Interactive Entertainment. +| Project | Used for | +|---|---| +| [MkPFS](https://github.com/PSBrew/MkPFS) by PSBrew | The Python original this port follows. | +| PS5 Game Compressor by Juma Sayeh | The PFSC repair logic. | +| [APR Emu](https://github.com/drakmor/ampr_emu) by Drakmor | `ampr_emu.index` and the asset-pack format and tools that `ampr` ports. | +| [PlayGo stub](https://github.com/drakmor/pgo_stub) by Drakmor | `libScePlayGo.sprx`. | + +Third-party components: [zlib](https://zlib.net) 1.3.1 (zlib license), [LZ4](https://github.com/lz4/lz4) +1.9.4 and [Tomlyn](https://github.com/xoofx/Tomlyn) (BSD-2-Clause), [Avalonia](https://avaloniaui.net), +[CommunityToolkit.Mvvm](https://github.com/CommunityToolkit/dotnet), +[System.CommandLine](https://github.com/dotnet/command-line-api) and [Spectre.Console](https://spectreconsole.net) +(MIT), and [Material Design Icons](https://pictogrammers.com/library/mdi/) (Apache-2.0). + +Licensed under **GPL-3.0-only**, like MkPFS; see [LICENSE.md](LICENSE.md). This project is not affiliated with +Sony Interactive Entertainment. diff --git a/assets/screenshots/gui-ampr-packs.png b/assets/screenshots/gui-ampr-packs.png new file mode 100644 index 0000000..4c8ea8c Binary files /dev/null and b/assets/screenshots/gui-ampr-packs.png differ diff --git a/assets/screenshots/gui-pack-exfat.png b/assets/screenshots/gui-pack-exfat.png new file mode 100644 index 0000000..f18c7d2 Binary files /dev/null and b/assets/screenshots/gui-pack-exfat.png differ diff --git a/assets/screenshots/gui-pack-ffpfsc.png b/assets/screenshots/gui-pack-ffpfsc.png new file mode 100644 index 0000000..b90097a Binary files /dev/null and b/assets/screenshots/gui-pack-ffpfsc.png differ diff --git a/assets/screenshots/gui-repair.png b/assets/screenshots/gui-repair.png new file mode 100644 index 0000000..3f60be6 Binary files /dev/null and b/assets/screenshots/gui-repair.png differ diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index a86c0f7..3106cde 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -19,8 +19,13 @@ add_library(mkpfs_zlib SHARED zlib/trees.c zlib/uncompr.c zlib/zutil.c + mkpfs_lz4.c + lz4/lz4.c + lz4/lz4hc.c ) -target_include_directories(mkpfs_zlib PRIVATE zlib) +target_include_directories(mkpfs_zlib PRIVATE zlib lz4) +# lz4.h marks its API visibility("default") on GCC/Clang; keep it hidden like zlib. +target_compile_definitions(mkpfs_zlib PRIVATE LZ4LIB_VISIBILITY=) if(MSVC) target_compile_definitions(mkpfs_zlib PRIVATE _CRT_SECURE_NO_DEPRECATE _CRT_NONSTDC_NO_DEPRECATE) diff --git a/native/build.ps1 b/native/build.ps1 index 8d31bc7..bc9e36e 100644 --- a/native/build.ps1 +++ b/native/build.ps1 @@ -1,6 +1,6 @@ <# .SYNOPSIS - Build native/mkpfs_zlib (zlib 1.3.1 + shim) for Windows into artifacts/native//. + Build native/mkpfs_zlib (zlib 1.3.1 + lz4 1.9.4 + shims) for Windows into artifacts/native//. .EXAMPLE powershell -NoProfile -ExecutionPolicy Bypass -File native/build.ps1 -Rid win-x64 #> diff --git a/native/build.sh b/native/build.sh index 2b6d78f..9ba7537 100644 --- a/native/build.sh +++ b/native/build.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Build native/mkpfs_zlib (zlib 1.3.1 + shim) for Linux/macOS into artifacts/native//. +# Build native/mkpfs_zlib (zlib 1.3.1 + lz4 1.9.4 + shims) for Linux/macOS into artifacts/native//. # Usage: sh native/build.sh linux-x64|linux-arm64|osx-x64|osx-arm64 [Release] set -eu diff --git a/native/lz4/LICENSE b/native/lz4/LICENSE new file mode 100644 index 0000000..4884916 --- /dev/null +++ b/native/lz4/LICENSE @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2011-2020, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/native/lz4/ORIGIN.md b/native/lz4/ORIGIN.md new file mode 100644 index 0000000..a0a9ce7 --- /dev/null +++ b/native/lz4/ORIGIN.md @@ -0,0 +1,12 @@ +# lz4 1.9.4 + +| | | +|---|---| +| **Source** | https://github.com/lz4/lz4/tree/v1.9.4 | +| **Commit** | `5ff839680134437dbf4678f3d0c7b371d84f4964` | +| **Files** | `lib/lz4.c`, `lib/lz4.h`, `lib/lz4hc.c`, `lib/lz4hc.h` and `lib/LICENSE` (BSD-2-Clause) | +| **Changes** | None | + +The files are identical (apart from line endings) to `lz4libs/` in python-lz4 v4.4.5, the encoder of the AMPR +pack oracle (ampr_emu `tools/ampr_pack.py`). lz4 1.10.0 changes HC levels 1 and 2, so it is not used. The +decoder accepts blocks from any lz4 version; the ampr_emu runtime decodes with 1.10.0. diff --git a/native/lz4/lz4.c b/native/lz4/lz4.c new file mode 100644 index 0000000..654bfdf --- /dev/null +++ b/native/lz4/lz4.c @@ -0,0 +1,2722 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2020, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4_HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 +#endif + +/* + * LZ4_ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + + +/*-************************************ +* Dependency +**************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +#define LZ4_STATIC_LINKING_ONLY +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ +#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */ +#endif /* _MSC_VER */ + +#ifndef LZ4_FORCE_INLINE +# ifdef _MSC_VER /* Visual Studio */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif + + +/*-************************************ +* Memory routines +**************************************/ + +/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION : + * Disable relatively high-level LZ4/HC functions that use dynamic memory + * allocation functions (malloc(), calloc(), free()). + * + * Note that this is a compile-time switch. And since it disables + * public/stable LZ4 v1 API functions, we don't recommend using this + * symbol to generate a library for distribution. + * + * The following public functions are removed when this symbol is defined. + * - lz4 : LZ4_createStream, LZ4_freeStream, + * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated) + * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC, + * LZ4_createHC (deprecated), LZ4_freeHC (deprecated) + * - lz4frame, lz4file : All LZ4F_* functions + */ +#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +# define ALLOC(s) lz4_error_memory_allocation_is_disabled +# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled +# define FREEMEM(p) lz4_error_memory_allocation_is_disabled +#elif defined(LZ4_USER_MEMORY_FUNCTIONS) +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + +#if ! LZ4_FREESTANDING +# include /* memset, memcpy */ +#endif +#if !defined(LZ4_memset) +# define LZ4_memset(p,v,s) memset((p),(v),(s)) +#endif +#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + + +/*-************************************ +* Types +**************************************/ +#include +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + +/*-************************************ +* Reading and writing into memory +**************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if !defined(LZ4_memcpy) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +# else +# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +# endif +#endif + +#if !defined(LZ4_memmove) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memmove __builtin_memmove +# else +# define LZ4_memmove memmove +# endif +#endif + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) LZ4_unalign; + +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalign*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign*)memPtr)->u32 = value; } + +#else /* safe and portable access using memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); +#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */ +# pragma warning(push) +# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ +#endif + LZ4_memcpy(&v[4], v, 4); +#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */ +# pragma warning(pop) +#endif + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} +#endif + + +/*-************************************ +* Common functions +**************************************/ +static unsigned LZ4_NbCommonBytes (reg_t val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT) +/*-************************************************************************************************* +* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. +* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics +* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. +****************************************************************************************************/ +# if defined(__clang__) && (__clang_major__ < 10) + /* Avoid undefined clang-cl intrinsics issue. + * See https://github.com/lz4/lz4/pull/1017 for details. */ + return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; +# else + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# endif +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; +# else + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; +# else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ + unsigned r; + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +#endif +# endif + } else /* 32 bits */ { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } + } +} + + +#define STEPSIZE sizeof(reg_t) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); + } + + if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + + +/*-************************************ +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); } + + +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize); +#if defined (__cplusplus) +} +#endif + +/*-****************************** +* Compression functions +********************************/ +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + } else { + const U64 prime8bytes = 11400714785074694791ULL; + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } +} + +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); +} + +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType, + const BYTE* srcBase) +{ + switch (tableType) + { + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time. + * Presumed already validated at this stage: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + int* inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + int result; + const BYTE* ip = (const BYTE*) source; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; + const BYTE* lowLimit; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; + const BYTE* const matchlimit = iend - LASTLITERALS; + + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 offset = 0; + U32 forwardH; + + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + const BYTE* filledIp; + + /* Find a match */ + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); + } + + /* Catch up */ + filledIp = ip; + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } + if (litLength >= RUN_MASK) { + int len = (int)(litLength - RUN_MASK); + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + + /* Encode Offset */ + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } + + /* Encode MatchLength */ + { unsigned matchCode; + + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; + if (ip==limit) { + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); + } + + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); + + anchor = ip; + + /* Test end of chunk */ + if (ip >= mflimitPlusOne) break; + + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + + /* Test next position */ + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); +} + + +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputSize) +{ + return LZ4_compress_fast(src, dst, srcSize, maxOutputSize, 1); +} + + +/* Note!: This function leaves the stream in an unclean/broken state! + * It is not safe to subsequently use the same state with a _fastReset() or + * _continue() call without resetting it. */ +static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } else { + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); + } } +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (LZ4_HEAPMODE) + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + +#if (LZ4_HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/*-****************************** +* Streaming functions +********************************/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); + return lz4s; +} +#endif + +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); + FREEMEM(LZ4_stream); + return (0); +} +#endif + + +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; + + if (dictSize < (int)HASH_UNIT) { + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + base = dictEnd - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = (U32)tableType; + + while (p <= dictEnd-HASH_UNIT) { + LZ4_putPosition(p, dict->hashTable, tableType, base); + p+=3; + } + + return (int)dict->dictSize; +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) +{ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + DEBUGLOG(4, "LZ4_renormDictT"); + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) +{ + const tableType_t tableType = byU32; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); + + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = source; + } + + /* Check overlapping input/dictionary space */ + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == source) { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); + else + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); + } + + /* external dictionary mode */ + { int result; + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { /* small data <= 4 KB */ + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; + int result; + + LZ4_renormDictT(streamPtr, srcSize); + + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)srcSize; + + return result; +} + + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); + + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } + + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize); + } + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + + +/* variant for decompress_unsafe() + * does not know end of input + * presumes input is well formed + * note : will consume at least one byte */ +size_t read_long_length_no_check(const BYTE** pp) +{ + size_t b, l = 0; + do { b = **pp; (*pp)++; l += b; } while (b==255); + DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1) + return l; +} + +/* core decoder variant for LZ4_decompress_fast*() + * for legacy support only : these entry points are deprecated. + * - Presumes input is correctly formed (no defense vs malformed inputs) + * - Does not know input size (presume input buffer is "large enough") + * - Decompress a full block (only) + * @return : nb of bytes read from input. + * Note : this variant is not optimized for speed, just for maintenance. + * the goal is to remove support of decompress_fast*() variants by v2.0 +**/ +LZ4_FORCE_INLINE int +LZ4_decompress_unsafe_generic( + const BYTE* const istart, + BYTE* const ostart, + int decompressedSize, + + size_t prefixSize, + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note: =0 if dictStart==NULL */ + ) +{ + const BYTE* ip = istart; + BYTE* op = (BYTE*)ostart; + BYTE* const oend = ostart + decompressedSize; + const BYTE* const prefixStart = ostart - prefixSize; + + DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); + if (dictStart == NULL) assert(dictSize == 0); + + while (1) { + /* start new sequence */ + unsigned token = *ip++; + + /* literals */ + { size_t ll = token >> ML_BITS; + if (ll==15) { + /* long literal length */ + ll += read_long_length_no_check(&ip); + } + if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */ + LZ4_memmove(op, ip, ll); /* support in-place decompression */ + op += ll; + ip += ll; + if ((size_t)(oend-op) < MFLIMIT) { + if (op==oend) break; /* end of block */ + DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op); + /* incorrect end of block : + * last match must start at least MFLIMIT==12 bytes before end of output block */ + return -1; + } } + + /* match */ + { size_t ml = token & 15; + size_t const offset = LZ4_readLE16(ip); + ip+=2; + + if (ml==15) { + /* long literal length */ + ml += read_long_length_no_check(&ip); + } + ml += MINMATCH; + + if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */ + + { const BYTE* match = op - offset; + + /* out of range */ + if (offset > (size_t)(op - prefixStart) + dictSize) { + DEBUGLOG(6, "offset out of range"); + return -1; + } + + /* check special case : extDict */ + if (offset > (size_t)(op - prefixStart)) { + /* extDict scenario */ + const BYTE* const dictEnd = dictStart + dictSize; + const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart)); + size_t const extml = (size_t)(dictEnd - extMatch); + if (extml > ml) { + /* match entirely within extDict */ + LZ4_memmove(op, extMatch, ml); + op += ml; + ml = 0; + } else { + /* match split between extDict & prefix */ + LZ4_memmove(op, extMatch, extml); + op += extml; + ml -= extml; + } + match = prefixStart; + } + + /* match copy - slow variant, supporting overlap copy */ + { size_t u; + for (u=0; u= ipmax before start of loop. Returns initial_error if so. + * @error (output) - error code. Must be set to 0 before call. +**/ +typedef size_t Rvl_t; +static const Rvl_t rvl_error = (Rvl_t)(-1); +LZ4_FORCE_INLINE Rvl_t +read_variable_length(const BYTE** ip, const BYTE* ilimit, + int initial_check) +{ + Rvl_t s, length = 0; + assert(ip != NULL); + assert(*ip != NULL); + assert(ilimit != NULL); + if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + return rvl_error; + } + do { + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length)<8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + } while (s==255); + + return length; +} + +/*! LZ4_decompress_generic() : + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, + * in order to remove useless branches during compilation optimization. + */ +LZ4_FORCE_INLINE int +LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + if ((src == NULL) || (outputSize < 0)) { return -1; } + + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; + + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(64 KB)); + + + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; + + const BYTE* match; + size_t offset; + unsigned token; + size_t length; + + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + + /* Special cases */ + assert(lowPrefix <= op); + if (unlikely(outputSize==0)) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if (unlikely(srcSize==0)) { return -1; } + + /* LZ4_FAST_DEC_LOOP: + * designed for modern OoO performance cpus, + * where copying reliably 32-bytes is preferable to an unpredictable branch. + * note : fast loop may show a regression for some client arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "skip fast decode loop"); + goto safe_decode; + } + + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, cpy); + ip += length; op = cpy; + } else { + cpy = op+length; + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } + /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ + LZ4_memcpy(op, ip, 16); + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + assert(match <= op); /* overflow check */ + + /* get matchlength */ + length = token & ML_MASK; + + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + length += MINMATCH; + if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: skip LZ4_wildCopy32 when true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + goto _output_error; /* end-of-block condition violated */ + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ + } + safe_decode: +#endif + + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (length != RUN_MASK) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((ip < shortiend) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, 16); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + + /* copy literals */ + cpy = op+length; +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((ip+length != iend) || (cpy > oend)) { + DEBUGLOG(6, "should have been last run of literals") + DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); + DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend); + goto _output_error; + } + } + LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ + ip += length; + op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } + + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; + } + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ + } + + /* end of decoding */ + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } +} + + +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + decode_full_block, noDict, + (BYTE*)dest, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + partial_decode, + noDict, (BYTE*)dst, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + DEBUGLOG(5, "LZ4_decompress_fast"); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, + size_t prefixSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +/*===== streaming decompression functions =====*/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal)); + return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ + FREEMEM(LZ4_stream); + return 0; +} +#endif + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +LZ4_FORCE_O2 +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; + } else { + /* The buffer wraps around, or they're switching to another buffer. */ + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +LZ4_FORCE_O2 int +LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* const lz4sd = + (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse); + int result; + + DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize); + assert(originalSize >= 0); + + if (lz4sd->prefixSize == 0) { + DEBUGLOG(5, "first invocation : no prefix nor extDict"); + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + DEBUGLOG(5, "continue using existing prefix"); + result = LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + lz4sd->prefixSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; + } else { + DEBUGLOG(5, "prefix becomes extDict"); + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + (size_t)dictSize, NULL, 0); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); +} + + +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} + +/* +These decompression functions are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); } + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_create (char* inputBuffer) +{ + (void)inputBuffer; + return LZ4_createStream(); +} +#endif + +char* LZ4_slideInputBuffer (void* state) +{ + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; +} + +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/native/lz4/lz4.h b/native/lz4/lz4.h new file mode 100644 index 0000000..491c608 --- /dev/null +++ b/native/lz4/lz4.h @@ -0,0 +1,842 @@ +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (C) 2011-2020, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + +/* --- Dependency --- */ +#include /* size_t */ + + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. + + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. +*/ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API LZ4LIB_VISIBILITY +#endif + +/*! LZ4_FREESTANDING : + * When this macro is set to 1, it enables "freestanding mode" that is + * suitable for typical freestanding environment which doesn't support + * standard C library. + * + * - LZ4_FREESTANDING is a compile-time switch. + * - It requires the following macros to be defined: + * LZ4_memcpy, LZ4_memmove, LZ4_memset. + * - It only enables LZ4/HC functions which don't use heap. + * All LZ4F_* functions are not supported. + * - See tests/freestanding.c to check its basic setup. + */ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) +# define LZ4_HEAPMODE 0 +# define LZ4HC_HEAPMODE 0 +# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +# if !defined(LZ4_memcpy) +# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." +# endif +# if !defined(LZ4_memset) +# error "LZ4_FREESTANDING requires macro 'LZ4_memset'." +# endif +# if !defined(LZ4_memmove) +# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." +# endif +#elif ! defined(LZ4_FREESTANDING) +# define LZ4_FREESTANDING 0 +#endif + + +/*------ Version ------*/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ + +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */ + + +/*-************************************ +* Tuning parameter +**************************************/ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + +/*! + * LZ4_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; ) + * Increasing memory usage improves compression ratio, at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT +#endif + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); + +/*! LZ4_decompress_safe() : + * compressedSize : is the exact complete size of the compressed block. + * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is incorrect (too large or negative) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). +*/ +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ +LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'targetDestSize'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + * or 0 if compression fails. + * + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+): + * the produced compressed content could, in specific circumstances, + * require to be decompressed into a destination buffer larger + * by at least 1 byte than the content to decompress. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); + + +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/** + Note about RC_INVOKED + + - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). + https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros + + - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) + and reports warning "RC4011: identifier truncated". + + - To eliminate the warning, we surround long preprocessor symbol with + "#if !defined(RC_INVOKED) ... #endif" block that means + "skip this block when rc.exe is trying to read it". +*/ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 accept any input as dictionary, + * results are generally better when using Zstandard's Dictionary Builder. + * Loading a size of 0 is allowed, and is the same as reset. + * @return : loaded dictionary size, in bytes (necessarily <= 64 KB) + */ +LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_compress_fast_continue() : + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * @return : size of compressed block + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. + */ +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_saveDict() : + * If last 64KB data cannot be guaranteed to remain available at its current memory location, + * save it into a safer place (char* safeBuffer). + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. + */ +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ + +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_setStreamDecode() : + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * @return : 1 if OK, 0 if error + */ +LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + +/*! LZ4_decompress_*_continue() : + * These decoding functions allow decompression of consecutive blocks in "streaming" mode. + * A block is an unsplittable entity, it must be presented entirely to a decompression function. + * Decompression functions only accepts one block at a time. + * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. +*/ +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); + + +/*! LZ4_decompress_*_usingDict() : + * These decoding functions work the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ + + +/*^************************************* + * !!!!!! STATIC LINKING ONLY !!!!!! + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +#define LZ4LIB_STATIC_API LZ4LIB_API +#else +#define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows + * efficient use of a static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. + */ +LZ4LIB_STATIC_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); + + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly constrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif + +/*! LZ4_stream_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_stream_t object. +**/ + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + LZ4_u32 dictSize; + /* Implicit padding to ensure structure is aligned */ +}; + +#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */ +union LZ4_stream_u { + char minStateSize[LZ4_STREAM_MINSIZE]; + LZ4_stream_t_internal internal_donotuse; +}; /* previously typedef'd to LZ4_stream_t */ + + +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead +**/ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); + + +/*! LZ4_streamDecode_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. +**/ +typedef struct { + const LZ4_byte* externalDict; + const LZ4_byte* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#define LZ4_STREAMDECODE_MINSIZE 32 +union LZ4_streamDecode_u { + char minStateSize[LZ4_STREAMDECODE_MINSIZE]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + + +/*-************************************ +* Obsolete Functions +**************************************/ + +/*! Deprecation warnings + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ +#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# else +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ +# endif +#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ + +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + + +#endif /* LZ4_H_98237428734687 */ + + +#if defined (__cplusplus) +} +#endif diff --git a/native/lz4/lz4hc.c b/native/lz4/lz4hc.c new file mode 100644 index 0000000..b21ad6b --- /dev/null +++ b/native/lz4/lz4hc.c @@ -0,0 +1,1631 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2020, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ + + +/* ************************************* +* Tuning Parameter +***************************************/ + +/*! HEAPMODE : + * Select how default compression function will allocate workplace memory, + * in stack (0:fastest), or in heap (1:requires malloc()). + * Since workplace is rather large, heap mode is recommended. +**/ +#ifndef LZ4HC_HEAPMODE +# define LZ4HC_HEAPMODE 1 +#endif + + +/*=== Dependency ===*/ +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" + + +/*=== Common definitions ===*/ +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif +#if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#endif + +#define LZ4_COMMONDEFS_ONLY +#ifndef LZ4_SRC_INCLUDED +#include "lz4.c" /* LZ4_count, constants, mem */ +#endif + + +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; + + +/*=== Constants ===*/ +#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) +#define LZ4_OPT_NUM (1<<12) + + +/*=== Macros ===*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor + +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } + + +/************************************** +* HC Compression +**************************************/ +static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) +{ + MEM_INIT(hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); +} + +static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) +{ + size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart); + size_t newStartingOffset = bufferSize + hc4->dictLimit; + assert(newStartingOffset >= bufferSize); /* check overflow */ + if (newStartingOffset > 1 GB) { + LZ4HC_clearTables(hc4); + newStartingOffset = 0; + } + newStartingOffset += 64 KB; + hc4->nextToUpdate = (U32)newStartingOffset; + hc4->prefixStart = start; + hc4->end = start; + hc4->dictStart = start; + hc4->dictLimit = (U32)newStartingOffset; + hc4->lowLimit = (U32)newStartingOffset; +} + + +/* Update chains up to ip (excluded) */ +LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) +{ + U16* const chainTable = hc4->chainTable; + U32* const hashTable = hc4->hashTable; + const BYTE* const prefixPtr = hc4->prefixStart; + U32 const prefixIdx = hc4->dictLimit; + U32 const target = (U32)(ip - prefixPtr) + prefixIdx; + U32 idx = hc4->nextToUpdate; + assert(ip >= prefixPtr); + assert(target >= prefixIdx); + + while (idx < target) { + U32 const h = LZ4HC_hashPtr(prefixPtr+idx-prefixIdx); + size_t delta = idx - hashTable[h]; + if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX; + DELTANEXTU16(chainTable, idx) = (U16)delta; + hashTable[h] = idx; + idx++; + } + + hc4->nextToUpdate = target; +} + +/** LZ4HC_countBack() : + * @return : negative value, nb of common bytes before ip/match */ +LZ4_FORCE_INLINE +int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, + const BYTE* const iMin, const BYTE* const mMin) +{ + int back = 0; + int const min = (int)MAX(iMin - ip, mMin - match); + assert(min <= 0); + assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31)); + assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31)); + while ( (back > min) + && (ip[back-1] == match[back-1]) ) + back--; + return back; +} + +#if defined(_MSC_VER) +# define LZ4HC_rotl32(x,r) _rotl(x,r) +#else +# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) +{ + size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; + if (bitsToRotate == 0) return pattern; + return LZ4HC_rotl32(pattern, (int)bitsToRotate); +} + +/* LZ4HC_countPattern() : + * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ +static unsigned +LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) +{ + const BYTE* const iStart = ip; + reg_t const pattern = (sizeof(pattern)==8) ? + (reg_t)pattern32 + (((reg_t)pattern32) << (sizeof(pattern)*4)) : pattern32; + + while (likely(ip < iEnd-(sizeof(pattern)-1))) { + reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; + if (!diff) { ip+=sizeof(pattern); continue; } + ip += LZ4_NbCommonBytes(diff); + return (unsigned)(ip - iStart); + } + + if (LZ4_isLittleEndian()) { + reg_t patternByte = pattern; + while ((ip>= 8; + } + } else { /* big endian */ + U32 bitOffset = (sizeof(pattern)*8) - 8; + while (ip < iEnd) { + BYTE const byte = (BYTE)(pattern >> bitOffset); + if (*ip != byte) break; + ip ++; bitOffset -= 8; + } } + + return (unsigned)(ip - iStart); +} + +/* LZ4HC_reverseCountPattern() : + * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) + * read using natural platform endianness */ +static unsigned +LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +{ + const BYTE* const iStart = ip; + + while (likely(ip >= iLow+4)) { + if (LZ4_read32(ip-4) != pattern) break; + ip -= 4; + } + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianness */ + while (likely(ip>iLow)) { + if (ip[-1] != *bytePtr) break; + ip--; bytePtr--; + } } + return (unsigned)(iStart - ip); +} + +/* LZ4HC_protectDictEnd() : + * Checks if the match is in the last 3 bytes of the dictionary, so reading the + * 4 byte MINMATCH would overflow. + * @returns true if the match index is okay. + */ +static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +{ + return ((U32)((dictLimit - 1) - matchIndex) >= 3); +} + +typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; +typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; + +LZ4_FORCE_INLINE int +LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_CCtx_internal* const hc4, + const BYTE* const ip, + const BYTE* const iLowLimit, const BYTE* const iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts, + const int patternAnalysis, const int chainSwap, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx; + const BYTE* const prefixPtr = hc4->prefixStart; + const U32 prefixIdx = hc4->dictLimit; + const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + const int withinStartDistance = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex); + const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const BYTE* const dictStart = hc4->dictStart; + const U32 dictIdx = hc4->lowLimit; + const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx; + int const lookBackLength = (int)(ip-iLowLimit); + int nbAttempts = maxNbAttempts; + U32 matchChainPos = 0; + U32 const pattern = LZ4_read32(ip); + U32 matchIndex; + repeat_state_e repeat = rep_untested; + size_t srcPatternLength = 0; + + DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); + /* First Match */ + LZ4HC_Insert(hc4, ip); + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)", + matchIndex, lowestMatchIndex); + + while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) { + int matchLength=0; + nbAttempts--; + assert(matchIndex < ipIndex); + if (favorDecSpeed && (ipIndex - matchIndex < 8)) { + /* do nothing */ + } else if (matchIndex >= prefixIdx) { /* within current Prefix */ + const BYTE* const matchPtr = prefixPtr + matchIndex - prefixIdx; + assert(matchPtr < ip); + assert(longest >= 1); + if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { + if (LZ4_read32(matchPtr) == pattern) { + int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, prefixPtr) : 0; + matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + *matchpos = matchPtr + back; + *startpos = ip + back; + } } } + } else { /* lowestMatchIndex <= matchIndex < dictLimit */ + const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx); + assert(matchIndex >= dictIdx); + if ( likely(matchIndex <= prefixIdx - 4) + && (LZ4_read32(matchPtr) == pattern) ) { + int back = 0; + const BYTE* vLimit = ip + (prefixIdx - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, prefixPtr, iHighLimit); + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + *matchpos = prefixPtr - prefixIdx + matchIndex + back; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip + back; + } } } + + if (chainSwap && matchLength==longest) { /* better match => select a better chain */ + assert(lookBackLength==0); /* search forward only */ + if (matchIndex + (U32)longest <= ipIndex) { + int const kTrigger = 4; + U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; + int pos; + for (pos = 0; pos < end; pos += step) { + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); + if (candidateDist > distanceToNextMatch) { + distanceToNextMatch = candidateDist; + matchChainPos = (U32)pos; + accel = 1 << kTrigger; + } } + if (distanceToNextMatch > 1) { + if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ + matchIndex -= distanceToNextMatch; + continue; + } } } + + { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex); + if (patternAnalysis && distNextMatch==1 && matchChainPos==0) { + U32 const matchCandidateIdx = matchIndex-1; + /* may be a repeated pattern */ + if (repeat == rep_untested) { + if ( ((pattern & 0xFFFF) == (pattern >> 16)) + & ((pattern & 0xFF) == (pattern >> 24)) ) { + repeat = rep_confirmed; + srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); + } else { + repeat = rep_not; + } } + if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) + && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < prefixIdx; + const BYTE* const matchPtr = (extDict ? dictStart - dictIdx : prefixPtr - prefixIdx) + matchCandidateIdx; + if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ + const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; + size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forwardPatternLength == iLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); + forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern); + } + { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr; + size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); + size_t currentSegmentLength; + if (!extDict + && matchPtr - backLength == prefixPtr + && dictIdx < prefixIdx) { + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); + backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern); + } + /* Limit backLength not go further than lowestMatchIndex */ + backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); + assert(matchCandidateIdx - backLength >= lowestMatchIndex); + currentSegmentLength = backLength + forwardPatternLength; + /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) + matchIndex = newMatchIndex; + else { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; + } + } else { + U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) { + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; + } else { + matchIndex = newMatchIndex; + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(prefixPtr - prefixIdx + matchIndex != ip); + if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + *matchpos = prefixPtr - prefixIdx + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip; + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } } } + continue; + } } + } } /* PA optimization */ + + /* follow current chain */ + matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos); + + } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ + + if ( dict == usingDictCtxHc + && nbAttempts > 0 + && ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) { + size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + assert(dictEndOffset <= 1 GB); + matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex; + + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0; + mlt -= back; + if (mlt > longest) { + longest = mlt; + *matchpos = prefixPtr - prefixIdx + matchIndex + back; + *startpos = ip + back; + } } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + dictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } } } + + return longest; +} + +LZ4_FORCE_INLINE int +LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts, + const int patternAnalysis, + const dictCtx_directive dict) +{ + const BYTE* uselessPtr = ip; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); +} + +/* LZ4HC_encodeSequence() : + * @return : 0 if ok, + * 1 if buffer issue detected */ +LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** _ip, + BYTE** _op, + const BYTE** _anchor, + int matchLength, + const BYTE* const match, + limitedOutput_directive limit, + BYTE* oend) +{ +#define ip (*_ip) +#define op (*_op) +#define anchor (*_anchor) + + size_t length; + BYTE* const token = op++; + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) + static const BYTE* start = NULL; + static U32 totalCost = 0; + U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); + U32 const ll = (U32)(ip - anchor); + U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; + U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; + U32 const cost = 1 + llAdd + ll + 2 + mlAdd; + if (start==NULL) start = anchor; /* only works for single segment */ + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ + DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5u, cost:%4u + %5u", + pos, + (U32)(ip - anchor), matchLength, (U32)(ip-match), + cost, totalCost); + totalCost += cost; +#endif + + /* Encode Literal length */ + length = (size_t)(ip - anchor); + LZ4_STATIC_ASSERT(notLimited == 0); + /* Check output limit */ + if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) { + DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)", + (int)length, (int)(oend - op)); + return 1; + } + if (length >= RUN_MASK) { + size_t len = length - RUN_MASK; + *token = (RUN_MASK << ML_BITS); + for(; len >= 255 ; len -= 255) *op++ = 255; + *op++ = (BYTE)len; + } else { + *token = (BYTE)(length << ML_BITS); + } + + /* Copy Literals */ + LZ4_wildCopy8(op, anchor, op + length); + op += length; + + /* Encode Offset */ + assert( (ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */ + LZ4_writeLE16(op, (U16)(ip - match)); op += 2; + + /* Encode MatchLength */ + assert(matchLength >= MINMATCH); + length = (size_t)matchLength - MINMATCH; + if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) { + DEBUGLOG(6, "Not enough room to write match length"); + return 1; /* Check output limit */ + } + if (length >= ML_MASK) { + *token += ML_MASK; + length -= ML_MASK; + for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; } + if (length >= 255) { length -= 255; *op++ = 255; } + *op++ = (BYTE)length; + } else { + *token += (BYTE)(length); + } + + /* Prepare next loop */ + ip += matchLength; + anchor = ip; + + return 0; +} +#undef ip +#undef op +#undef anchor + +LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int* srcSizePtr, + int const maxOutputSize, + int maxNbAttempts, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + const int inputSize = *srcSizePtr; + const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* optr = (BYTE*) dest; + BYTE* op = (BYTE*) dest; + BYTE* oend = op + maxOutputSize; + + int ml0, ml, ml2, ml3; + const BYTE* start0; + const BYTE* ref0; + const BYTE* ref = NULL; + const BYTE* start2 = NULL; + const BYTE* ref2 = NULL; + const BYTE* start3 = NULL; + const BYTE* ref3 = NULL; + + /* init */ + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* Main Loop */ + while (ip <= mflimit) { + ml = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); + if (ml encode ML1 */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + continue; + } + + if (start0 < ip) { /* first match was skipped at least once */ + if (start2 < ip + ml0) { /* squeezing ML1 between ML0(original ML1) and ML2 */ + ip = start0; ref = ref0; ml = ml0; /* restore initial ML1 */ + } } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) { /* First Match too small : removed */ + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + /* At this stage, we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) */ + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ + + if (start2 + ml2 <= mflimit) { + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, + maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); + } else { + ml3 = ml2; + } + + if (ml3 == ml2) { /* No better match => encode ML1 and ML2 */ + /* ip & ref are known; Now for ml */ + if (start2 < ip+ml) ml = (int)(start2 - ip); + /* Now, encode 2 sequences */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + ip = start2; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) { + ml = ml2; + ref = ref2; + goto _dest_overflow; + } + continue; + } + + if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+ml) { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < MINMATCH) { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; + * let's write the first one ML1. + * ip & ref are known; Now decide ml. + */ + if (start2 < ip+ml) { + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } else { + ml = (int)(start2 - ip); + } + } + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; + + /* ML2 becomes ML1 */ + ip = start2; ref = ref2; ml = ml2; + + /* ML3 becomes ML2 */ + start2 = start3; ref2 = ref3; ml2 = ml3; + + /* let's find a new ML3 */ + goto _Search3; + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) (((char*)op)-dest); + +_dest_overflow: + if (limit == fillOutput) { + /* Assumption : ip, anchor, ml and ref must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing"); + op = optr; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(ml >= 0); + if ((size_t)ml > maxMlSize) ml = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ml >= MFLIMIT) { + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, notLimited, oend); + } } + goto _last_literals; + } + /* compression failed */ + return 0; +} + + +static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, + const char* const source, char* dst, + int* srcSizePtr, int dstCapacity, + int const nbSearches, size_t sufficient_len, + const limitedOutput_directive limit, int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed); + + +LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + typedef enum { lz4hc, lz4opt } lz4hc_strat_e; + typedef struct { + lz4hc_strat_e strat; + int nbSearches; + U32 targetLength; + } cParams_t; + static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { + { lz4hc, 2, 16 }, /* 0, unused */ + { lz4hc, 2, 16 }, /* 1, unused */ + { lz4hc, 2, 16 }, /* 2, unused */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + }; + + DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", + ctx, src, *srcSizePtr, limit); + + if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + + ctx->end += *srcSizePtr; + if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */ + cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); + { cParams_t const cParam = clTable[cLevel]; + HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; + int result; + + if (cParam.strat == lz4hc) { + result = LZ4HC_compress_hashChain(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, limit, dict); + } else { + assert(cParam.strat == lz4opt); + result = LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } + if (result <= 0) ctx->dirty = 1; + return result; + } +} + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); + +static int +LZ4HC_compress_generic_noDictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + assert(ctx->dictCtx == NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); +} + +static int +LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + const size_t position = (size_t)(ctx->end - ctx->prefixStart) + (ctx->dictLimit - ctx->lowLimit); + assert(ctx->dictCtx != NULL); + if (position >= 64 KB) { + ctx->dictCtx = NULL; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else if (position == 0 && *srcSizePtr > 4 KB) { + LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4HC_setExternalDict(ctx, (const BYTE *)src); + ctx->compressionLevel = (short)cLevel; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); + } +} + +static int +LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + if (ctx->dictCtx == NULL) { + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } +} + + +int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } + +static size_t LZ4_streamHC_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_streamHC_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_streamHC_t); +#else + return 1; /* effectively disabled */ +#endif +} + +/* state is presumed correctly initialized, + * in which case its size and alignment have already been validate */ +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; + if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) return 0; + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); + LZ4HC_init_internal (ctx, (const BYTE*)src); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); + else + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited); +} + +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); +} + +int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + int cSize; +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (statePtr==NULL) return 0; +#else + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; +#endif + cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + FREEMEM(statePtr); +#endif + return cSize; +} + +/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */ +int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source); + LZ4_setCompressionLevel(ctx, cLevel); + return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput); +} + + + +/************************************** +* Streaming Functions +**************************************/ +/* allocation */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamHC_t* LZ4_createStreamHC(void) +{ + LZ4_streamHC_t* const state = + (LZ4_streamHC_t*)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t)); + if (state == NULL) return NULL; + LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT); + return state; +} + +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) +{ + DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + FREEMEM(LZ4_streamHCPtr); + return 0; +} +#endif + + +LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) +{ + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; + DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size); + /* check conditions */ + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_streamHC_t)) return NULL; + if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL; + /* init */ + { LZ4HC_CCtx_internal* const hcstate = &(LZ4_streamHCPtr->internal_donotuse); + MEM_INIT(hcstate, 0, sizeof(*hcstate)); } + LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + return LZ4_streamHCPtr; +} + +/* just a stub */ +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (LZ4_streamHCPtr->internal_donotuse.dirty) { + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + } else { + /* preserve end - prefixStart : can trigger clearTable's threshold */ + if (LZ4_streamHCPtr->internal_donotuse.end != NULL) { + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.prefixStart; + } else { + assert(LZ4_streamHCPtr->internal_donotuse.prefixStart == NULL); + } + LZ4_streamHCPtr->internal_donotuse.prefixStart = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + } + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; + if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; +} + +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) +{ + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); +} + +/* LZ4_loadDictHC() : + * LZ4_streamHCPtr is presumed properly initialized */ +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* dictionary, int dictSize) +{ + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d)", LZ4_streamHCPtr, dictionary, dictSize); + assert(LZ4_streamHCPtr != NULL); + if (dictSize > 64 KB) { + dictionary += (size_t)dictSize - 64 KB; + dictSize = 64 KB; + } + /* need a full initialization, there are bad side-effects when using resetFast() */ + { int const cLevel = ctxPtr->compressionLevel; + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); + } + LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); + return dictSize; +} + +void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; +} + +/* compression */ + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) +{ + DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); + if (ctxPtr->end >= ctxPtr->prefixStart + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart); + ctxPtr->prefixStart = newBlock; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; +} + +static int +LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) +{ + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(5, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", + LZ4_streamHCPtr, src, *srcSizePtr, limit); + assert(ctxPtr != NULL); + /* auto-init if forgotten */ + if (ctxPtr->prefixStart == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart); + if (dictSize > 64 KB) dictSize = 64 KB; + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)src != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; + const BYTE* const dictBegin = ctxPtr->dictStart; + const BYTE* const dictEnd = ctxPtr->dictStart + (ctxPtr->dictLimit - ctxPtr->lowLimit); + if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit += (U32)(sourceEnd - ctxPtr->dictStart); + ctxPtr->dictStart += (U32)(sourceEnd - ctxPtr->dictStart); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) { + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + } } } + + return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); +} + +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) +{ + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); + else + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited); +} + +int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) +{ + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput); +} + + + +/* LZ4_saveDictHC : + * save history content + * into a user-provided buffer + * which is then used to continue compression + */ +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; + int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart); + DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + assert(prefixSize >= 0); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + LZ4_memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit; + streamPtr->end = (const BYTE*)safeBuffer + dictSize; + streamPtr->prefixStart = streamPtr->end - dictSize; + streamPtr->dictLimit = endIndex - (U32)dictSize; + streamPtr->lowLimit = endIndex - (U32)dictSize; + streamPtr->dictStart = streamPtr->prefixStart; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) + streamPtr->nextToUpdate = streamPtr->dictLimit; + } + return dictSize; +} + + +/*************************************************** +* Deprecated Functions +***************************************************/ + +/* These functions currently generate deprecation warnings */ + +/* Wrappers for deprecated compression functions */ +int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } + + +/* Deprecated streaming functions */ +int LZ4_sizeofStreamStateHC(void) { return sizeof(LZ4_streamHC_t); } + +/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) + * @return : 0 on success, !=0 if error */ +int LZ4_resetStreamStateHC(void* state, char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4)); + if (hc4 == NULL) return 1; /* init failed */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_createHC (const char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_createStreamHC(); + if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} +#endif + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited); +} + +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); +} + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4_streamHC_t* const ctx = (LZ4_streamHC_t*)LZ4HC_Data; + const BYTE* bufferStart = ctx->internal_donotuse.prefixStart - ctx->internal_donotuse.dictLimit + ctx->internal_donotuse.lowLimit; + LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel); + /* avoid const char * -> char * conversion warning :( */ + return (char*)(uptrval)bufferStart; +} + + +/* ================================================ + * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX]) + * ===============================================*/ +typedef struct { + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + +/* price in bytes */ +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) +{ + int price = litlen; + assert(litlen >= 0); + if (litlen >= (int)RUN_MASK) + price += 1 + ((litlen-(int)RUN_MASK) / 255); + return price; +} + + +/* requires mlen >= MINMATCH */ +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) +{ + int price = 1 + 2 ; /* token + 16-bit offset */ + assert(litlen >= 0); + assert(mlen >= MINMATCH); + + price += LZ4HC_literalsPrice(litlen); + + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255); + + return price; +} + + +typedef struct { + int off; + int len; +} LZ4HC_match_t; + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + LZ4HC_match_t match = { 0 , 0 }; + const BYTE* matchPtr = NULL; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + int matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + if (matchLength <= minLen) return match; + if (favorDecSpeed) { + if ((matchLength>18) & (matchLength<=36)) matchLength=18; /* favor shortcut */ + } + match.len = matchLength; + match.off = (int)(ip-matchPtr); + return match; +} + + +static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + size_t sufficient_len, + const limitedOutput_directive limit, + int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + int retval = 0; +#define TRAILING_LITERALS 3 +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); +#else + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ +#endif + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; + int ovml = MINMATCH; /* overflow - last sequence */ + const BYTE* ovref = NULL; + + /* init */ +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + if (opt == NULL) goto _return_label; +#endif + DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; + + /* Main Loop */ + while (ip <= mflimit) { + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + if (firstMatch.len==0) { ip++; continue; } + + if ((size_t)firstMatch.len > sufficient_len) { + /* good enough solution : immediate encoding */ + int const firstML = firstMatch.len; + const BYTE* const matchPos = ip - firstMatch.off; + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) { /* updates ip, op and anchor */ + ovml = firstML; + ovref = matchPos; + goto _dest_overflow; + } + continue; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int mlen = MINMATCH; + int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + assert(matchML < LZ4_OPT_NUM); + for ( ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + + /* check further positions */ + for (cur = 1; cur < last_match_pos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + + if (curPtr > mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + } + + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); + if (!newMatch.len) continue; + + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; + goto encode; + } + + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } + + assert((U32)favorDecSpeed <= 1); + if (pos > last_match_pos+TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + +encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) { /* updates ip, op and anchor */ + ovml = ml; + ovref = ip - offset; + goto _dest_overflow; + } } } + } /* while (ip <= mflimit) */ + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) { /* Check output limit */ + retval = 0; + goto _return_label; + } + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + retval = (int) ((char*)op-dst); + goto _return_label; + +_dest_overflow: +if (limit == fillOutput) { + /* Assumption : ip, anchor, ovml and ovref must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing (only %i bytes remaining)", (int)(oend-1-opSaved)); + op = opSaved; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(ovml >= 0); + if ((size_t)ovml > maxMlSize) ovml = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) { + DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml); + DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovref, notLimited, oend); + DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor); + } } + goto _last_literals; +} +_return_label: +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + FREEMEM(opt); +#endif + return retval; +} diff --git a/native/lz4/lz4hc.h b/native/lz4/lz4hc.h new file mode 100644 index 0000000..e937acf --- /dev/null +++ b/native/lz4/lz4hc.h @@ -0,0 +1,413 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Header File + Copyright (C) 2011-2020, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#ifndef LZ4_HC_H_19834876238432 +#define LZ4_HC_H_19834876238432 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* --- Dependency --- */ +/* note : lz4hc requires lz4.h/lz4.c for compilation */ +#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ + + +/* --- Useful constants --- */ +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 10 +#define LZ4HC_CLEVEL_MAX 12 + + +/*-************************************ + * Block Compression + **************************************/ +/*! LZ4_compress_HC() : + * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm. + * `dst` must be already allocated. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + */ +LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); + + +/* Note : + * Decompression functions are provided within "lz4.h" (BSD license) + */ + + +/*! LZ4_compress_HC_extStateHC() : + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * `state` size is provided by LZ4_sizeofStateHC(). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). + */ +LZ4LIB_API int LZ4_sizeofStateHC(void); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + +/*! LZ4_compress_HC_destSize() : v1.9.0+ + * Will compress as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src` + */ +LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); + + +/*-************************************ + * Streaming Compression + * Bufferless synchronous API + **************************************/ + typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ + +/*! LZ4_createStreamHC() and LZ4_freeStreamHC() : + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * A same state can be used multiple times consecutively, + * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks. + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); +LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); + +/* + These functions compress data in successive blocks of any size, + using previous blocks as dictionary, to improve compression ratio. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller than 64 KB. + Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). + + Before starting compression, state must be allocated and properly initialized. + LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. + + Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) + or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). + LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once, + which is automatically the case when state is created using LZ4_createStreamHC(). + + After reset, a first "fictional block" can be designated as initial dictionary, + using LZ4_loadDictHC() (Optional). + + Invoke LZ4_compress_HC_continue() to compress each successive block. + The number of blocks is unlimited. + Previous input blocks, including initial dictionary when present, + must remain accessible and unmodified during compression. + + It's allowed to update compression level anytime between blocks, + using LZ4_setCompressionLevel() (experimental). + + 'dst' buffer should be sized to handle worst case scenarios + (see LZ4_compressBound(), it ensures compression success). + In case of failure, the API does not guarantee recovery, + so the state _must_ be reset. + To ensure compression success + whenever `dst` buffer size cannot be made >= LZ4_compressBound(), + consider using LZ4_compress_HC_continue_destSize(). + + Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, + it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) + + After completing a streaming compression, + it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state, + just by resetting it, using LZ4_resetStreamHC_fast(). +*/ + +LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */ +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, + const char* src, char* dst, + int srcSize, int maxDstSize); + +/*! LZ4_compress_HC_continue_destSize() : v1.9.0+ + * Similar to LZ4_compress_HC_continue(), + * but will read as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided into 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. + * Note that this function may not consume the entire input. + */ +LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); + +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ + +/*-****************************************************************** + * PRIVATE DEFINITIONS : + * Do not use these definitions directly. + * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. + * Declare an `LZ4_streamHC_t` directly, rather than any type below. + * Even then, only do so in the context of static linking, as definitions may change between versions. + ********************************************************************/ + +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<= LZ4HC_CLEVEL_OPT_MIN. + */ +LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( + LZ4_streamHC_t* LZ4_streamHCPtr, int favor); + +/*! LZ4_resetStreamHC_fast() : v1.9.0+ + * When an LZ4_streamHC_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStreamHC()). + * + * LZ4_streamHCs are guaranteed to be in a valid state when: + * - returned from LZ4_createStreamHC() + * - reset by LZ4_resetStreamHC() + * - memset(stream, 0, sizeof(LZ4_streamHC_t)) + * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that + * returned success + * + * Note: + * A stream that was last used in a compression call that returned an error + * may be passed to this function. However, it will be fully reset, which will + * clear any existing history and settings from the context. + */ +LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + +/*! LZ4_compress_HC_extStateHC_fastReset() : + * A variant of LZ4_compress_HC_extStateHC(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStreamHC_fast() for a definition of + * "correctly initialized"). From a high level, the difference is that this + * function initializes the provided state with a call to + * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a + * call to LZ4_resetStreamHC(). + */ +LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( + void* state, + const char* src, char* dst, + int srcSize, int dstCapacity, + int compressionLevel); + +/*! LZ4_attach_HC_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a + * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDictHC() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * A dictionary should only be attached to a stream without any history (i.e., + * a stream that has just been reset). + * + * The dictionary will remain attached to the working stream only for the + * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the + * dictionary context association from the working stream. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the lifetime of the stream session. + */ +LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary( + LZ4_streamHC_t *working_stream, + const LZ4_streamHC_t *dictionary_stream); + +#if defined (__cplusplus) +} +#endif + +#endif /* LZ4_HC_SLO_098092834 */ +#endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/native/mkpfs_lz4.c b/native/mkpfs_lz4.c new file mode 100644 index 0000000..97a7c6c --- /dev/null +++ b/native/mkpfs_lz4.c @@ -0,0 +1,71 @@ +/* + * mkpfs_lz4: raw LZ4 block API over lz4 1.9.4 for AMPR asset packs. + * + * The encoder calls lz4 exactly like python-lz4 4.4.5 lz4/block/_block.c (store_size=False), which + * ampr_emu's ampr_pack.py uses: fast mode resets a stream and calls LZ4_compress_fast_continue, and + * HC mode resets an HC stream and calls LZ4_compress_HC_continue. LZ4_compress_fast emits different + * (larger) blocks, so it must not be used for parity. + * All lz4 symbols stay hidden; only the mkpfs_lz4_* entry points are exported. + */ + +#include +#include + +#include "lz4.h" +#include "lz4hc.h" + +#if defined(_WIN32) +#define MKPFS_API __declspec(dllexport) +#else +#define MKPFS_API __attribute__((visibility("default"))) +#endif + +typedef struct mkpfs_lz4_encoder { + LZ4_stream_t fast; + LZ4_streamHC_t hc; +} mkpfs_lz4_encoder; + +MKPFS_API const char *mkpfs_lz4_version(void) { + return LZ4_versionString(); +} + +/* Returns 0 when source_len exceeds LZ4_MAX_INPUT_SIZE. */ +MKPFS_API int32_t mkpfs_lz4_bound(int32_t source_len) { + return LZ4_compressBound(source_len); +} + +/* Holds both stream states (about 280 KB), so one encoder per worker avoids large stack frames. */ +MKPFS_API void *mkpfs_lz4_encoder_create(void) { + return calloc(1, sizeof(mkpfs_lz4_encoder)); +} + +MKPFS_API void mkpfs_lz4_encoder_free(void *handle) { + free(handle); +} + +/* Compress one independent raw block. Returns the block size, or 0 when dst is too small. */ +MKPFS_API int32_t mkpfs_lz4_compress_fast(void *handle, const uint8_t *src, int32_t src_len, + uint8_t *dst, int32_t dst_cap, int32_t acceleration) { + mkpfs_lz4_encoder *encoder = (mkpfs_lz4_encoder *)handle; + if (encoder == NULL) return 0; + LZ4_resetStream(&encoder->fast); + return LZ4_compress_fast_continue(&encoder->fast, (const char *)src, (char *)dst, src_len, dst_cap, + acceleration); +} + +/* Compress one independent raw block at HC level 1..12. Returns the block size, or 0 when dst is too small. */ +MKPFS_API int32_t mkpfs_lz4_compress_hc(void *handle, const uint8_t *src, int32_t src_len, + uint8_t *dst, int32_t dst_cap, int32_t level) { + mkpfs_lz4_encoder *encoder = (mkpfs_lz4_encoder *)handle; + if (encoder == NULL) return 0; + LZ4_resetStreamHC(&encoder->hc, level); + return LZ4_compress_HC_continue(&encoder->hc, (const char *)src, (char *)dst, src_len, dst_cap); +} + +/* + * Decode one raw block (LZ4_decompress_safe, as the ampr_emu runtime does). + * Returns the decoded size, or a negative value for malformed input or a too small dst. + */ +MKPFS_API int32_t mkpfs_lz4_decompress_safe(const uint8_t *src, int32_t src_len, uint8_t *dst, int32_t dst_cap) { + return LZ4_decompress_safe((const char *)src, (char *)dst, src_len, dst_cap); +} diff --git a/native/zlib/ORIGIN.md b/native/zlib/ORIGIN.md index 0a14fef..f9467af 100644 --- a/native/zlib/ORIGIN.md +++ b/native/zlib/ORIGIN.md @@ -1,9 +1,12 @@ -zlib 1.3.1 +# zlib 1.3.1 -Source: https://github.com/madler/zlib/tree/v1.3.1 -Commit: 51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf +| | | +|---|---| +| **Source** | https://github.com/madler/zlib/tree/v1.3.1 | +| **Commit** | `51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf` | +| **Files** | The C and header files PS5 Game Compressor's `third_party/zlib` needs, plus `LICENSE` | +| **Changes** | One trailing blank line removed from `trees.h` | -Copied from PS5 Game Compressor `third_party/zlib`, which took the C and header files it needs, -plus LICENSE, from that release. One trailing blank line was removed from `trees.h`. -No other changes. Game Compressor and Python 3.11 (`zlib.ZLIB_RUNTIME_VERSION`) use the same -release, so PFSC blocks compressed here are byte-identical to theirs. +Copied from PS5 Game Compressor `third_party/zlib`, which took them from that release. Game Compressor and +Python 3.11 (`zlib.ZLIB_RUNTIME_VERSION`) use the same release, so PFSC blocks compressed here are +byte-identical to theirs. diff --git a/src/MkPFS.Build/AMPRPack/AMPRBlockCompressor.cs b/src/MkPFS.Build/AMPRPack/AMPRBlockCompressor.cs new file mode 100644 index 0000000..5875ad2 --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRBlockCompressor.cs @@ -0,0 +1,66 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using MkPFS.Core.AMPR; +using MkPFS.Core.Compression; +using MkPFS.Core.Util; + +namespace MkPFS.Build.AMPRPack; + +/// One encoded block (Python CompressedBlock). +/// Decoded bytes. +/// Bytes written to the volume. +/// ChunkCodecRaw or ChunkCodecLZ4. +/// CRC-32 of . +/// Deduplication key of the decoded bytes. +public sealed record AMPRCompressedBlock(byte[] Raw, byte[] Stored, int Codec, uint RawCrc, byte[] Fingerprint); + +/// Per-block codec decision (Python _compress_block). +public static class AMPRBlockCompressor +{ + /// + /// Compress one block with the rule's LZ4 mode and keep it only if it saves enough. Random/mixed blocks whose + /// I/O page count does not drop must also meet the stricter I/O-neutral thresholds. + /// + /// Encoder owned by the calling worker. + /// Block bytes (not empty). + /// Effective rule. + /// I/O page size of the target group. + /// The block as it will be stored. + public static AMPRCompressedBlock Compress(LZ4Encoder encoder, byte[] raw, AMPRRule rule, long ioPageSize) + { + uint rawCrc = Crc32.Update(0, raw); + // Python uses crc32 + length + BLAKE2b-128; any collision-resistant digest gives the same dedup decisions. + byte[] fingerprint = new byte[12 + 32]; + BinaryPrimitives.WriteUInt32LittleEndian(fingerprint, rawCrc); + BinaryPrimitives.WriteInt64LittleEndian(fingerprint.AsSpan(4), raw.Length); + SHA256.HashData(raw, fingerprint.AsSpan(12)); + if (rule.Action == "store") + { + return new AMPRCompressedBlock(raw, raw, AMPRPackFormat.ChunkCodecRaw, rawCrc, fingerprint); + } + + byte[] compressed = rule.Mode == "hc" + ? encoder.CompressHC(raw, (int)Math.Clamp(rule.Level, 1, 12)) + : encoder.CompressFast(raw, (int)Math.Clamp(rule.Acceleration, 1, int.MaxValue)); + long saved = raw.Length - compressed.Length; + double ratio = raw.Length > 0 ? (double)saved / raw.Length : 0.0; + long requiredBytes = rule.MinSavingsBytes; + double requiredRatio = rule.MinSavingsRatio; + // An isolated cold read of a random/mixed chunk costs whole I/O pages; require a stronger gain when + // compression does not reduce the page count. + if (rule.ResolvedLayout() != "streaming") + { + long rawPages = (raw.Length + ioPageSize - 1) / ioPageSize; + long compressedPages = (compressed.Length + ioPageSize - 1) / ioPageSize; + if (compressedPages >= rawPages) + { + requiredBytes = Math.Max(requiredBytes, rule.IONeutralMinSavingsBytes); + requiredRatio = Math.Max(requiredRatio, rule.IONeutralMinSavingsRatio); + } + } + + return saved < requiredBytes || ratio < requiredRatio + ? new AMPRCompressedBlock(raw, raw, AMPRPackFormat.ChunkCodecRaw, rawCrc, fingerprint) + : new AMPRCompressedBlock(raw, compressed, AMPRPackFormat.ChunkCodecLZ4, rawCrc, fingerprint); + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRBuildTypes.cs b/src/MkPFS.Build/AMPRPack/AMPRBuildTypes.cs new file mode 100644 index 0000000..8ffb7af --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRBuildTypes.cs @@ -0,0 +1,123 @@ +using MkPFS.Core.AMPR; + +namespace MkPFS.Build.AMPRPack; + +/// Counters reported by a pack build (Python BuildStats; the CLI prints them as JSON). +public sealed class AMPRBuildStats +{ + /// AMPRIDX3 files. + public long FilesTotal { get; set; } + + /// Files stored in pack volumes. + public long FilesPacked { get; set; } + + /// Files left on the filesystem. + public long FilesLoose { get; set; } + + /// Relative paths that must stay in /app0. + public List LoosePaths { get; } = []; + + /// Files left loose by sampling. + public long FilesAutoLoose { get; set; } + + /// Bytes of auto-loose files. + public long AutoLooseLogicalBytes { get; set; } + + /// Bytes read while sampling. + public long AutoLooseSampledBytes { get; set; } + + /// Chunk records. + public long Chunks { get; set; } + + /// LZ4 chunk records. + public long ChunksLZ4 { get; set; } + + /// RAW chunk records. + public long ChunksRaw { get; set; } + + /// Chunk records that reuse an earlier block. + public long ChunksShared { get; set; } + + /// Decoded bytes of packed files. + public long LogicalBytes { get; set; } + + /// Bytes written to volumes (excluding padding). + public long StoredBytes { get; set; } + + /// Alignment padding in volumes. + public long PaddingBytes { get; set; } + + /// I/O pages covered by volume payloads. + public long IOPagesTouched { get; set; } + + /// Chunks placed page-contained or page-aligned. + public long IOPageSafeChunks { get; set; } + + /// Chunks in dense streaming extents. + public long DenseStreamingChunks { get; set; } +} + +/// Progress snapshot (Python BuildProgress). +/// reading-index, planning, packing, finalizing, publishing or complete. +/// Files done in this phase. +/// Files in this phase. +/// Bytes packed. +/// Bytes to pack. +/// File being processed, if any. +public sealed record AMPRBuildProgress( + string Phase, long FilesDone, long FilesTotal, long LogicalBytesDone, long LogicalBytesTotal, string CurrentPath = ""); + +/// A planned file (Python SelectedFile). +/// AMPRIDX3 row + 1. +/// AMPRIDX3 row. +/// Source path under the root. +/// Path relative to /app0. +/// Effective rule (loose for files that stay on the filesystem). +/// Size on disk (index size for loose files). +/// Modification time on disk (index time for loose files). +/// Pack lane within the rule's group. +public sealed record AMPRSelectedFile( + int FileId, AMPRIndexEntry IndexEntry, string Source, string Relative, AMPRRule Rule, long Size, long MTime, int Lane); + +/// Result of a pack build. +/// Published manifest. +/// Counters. +/// Auto-loose and missing-source notes. +/// Pack volumes named by the manifest. +public sealed record AMPRBuildResult(string IndexPath, AMPRBuildStats Stats, IReadOnlyList Warnings, int Volumes = 0) +{ + /// ampr_emu AMPR_EMU_PACK_MAX_FILES: manifest file records the runtime loads. + public const long RuntimeMaxFiles = 2_000_000; + + /// ampr_emu AMPR_EMU_PACK_MAX_CHUNKS: chunk records the runtime loads. + public const long RuntimeMaxChunks = 16_000_000; + + /// ampr_emu AMPR_EMU_PACK_MAX_PACKS: pack volumes the runtime loads. + public const int RuntimeMaxVolumes = 1024; + + /// + /// Limits of the default ampr_emu build that this set exceeds; the runtime then rejects the whole manifest + /// (apr.pack.index.invalid). ampr_pack.py only checks the format limits, so it publishes such sets. + /// + /// One message per exceeded limit. + public IReadOnlyList RuntimeLimitWarnings() + { + List messages = []; + if (Stats.FilesTotal > RuntimeMaxFiles) + { + messages.Add($"{Stats.FilesTotal} files exceed the AMPR Emu limit of {RuntimeMaxFiles}; the console will reject this pack set"); + } + + if (Stats.Chunks > RuntimeMaxChunks) + { + messages.Add($"{Stats.Chunks} chunks exceed the AMPR Emu limit of {RuntimeMaxChunks}; use larger blocks or leave some files loose"); + } + + if (Volumes > RuntimeMaxVolumes) + { + messages.Add($"{Volumes} pack volumes exceed the AMPR Emu limit of {RuntimeMaxVolumes}; raise max_pack_size or lower pack_count"); + } + + return messages; + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRCommandLog.cs b/src/MkPFS.Build/AMPRPack/AMPRCommandLog.cs new file mode 100644 index 0000000..4970081 --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRCommandLog.cs @@ -0,0 +1,392 @@ +using System.Buffers.Binary; +using System.Text; + +namespace MkPFS.Build.AMPRPack; + +/// One AMPRIDX3 row as the trace tools see it (Python IndexEntry). +/// Row number plus one. +/// Indexed path (/app0/...); invalid UTF-8 is replaced. +/// File size. +/// Modification time. +public sealed record AMPRTraceIndexEntry(int FileId, string Path, long Size, long MTime); + +/// APR command fields the profiler uses (Python decode_command keeps more). +/// Command name, as ampr_emu's parser names it. +/// Command length in 32-bit words. +/// File id (AprReadFile). +/// File offset (reads that carry one). +/// Read length (reads). +public readonly record struct AMPRCommand(string Name, int Dwords, long FileId = 0, long FileOffset = 0, long Length = 0); + +/// +/// AMPR Emu's APR submit journal (ampr_commands.bin, AMPRCMD1), written by the debug emulator build: +/// record header, command decoder and the lenient AMPRIDX3 reader of ampr_emu tools/parse_ampr_command_log.py. +/// +public static class AMPRCommandLog +{ + /// Record magic. + public static ReadOnlySpan Magic => "AMPRCMD1"u8; + + /// Supported record version. + public const int Version = 1; + + /// Size of the fixed record header (<8sHHIQQQQQQIIIIIIII). + public const int HeaderSize = 96; + + /// Submit domain of APR commands (2 is AMM). + public const int DomainApr = 1; + + /// Fixed part of one journal record. + /// Record version. + /// Header size, including any extension. + /// Header plus payload. + /// Submit sequence, from 1. + /// Monotonic time of the submit. + /// FNV-1a of the payload (). + /// Payload size. + /// Commands in the payload, or 0 when unknown. + /// Submit priority (the gather/scatter state key). + /// 1 APR, 2 AMM. + public readonly record struct Header( + int Version, int HeaderBytes, long RecordBytes, ulong Sequence, ulong MonotonicNs, ulong PayloadHash, long PayloadBytes, + long CommandCount, long Priority, long Domain); + + /// Parse a record header; the magic is checked by the caller. + /// At least bytes. + /// Header fields. + public static Header ReadHeader(ReadOnlySpan raw) => new( + BinaryPrimitives.ReadUInt16LittleEndian(raw[8..]), + BinaryPrimitives.ReadUInt16LittleEndian(raw[10..]), + BinaryPrimitives.ReadUInt32LittleEndian(raw[12..]), + BinaryPrimitives.ReadUInt64LittleEndian(raw[16..]), + BinaryPrimitives.ReadUInt64LittleEndian(raw[32..]), + BinaryPrimitives.ReadUInt64LittleEndian(raw[56..]), + BinaryPrimitives.ReadUInt32LittleEndian(raw[64..]), + BinaryPrimitives.ReadUInt32LittleEndian(raw[72..]), + BinaryPrimitives.ReadUInt32LittleEndian(raw[76..]), + BinaryPrimitives.ReadUInt32LittleEndian(raw[80..])); + + /// FNV-1a 64 of a payload, never 0 (Python fnv1a64). + /// Payload. + /// Hash. + public static ulong Fnv1a64(ReadOnlySpan data) + { + ulong hash = 1469598103934665603UL; + foreach (byte b in data) + { + hash = (hash ^ b) * 1099511628211UL; + } + + return hash == 0 ? 1 : hash; + } + + /// + /// Read an AMPRIDX3 file the way the trace tools do (Python load_index): bounds are checked, but paths that + /// are not UTF-8 are decoded with replacement characters and the hash table is not used. + /// + /// Index file. + /// Rows by file id. + /// The index is malformed. + public static Dictionary LoadIndex(string path) + { + byte[] data = File.ReadAllBytes(path); + if (data.Length < 48) + { + throw new InvalidDataException($"index is too small: {path}"); + } + + ReadOnlySpan span = data; + if (!span[..8].SequenceEqual("AMPRIDX3"u8)) + { + throw new InvalidDataException($"unsupported index magic {PythonBytesRepr(span[..8])}; expected AMPRIDX3"); + } + + uint version = BinaryPrimitives.ReadUInt32LittleEndian(span[8..]); + uint entrySize = BinaryPrimitives.ReadUInt32LittleEndian(span[12..]); + ulong entryCount = BinaryPrimitives.ReadUInt64LittleEndian(span[16..]); + ulong pathBytes = BinaryPrimitives.ReadUInt64LittleEndian(span[24..]); + ulong hashOffset = BinaryPrimitives.ReadUInt64LittleEndian(span[32..]); + uint hashSlotSize = BinaryPrimitives.ReadUInt32LittleEndian(span[40..]); + uint hashSlotCount = BinaryPrimitives.ReadUInt32LittleEndian(span[44..]); + if (version != 3) + { + throw new InvalidDataException($"unsupported AMPRIDX3 version {version}; expected 3"); + } + + if (entrySize != 24) + { + throw new InvalidDataException($"unsupported AMPRIDX3 entry size {entrySize}; expected 24"); + } + + UInt128 pathStart = 48 + ((UInt128)entryCount * entrySize); + UInt128 pathEnd = pathStart + pathBytes; + if (pathEnd > (UInt128)data.Length) + { + throw new InvalidDataException("AMPRIDX3 path blob exceeds file size"); + } + + if (hashOffset < pathEnd || hashOffset > (ulong)data.Length) + { + throw new InvalidDataException("AMPRIDX3 hash offset is invalid"); + } + + if (hashSlotSize != 16) + { + throw new InvalidDataException($"unsupported AMPRIDX3 hash slot size {hashSlotSize}"); + } + + if (hashOffset + ((UInt128)hashSlotCount * hashSlotSize) > (UInt128)data.Length) + { + throw new InvalidDataException("AMPRIDX3 hash table exceeds file size"); + } + + ReadOnlySpan blob = span[(int)pathStart..(int)pathEnd]; + Dictionary entries = new((int)entryCount); + for (int index = 0; index < (int)entryCount; index++) + { + ReadOnlySpan record = span.Slice(48 + (index * 24), 24); + uint pathOff = BinaryPrimitives.ReadUInt32LittleEndian(record); + uint pathLen = BinaryPrimitives.ReadUInt32LittleEndian(record[4..]); + ulong size = BinaryPrimitives.ReadUInt64LittleEndian(record[8..]); + long mtime = BinaryPrimitives.ReadInt64LittleEndian(record[16..]); + if ((ulong)pathOff + pathLen > (ulong)blob.Length) + { + throw new InvalidDataException($"AMPRIDX3 entry {index} path is out of bounds"); + } + + string decoded = Encoding.UTF8.GetString(blob.Slice((int)pathOff, (int)pathLen)); + entries[index + 1] = new AMPRTraceIndexEntry(index + 1, decoded, (long)size, mtime); + } + + return entries; + } + + /// + /// Decode the command at (Python decode_command): its name and length for every + /// command, and the file fields of APR reads. + /// + /// Submit payload. + /// Byte offset of the command. + /// The command. + /// The command is truncated or has an invalid length (Python + /// ValueError, same message). + public static AMPRCommand Decode(ReadOnlySpan data, int offset) + { + if ((offset & 3) != 0 || offset + 4 > data.Length) + { + throw new InvalidDataException("unaligned or truncated command"); + } + + uint w0 = U32(data, offset); + uint opcode8 = w0 & 0xFF; + uint opcode12 = w0 & 0xFFF; + int dwords; + if (opcode8 == 1) + { + dwords = (int)((w0 >> 8) & 0xF) + 1; + return dwords is >= 2 and <= 4 && Need(data, offset, dwords) + ? new AMPRCommand("WaitOnAddress", dwords) + : throw new InvalidDataException("invalid WaitOnAddress length"); + } + + if (opcode8 == 2) + { + dwords = (int)((w0 >> 8) & 0xF) + 1; + return dwords == 4 || !(dwords is >= 1 and <= 5 && Need(data, offset, dwords)) + ? throw new InvalidDataException("invalid WaitOnCounter length") + : new AMPRCommand("WaitOnCounter", dwords); + } + + if (opcode8 is 5 or 117) + { + dwords = (int)((w0 >> 8) & 3) + 1; + if (!(dwords is >= 2 and <= 4 && Need(data, offset, dwords))) + { + throw new InvalidDataException("invalid WriteAddress length"); + } + + string name = (U32(data, offset + 4) & 7) switch + { + 1 => "WriteAddressFromTimeCounter", + 2 => "WriteAddressFromCounter", + 3 => "WriteAddressFromCounterPair", + _ => "WriteAddress", + }; + return new AMPRCommand(name, dwords); + } + + if (opcode8 is 6 or 118) + { + dwords = (int)((w0 >> 8) & 0xF) + 1; + return dwords is >= 1 and <= 3 && Need(data, offset, dwords) + ? new AMPRCommand("WriteCounter", dwords) + : throw new InvalidDataException("invalid WriteCounter length"); + } + + if (opcode12 is 1032 or 1144) + { + return Need(data, offset, 5) ? new AMPRCommand("WriteKernelEventQueue", 5) : throw new InvalidDataException("truncated WriteKernelEventQueue"); + } + + if ((w0 & 0xFFFF000F) == 0x5452000F) + { + uint markerType = (w0 >> 12) & 0xF; + int payloadDwords = (int)((w0 >> 8) & 0xF); + dwords = payloadDwords + 1; + if (!Need(data, offset, dwords)) + { + throw new InvalidDataException("truncated NOP/marker packet"); + } + + if (w0 == 0x5452300F) + { + return new AMPRCommand("MarkerPop", 1); + } + + if (markerType is 5 or 6 && payloadDwords < 1) + { + throw new InvalidDataException("marker with color has no color dword"); + } + + return new AMPRCommand(markerType switch + { + 1 or 5 => "MarkerSet", + 2 or 6 => "MarkerPush", + 4 => "MarkerContinuation", + _ => "Nop", + }, dwords); + } + + if (opcode8 == 40) + { + dwords = (int)((w0 >> 8) & 7) + 1; + if (!(dwords is >= 5 and <= 6 && Need(data, offset, dwords))) + { + throw new InvalidDataException("invalid AprReadFile length"); + } + + uint w4 = U32(data, offset + 16); + long fileOffset = ((w0 >> 12) & 0x3FFFF) | (w4 & 0xFFFC0000); + if (dwords >= 6) + { + fileOffset |= (long)(U32(data, offset + 20) & 0xFF) << 32; + } + + return new AMPRCommand("AprReadFile", dwords, U32(data, offset + 8) & 0x7FFFFFFF, fileOffset, U32(data, offset + 4) + 1L); + } + + if (opcode8 == 41) + { + dwords = (int)((w0 >> 8) & 3) + 1; + if (!(dwords is >= 2 and <= 3 && Need(data, offset, dwords))) + { + throw new InvalidDataException("invalid AprReadGather length"); + } + + long fileOffset = (w0 >> 12) & 0x3FFFF; + if (dwords >= 3) + { + fileOffset |= (long)(U32(data, offset + 8) & 0x3FFFFF) << 18; + } + + return new AMPRCommand("AprReadGather", dwords, FileOffset: fileOffset, Length: U32(data, offset + 4) + 1L); + } + + if (opcode12 == 0x22A) + { + return Need(data, offset, 3) + ? new AMPRCommand("AprReadScatter", 3, Length: U32(data, offset + 4) + 1L) + : throw new InvalidDataException("truncated AprReadScatter"); + } + + if (opcode8 == 43) + { + dwords = (int)((w0 >> 8) & 7) + 1; + if (!(dwords is >= 4 and <= 5 && Need(data, offset, dwords))) + { + throw new InvalidDataException("invalid AprReadGatherScatter length"); + } + + uint w3 = U32(data, offset + 12); + long fileOffset = ((w0 >> 12) & 0x3FFFF) | (w3 & 0xFFFC0000); + if (dwords >= 5) + { + fileOffset |= (long)(U32(data, offset + 16) & 0xFF) << 32; + } + + return new AMPRCommand("AprReadGatherScatter", dwords, FileOffset: fileOffset, Length: U32(data, offset + 4) + 1L); + } + + if (w0 == 47) + { + return new AMPRCommand("AprResetGatherScatterState", 1); + } + + if (opcode12 == 557) + { + return Need(data, offset, 3) ? new AMPRCommand("AprMapBegin", 3) : throw new InvalidDataException("truncated AprMapBegin"); + } + + if (opcode12 == 813) + { + return Need(data, offset, 4) ? new AMPRCommand("AprMapDirectBegin", 4) : throw new InvalidDataException("truncated AprMapDirectBegin"); + } + + if (w0 == 46) + { + return new AMPRCommand("AprMapEnd", 1); + } + + // AMM commands (Python decode_amm); a truncated one falls through to Unknown. + dwords = opcode12 switch + { + 0x221 => 3, + 0x321 => 4, + 0x325 => 4, + 0x425 => 5, + 0x222 or 0x228 => 3, + 0x323 or 0x324 or 0x327 => 4, + 0x423 or 0x424 => 5, + 0x326 => 4, + 0x426 => 5, + _ => 0, + }; + if (dwords > 0 && Need(data, offset, dwords)) + { + return new AMPRCommand("Amm", dwords); + } + + int guessed = (int)((w0 >> 8) & 0xF) + 1; + return new AMPRCommand("Unknown", Need(data, offset, guessed) ? guessed : 1); + } + + private static uint U32(ReadOnlySpan data, int offset) => BinaryPrimitives.ReadUInt32LittleEndian(data[offset..]); + + private static bool Need(ReadOnlySpan data, int offset, int dwords) => + dwords > 0 && offset >= 0 && offset + ((long)dwords * 4) <= data.Length; + + // Python repr(bytes): b'...' with printable ASCII kept, \t \n \r and other bytes as \xNN. + private static string PythonBytesRepr(ReadOnlySpan value) + { + bool hasSingle = value.Contains((byte)'\''); + bool hasDouble = value.Contains((byte)'"'); + char quote = hasSingle && !hasDouble ? '"' : '\''; + StringBuilder builder = new("b"); + builder.Append(quote); + foreach (byte b in value) + { + _ = b switch + { + (byte)'\\' => builder.Append(@"\\"), + (byte)'\t' => builder.Append(@"\t"), + (byte)'\n' => builder.Append(@"\n"), + (byte)'\r' => builder.Append(@"\r"), + _ when b == quote => builder.Append('\\').Append(quote), + >= 0x20 and < 0x7F => builder.Append((char)b), + _ => builder.Append(@"\x").Append(b.ToString("x2", System.Globalization.CultureInfo.InvariantCulture)), + }; + } + + return builder.Append(quote).ToString(); + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRGameBuilder.cs b/src/MkPFS.Build/AMPRPack/AMPRGameBuilder.cs new file mode 100644 index 0000000..f66d798 --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRGameBuilder.cs @@ -0,0 +1,425 @@ +using MkPFS.Build.Exfat; +using MkPFS.Build.PFS; +using MkPFS.Core.AMPR; +using MkPFS.Core.Diagnostics; +using MkPFS.Core.Util; + +namespace MkPFS.Build.AMPRPack; + +/// Settings for . +public sealed record AMPRGameOptions +{ + /// Pack rules. + public required AMPRPackConfig Config { get; init; } + + /// Folder whose files (AMPR Emu and other fakelib libraries) go into the output's library folder. + public string? LibsDir { get; init; } + + /// Decode every chunk and compare packed and loose files with the source after the build. + public bool Verify { get; init; } = true; + + /// exFAT image path (or existing folder for <titleId>.exfat) to build from the output. + public string? ExfatImage { get; init; } + + /// Free space to leave inside the exFAT image (0: tight). A debug run of AMPR Emu writes its log there. + public long ExfatFreeBytes { get; init; } +} + +/// Result of . +/// Playable /app0 folder. +/// fakelib or fakelib2. +/// Pack build result. +/// Files copied from the source. +/// Bytes of the source files. +/// Bytes of the output folder. +/// exFAT image written, if any. +public sealed record AMPRGameResult( + string OutputDir, string LibraryDir, AMPRBuildResult Pack, int LooseFiles, long SourceBytes, long OutputBytes, string? ExfatImage); + +/// +/// Turns an unpacked game folder into a folder that runs from AMPR packs as is (not in ampr_pack.py): the library +/// folder with the selected AMPR Emu, an ampr_emu.index of that final tree, the packs, and every file that +/// stays loose. The source is only read. +/// +public static class AMPRGameBuilder +{ + /// The emulator library that serves the packs. + public const string EmulatorLibrary = "libSceAmpr.sprx"; + + // Every AMPR Emu build that reads asset packs (0.4.2.1 and newer) embeds its manifest magic; older ones do not. + private static readonly byte[] PackSupportMarker = "AMPRPAK4"u8.ToArray(); + + /// Build the playable folder, then optionally an exFAT image of it. + /// Unpacked game folder (/app0). + /// New or empty output folder. + /// Settings. + /// Step messages. + /// Pack progress. + /// exFAT progress. + /// Summary. + /// The inputs cannot give a playable folder, or a check failed. + public static AMPRGameResult Build( + string sourceRoot, + string outputDir, + AMPRGameOptions options, + IMkPFSLog log, + Action? packProgress = null, + IProgressSink? exfatProgress = null) + { + sourceRoot = Path.GetFullPath(sourceRoot); + outputDir = Path.GetFullPath(outputDir); + string? image = options.ExfatImage is null ? null : Path.GetFullPath(options.ExfatImage); + if (image is not null && Directory.Exists(image)) + { + image = Path.Combine(image, GameParams.DefaultImageBasename(sourceRoot) + ".exfat"); + } + + CheckInputs(sourceRoot, outputDir, options.Config, image, log); + string libraryDir = Directory.Exists(Path.Combine(sourceRoot, "fakelib2")) ? "fakelib2" : "fakelib"; + if (options.LibsDir is { } libsDir && !Directory.Exists(libsDir)) + { + throw new BuildException($"library folder not found: {libsDir}"); + } + + // The selected library replaces the game's own, so check whichever one will end up in the output. + string? selected = options.LibsDir is { } dir && File.Exists(Path.Combine(dir, EmulatorLibrary)) ? Path.Combine(dir, EmulatorLibrary) : null; + CheckEmulator(selected ?? Path.Combine(sourceRoot, libraryDir, EmulatorLibrary), libraryDir); + + bool created = !Directory.Exists(outputDir); + Directory.CreateDirectory(outputDir); + AMPRGameResult result; + try + { + result = BuildInto(sourceRoot, outputDir, libraryDir, options, log, packProgress); + } + catch + { + // The folder was new or empty, so nothing of the user's is lost; a retry then starts clean. + RemoveOutput(outputDir, created); + throw; + } + + if (image is null) + { + return result; + } + + // A failed image leaves the finished folder in place, and only the partial image is removed. + log.Info(options.ExfatFreeBytes > 0 + ? $"Building the exFAT image (64 KiB clusters, {Sizes.HumanReadable(options.ExfatFreeBytes)} free)" + : "Building the exFAT image (64 KiB clusters)"); + string written; + try + { + written = ExfatImageWriter.Write(outputDir, image, null, exfatProgress, options.ExfatFreeBytes); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidOperationException) + { + File.Delete(image); + throw new BuildException($"the game folder is ready, but the exFAT image failed: {ex.Message}"); + } + + log.Info($"exFAT image: {Sizes.HumanReadable(new FileInfo(written).Length)}: {written}"); + log.Info("Mount it read-only (ShadowMountPlus mount_read_only=1, the default) so AMPR Emu never rebuilds the index"); + return result with { ExfatImage = written }; + } + + private static AMPRGameResult BuildInto( + string sourceRoot, + string outputDir, + string libraryDir, + AMPRGameOptions options, + IMkPFSLog log, + Action? packProgress) + { + // 1. Libraries: the source's library folder, then the selected files over it. The index needs their final sizes. + log.Info($"[1/5] Libraries: {libraryDir}/"); + if (libraryDir == "fakelib2") + { + log.Info(" The game has fakelib2/, which ShadowMountPlus mounts instead of fakelib/; using it"); + } + + CopyTree(Path.Combine(sourceRoot, libraryDir), Path.Combine(outputDir, libraryDir)); + if (options.LibsDir is { } libs) + { + AddLibraries(libs, Path.Combine(outputDir, libraryDir), log); + } + + // 2. Index of the final tree, before packing: the pack manifest addresses files by their row in it. + log.Info($"[2/5] Writing {AmprIndex.IndexName}"); + string index = Path.Combine(outputDir, AmprIndex.IndexName); + int rows = AmprIndex.BuildWithOverlay(sourceRoot, outputDir, libraryDir, index); + if (rows == 0) + { + throw new BuildException($"the game folder has no files: {sourceRoot}"); + } + + log.Info($" {rows} files indexed"); + + // 3. Packs, read from the source. + log.Info("[3/5] Packing"); + AMPRBuildResult pack = AMPRPackBuilder.Build(sourceRoot, index, outputDir, options.Config, progress: packProgress); + foreach (string warning in pack.Warnings) + { + log.Warning("warning: " + warning); + } + + foreach (string warning in pack.RuntimeLimitWarnings()) + { + log.Warning("warning: " + warning); + } + + log.Info($" {pack.Stats.FilesPacked} files packed into {pack.Volumes} volume(s), {pack.Stats.FilesLoose} stay loose"); + if (pack.Stats.FilesPacked == 0) + { + log.Warning("warning: no files were packed; the output is a plain copy of the game"); + } + + // 4. Loose files: known only now, because the packer leaves large incompressible files loose by sampling them. + log.Info("[4/5] Copying loose files"); + int loose = CopyLooseFiles(sourceRoot, outputDir, libraryDir, pack.Stats.LoosePaths); + CopyDirectories(sourceRoot, outputDir); + log.Info($" {loose} files copied"); + + // 5. Checks. + if (options.Verify) + { + log.Info("[5/5] Verifying"); + Verify(sourceRoot, outputDir, pack.IndexPath, log); + } + else + { + log.Info("[5/5] Verification skipped"); + } + + long sourceBytes = TreeBytes(sourceRoot); + long outputBytes = TreeBytes(outputDir); + log.Info($"Game folder: {Sizes.HumanReadable(sourceBytes)} -> {Sizes.HumanReadable(outputBytes)}: {outputDir}"); + + return new AMPRGameResult(outputDir, libraryDir, pack, loose, sourceBytes, outputBytes, null); + } + + /// when is an AMPR Emu build that reads asset packs. + /// libSceAmpr.sprx. + /// Whether the build embeds the AMPRPAK4 manifest magic. + public static bool SupportsPacks(string library) => + File.ReadAllBytes(library).AsSpan().IndexOf(PackSupportMarker) >= 0; + + private static void CheckInputs(string source, string output, AMPRPackConfig config, string? image, IMkPFSLog log) + { + if (!Directory.Exists(source)) + { + throw new BuildException($"game folder not found: {source}"); + } + + if (IsSameOrInside(output, source) || IsSameOrInside(source, output)) + { + throw new BuildException("the game folder and the output folder must not contain each other"); + } + + if (File.Exists(output) || (Directory.Exists(output) && Directory.EnumerateFileSystemEntries(output).Any())) + { + throw new BuildException($"the output folder must be new or empty: {output}"); + } + + if (File.Exists(Path.Combine(source, config.IndexName))) + { + throw new BuildException($"the game folder already has {config.IndexName}; start from the unpacked game"); + } + + if (!File.Exists(Path.Combine(source, "sce_sys", "param.json")) && !File.Exists(Path.Combine(source, "sce_sys", "param.sfo"))) + { + log.Warning("warning: the game folder has no sce_sys/param.json; is it the game's /app0 folder?"); + } + + if (image is not null) + { + if (IsSameOrInside(image, output) || IsSameOrInside(image, source)) + { + throw new BuildException("the exFAT image must be outside the game and output folders"); + } + + if (File.Exists(image)) + { + throw new BuildException($"the exFAT image already exists: {image}"); + } + } + } + + private static void RemoveOutput(string outputDir, bool created) + { + try + { + if (created) + { + Directory.Delete(outputDir, recursive: true); + return; + } + + foreach (FileSystemInfo entry in new DirectoryInfo(outputDir).EnumerateFileSystemInfos()) + { + if (entry is DirectoryInfo sub) + { + sub.Delete(recursive: true); + } + else + { + entry.Delete(); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Keep the build error; the leftover folder is reported by the next run's empty-folder check. + } + } + + private static void AddLibraries(string libsDir, string target, IMkPFSLog log) + { + Directory.CreateDirectory(target); + foreach (FileInfo file in new DirectoryInfo(libsDir).EnumerateFiles() + .Where(f => !f.Name.StartsWith('.') && !NameRules.IsIgnoredName(f.Name)) + .OrderBy(f => f.Name, StringComparer.Ordinal)) + { + string destination = Path.Combine(target, file.Name); + if (File.Exists(destination) && SameContent(file.FullName, destination)) + { + log.Info($" {file.Name}: already up to date"); + continue; + } + + string state = File.Exists(destination) ? "replaced" : "added"; + file.CopyTo(destination, overwrite: true); + File.SetLastWriteTimeUtc(destination, file.LastWriteTimeUtc); + log.Info($" {file.Name}: {state}"); + } + } + + private static void CheckEmulator(string library, string libraryDir) + { + if (!File.Exists(library)) + { + throw new BuildException($"no {libraryDir}/{EmulatorLibrary}: select the AMPR Emu library folder"); + } + + if (!SupportsPacks(library)) + { + throw new BuildException( + $"{libraryDir}/{EmulatorLibrary} cannot read asset packs; use AMPR Emu 0.4.2.1 or newer from https://github.com/drakmor/ampr_emu/releases"); + } + } + + private static int CopyLooseFiles(string source, string output, string libraryDir, IReadOnlyList loosePaths) + { + string libraryKey = AmprIndex.KeyFor(libraryDir + "/"); + int copied = 0; + foreach (string relative in loosePaths) + { + if (AmprIndex.KeyFor(relative).StartsWith(libraryKey, StringComparison.Ordinal)) + { + continue; // already final in the output + } + + string from = AMPRAssetPath.SafeOutputPath(source, relative); + string to = AMPRAssetPath.SafeOutputPath(output, relative); + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + File.Copy(from, to, overwrite: false); + File.SetLastWriteTimeUtc(to, File.GetLastWriteTimeUtc(from)); + copied++; + } + + return copied; + } + + // Empty folders too, so the output has the source's full directory tree. + private static void CopyDirectories(string source, string output) + { + foreach (string dir in Directory.EnumerateDirectories(source, "*", new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = 0 })) + { + string relative = Path.GetRelativePath(source, dir); + if (!relative.Split(Path.DirectorySeparatorChar).Any(NameRules.IsIgnoredName)) + { + Directory.CreateDirectory(Path.Combine(output, relative)); + } + } + } + + private static void CopyTree(string from, string to) + { + if (!Directory.Exists(from)) + { + return; + } + + Directory.CreateDirectory(to); + foreach (string dir in Directory.EnumerateDirectories(from, "*", SearchOption.AllDirectories)) + { + Directory.CreateDirectory(Path.Combine(to, Path.GetRelativePath(from, dir))); + } + + foreach (string file in Directory.EnumerateFiles(from, "*", SearchOption.AllDirectories)) + { + string destination = Path.Combine(to, Path.GetRelativePath(from, file)); + File.Copy(file, destination); + File.SetLastWriteTimeUtc(destination, File.GetLastWriteTimeUtc(file)); + } + } + + // Every packed file decodes to the source bytes, and every loose index row is in the output with its size. + private static void Verify(string source, string output, string manifest, IMkPFSLog log) + { + try + { + AMPRVerifyResult blocks = AMPRPackTools.Verify(manifest); + AMPRSourceCompareResult compare = AMPRPackTools.VerifyAgainstRoot(manifest, source); + log.Info($" Packs: {blocks.Files} files, {blocks.PhysicalChunks} chunks decoded; {compare.Files} files match the source"); + } + catch (AMPRPackException ex) + { + throw new BuildException($"pack verification failed: {ex.Message}"); + } + + List missing = []; + int loose = 0; + foreach (AMPRListEntry entry in AMPRPackTools.List(manifest)) + { + if (entry.Packed) + { + continue; + } + + FileInfo file = new(AMPRAssetPath.SafeOutputPath(output, AMPRAssetPath.Relative(entry.Path))); + if (!file.Exists || (ulong)file.Length != entry.LogicalSize) + { + missing.Add(AMPRAssetPath.Relative(entry.Path)); + } + + loose++; + } + + if (missing.Count > 0) + { + throw new BuildException($"{missing.Count} loose file(s) are missing or differ in the output, first: {missing[0]}"); + } + + log.Info($" Loose files: {loose} present with their indexed sizes"); + } + + private static bool IsSameOrInside(string path, string folder) + { + StringComparison comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + string a = Path.TrimEndingDirectorySeparator(path); + string b = Path.TrimEndingDirectorySeparator(folder); + return a.Equals(b, comparison) || a.StartsWith(b + Path.DirectorySeparatorChar, comparison); + } + + private static bool SameContent(string a, string b) + { + FileInfo x = new(a); + FileInfo y = new(b); + return x.Length == y.Length && File.ReadAllBytes(a).AsSpan().SequenceEqual(File.ReadAllBytes(b)); + } + + private static long TreeBytes(string root) => + Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories).Sum(f => new FileInfo(f).Length); +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRPackBuilder.cs b/src/MkPFS.Build/AMPRPack/AMPRPackBuilder.cs new file mode 100644 index 0000000..ce9213d --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRPackBuilder.cs @@ -0,0 +1,471 @@ +using MkPFS.Core.AMPR; +using MkPFS.Core.Compression; +using MkPFS.Core.Util; + +namespace MkPFS.Build.AMPRPack; + +/// +/// Builds an AMPR asset-pack set (manifest, volumes, CRC sidecar, optional runtime settings) from an /app0 +/// tree and its AMPRIDX3 index. Port of ampr_pack build_packs and _publish_transaction; output is +/// byte-identical to the oracle for the same inputs. +/// +public static class AMPRPackBuilder +{ + /// Build and publish a pack set. + /// /app0 source directory. + /// AMPRIDX3 index of that tree. + /// Output directory. + /// Configuration. + /// CLI include globs. + /// CLI exclude globs. + /// Leave missing sources loose. + /// Progress callback. + /// Manifest path, counters and warnings. + /// The build failed; temporary files are removed. + public static AMPRBuildResult Build( + string root, + string amprIndex, + string outputDir, + AMPRPackConfig config, + IReadOnlyList? includePatterns = null, + IReadOnlyList? excludePatterns = null, + bool allowMissing = false, + Action? progress = null) + { + root = Path.GetFullPath(root); + outputDir = Path.GetFullPath(outputDir); + Directory.CreateDirectory(outputDir); + progress?.Invoke(new AMPRBuildProgress("reading-index", 0, 0, 0, 0, amprIndex)); + List entries = AMPRIndexReader.Read(amprIndex); + AMPRBuildStats stats = new() { FilesTotal = entries.Count }; + (List selected, List warnings) = AMPRPackPlanner.Plan( + entries, root, config, stats, includePatterns ?? [], excludePatterns ?? [], allowMissing, progress); + + List packedItems = [.. selected.Where(item => item.Rule.Action != "loose")]; + PackingProgress packing = new(progress, packedItems.Count, packedItems.Sum(item => item.Size)); + packing.Report("packing"); + AMPRPackLayout layout = new(config, outputDir); + string? tempIndex = null; + string? tempCrc = null; + string? tempRuntime = null; + try + { + PackedContent content = PackFiles(selected, config, layout, stats, packing); + + // Names go into the string table before the build id; sizes and placement are final now. + packing.Report("finalizing"); + List provisional = []; + foreach (AMPRPackVolumeWriter writer in layout.Volumes) + { + (uint nameOffset, uint nameLength) = content.Strings.Add(writer.Name); + provisional.Add(new AMPRPackRecord( + (ulong)(writer.ProjectedFinalSize() - writer.PayloadOffset), + (ulong)writer.ProjectedFinalSize(), + nameOffset, + nameLength, + writer.Flags, + (uint)writer.IOPageSize)); + } + + byte[] crcPayload = AMPRChunkCrcs.EncodePayload(content.ChunkCrcs); + byte[] strings = content.Strings.ToArray(); + byte[] buildId = AMPRBuildId.Compute([ + config.CanonicalBytes(), + Records(content.Files, AMPRPackFormat.FileRecordSize, (r, s) => r.Write(s)), + Records(content.Chunks, AMPRPackFormat.ChunkRecordSize, (r, s) => r.Write(s)), + crcPayload, + Records(provisional, AMPRPackFormat.PackRecordSize, (r, s) => r.Write(s)), + strings, + ]); + + List finalRecords = []; + for (int i = 0; i < layout.Volumes.Count; i++) + { + AMPRPackVolumeWriter writer = layout.Volumes[i]; + AMPRPackRecord actual = writer.Finalize(buildId) with + { + NameOffset = provisional[i].NameOffset, + NameLength = provisional[i].NameLength, + }; + finalRecords.Add(actual); + stats.PaddingBytes += writer.PaddingBytes; + stats.IOPagesTouched += ((long)actual.PayloadBytes + writer.IOPageSize - 1) / writer.IOPageSize; + } + + byte[] indexData = new AMPRPackManifest(buildId, 0, content.Files, content.Chunks, finalRecords, strings).Serialize(); + string indexPath = AMPRAssetPath.SafeOutputPath(outputDir, config.IndexName); + Directory.CreateDirectory(Path.GetDirectoryName(indexPath)!); + tempIndex = Path.Combine(Path.GetDirectoryName(indexPath)!, $".{Path.GetFileName(indexPath)}.tmp-{Environment.ProcessId}"); + WriteDurable(tempIndex, indexData); + tempCrc = tempIndex + ".crc"; + WriteDurable(tempCrc, AMPRChunkCrcs.Build(buildId, content.ChunkCrcs)); + if (config.Runtime is not null) + { + tempRuntime = tempIndex + ".runtime"; + WriteDurable(tempRuntime, config.Runtime.Encode(buildId)); + } + + packing.Report("publishing"); + PublishTransaction(layout, tempIndex, indexPath, tempCrc, tempRuntime); + tempIndex = tempCrc = tempRuntime = null; + packing.Report("complete"); + return new AMPRBuildResult(indexPath, stats, warnings, finalRecords.Count); + } + catch + { + layout.Abort(); + foreach (string? temp in (string?[])[tempRuntime, tempCrc, tempIndex]) + { + if (temp is not null) + { + File.Delete(temp); + } + } + + throw; + } + } + + private static PackedContent PackFiles( + List selected, AMPRPackConfig config, AMPRPackLayout layout, AMPRBuildStats stats, PackingProgress packing) + { + PackedContent content = new(); + Dictionary dedupe = new(StringComparer.Ordinal); + List sortedGroups = config.SortedGroupNames(); + using EncoderPool encoders = new(); + TaskScheduler scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, (int)config.Workers).ConcurrentScheduler; + foreach (AMPRSelectedFile item in selected) + { + string path = AMPRAssetPath.Canonical(item.IndexEntry.Path); + (uint pathOffset, uint pathLength) = content.Strings.Add(path); + ulong pathHash = AMPRAssetPath.Hash(path); + if (item.Rule.Action == "loose") + { + stats.LoosePaths.Add(item.Relative); + content.Files.Add(new AMPRFileRecord(pathHash, item.IndexEntry.Size, item.IndexEntry.MTime, 0, 0, pathOffset, pathLength, 0, 0)); + stats.FilesLoose++; + continue; + } + + packing.Report("packing", item.Relative); + AMPRGroupConfig group = config.Groups[item.Rule.Group]; + long ioPageSize = group.IOPageSize != 0 ? group.IOPageSize : config.IOPageSize; + int firstChunk = content.Chunks.Count; + uint fileFlags = AMPRPackFormat.FileFlagPacked; + if (item.Rule.Action == "store") + { + fileFlags |= AMPRPackFormat.FileFlagStoreOnly; + } + + string layoutMode = item.Rule.ResolvedLayout(); + fileFlags |= layoutMode switch + { + "streaming" => AMPRPackFormat.FileFlagStreaming, + "random" => AMPRPackFormat.FileFlagRandomAccess, + _ => 0U, + }; + if (item.Rule.Hot) + { + fileFlags |= AMPRPackFormat.FileFlagHot; + } + + bool striped = group.StripeLargeFiles && group.PackCount > 1 && item.Size >= group.StripeThreshold; + bool allowDeduplicate = config.Deduplicate && (layoutMode != "streaming" || config.DeduplicateStreaming); + string physicalLayout = layoutMode == "streaming" ? "dense" : "isolated"; + int chunkIndex = 0; + using (FileStream source = File.OpenRead(item.Source)) + { + (long Length, DateTime Written) before = AMPRPackPlanner.FileStamp(item.Source); + foreach (AMPRCompressedBlock block in CompressBlocks(source, item, config, ioPageSize, encoders, scheduler)) + { + int lane = item.Lane; + if (striped) + { + lane = (int)((item.Lane + (chunkIndex / group.StripeGroupBlocks)) % group.PackCount); + } + + string domain = config.DeduplicateScope == "group" ? item.Rule.Group : $"{item.Rule.Group}:{lane}"; + string dedupeKey = $"{domain}\0{physicalLayout}\0{block.Codec}\0{Convert.ToHexString(block.Fingerprint)}"; + AMPRChunkRecord record; + if (allowDeduplicate && dedupe.TryGetValue(dedupeKey, out AMPRChunkRecord prior)) + { + int flags = AMPRPackFormat.ChunkFlagShared + | (prior.Flags & (AMPRPackFormat.ChunkFlagPageContained | AMPRPackFormat.ChunkFlagPageAligned)) + | (layoutMode == "streaming" ? AMPRPackFormat.ChunkFlagStreaming : 0); + record = prior with { Flags = flags }; + stats.ChunksShared++; + } + else + { + // Only dense streaming extents need a page-aligned start; random/mixed chunks share pages + // but never straddle one. + bool extentStart = layoutMode == "streaming" + && (chunkIndex == 0 || (striped && chunkIndex % group.StripeGroupBlocks == 0)); + AMPRPackVolumeWriter writer = layout.WriterFor(group, lane, block.Stored.Length, striped, layoutMode, extentStart); + (long offset, int placement) = writer.Write(block.Stored, layoutMode, extentStart); + int flags = placement | (layoutMode == "streaming" ? AMPRPackFormat.ChunkFlagStreaming : 0); + if (layoutMode != "streaming" && !IsPageSafe(block.Stored.Length, placement, writer.IOPageSize)) + { + throw new AMPRPackException("non-streaming chunk was not placed page-safely"); + } + + record = new AMPRChunkRecord((ulong)offset, block.Stored.Length, block.Raw.Length, writer.PackId, block.Codec, flags); + if (allowDeduplicate) + { + dedupe[dedupeKey] = record; + } + + stats.StoredBytes += block.Stored.Length; + if (layoutMode == "streaming") + { + stats.DenseStreamingChunks++; + } + } + + if (IsPageSafe(record.StoredSize, record.Flags, layout.Volumes[record.PackId].IOPageSize)) + { + stats.IOPageSafeChunks++; + } + + content.Chunks.Add(record); + content.ChunkCrcs.Add(block.RawCrc); + chunkIndex++; + stats.Chunks++; + stats.LogicalBytes += block.Raw.Length; + packing.AddBytes(block.Raw.Length, item.Relative); + if (record.Codec == AMPRPackFormat.ChunkCodecLZ4) + { + stats.ChunksLZ4++; + } + else + { + stats.ChunksRaw++; + } + } + + if (AMPRPackPlanner.FileStamp(item.Source) != before) + { + throw new AMPRPackException($"source changed while packing: {item.Relative}"); + } + } + + content.Files.Add(new AMPRFileRecord( + pathHash, + (ulong)item.Size, + config.PreserveMTime ? item.MTime : item.IndexEntry.MTime, + (uint)firstChunk, + (uint)chunkIndex, + pathOffset, + pathLength, + fileFlags, + (byte)item.Rule.BlockShift, + (byte)sortedGroups.IndexOf(item.Rule.Group))); + stats.FilesPacked++; + packing.FileDone(item.Relative); + } + + return content; + } + + // Reads blocks in order and compresses up to max(2, 2 * workers) of them ahead on at most `workers` threads, + // yielding results in file order (Python _iter_compressed_blocks). + private static IEnumerable CompressBlocks( + FileStream source, AMPRSelectedFile item, AMPRPackConfig config, long ioPageSize, EncoderPool encoders, TaskScheduler scheduler) + { + long blockSize = 1L << item.Rule.BlockShift; + long remaining = item.Size; + int window = (int)Math.Max(2, config.Workers * 2); + Queue> pending = new(); + while (remaining > 0 || pending.Count > 0) + { + while (remaining > 0 && pending.Count < window) + { + byte[] raw = new byte[Math.Min(blockSize, remaining)]; + int read = source.ReadAtLeast(raw, raw.Length, throwOnEndOfStream: false); + if (read == 0) + { + throw new AMPRPackException("source file was truncated while packing"); + } + + if (read < raw.Length) + { + Array.Resize(ref raw, read); + } + + remaining -= read; + AMPRRule rule = item.Rule; + pending.Enqueue(Task.Factory.StartNew( + () => encoders.Run(encoder => AMPRBlockCompressor.Compress(encoder, raw, rule, ioPageSize)), + CancellationToken.None, + TaskCreationOptions.DenyChildAttach, + scheduler)); + } + + yield return pending.Dequeue().GetAwaiter().GetResult(); + } + + if (source.ReadByte() != -1) + { + throw new AMPRPackException("source file grew while packing"); + } + } + + private static bool IsPageSafe(long storedSize, int flags, long ioPageSize) => + storedSize <= ioPageSize + ? (flags & AMPRPackFormat.ChunkFlagPageContained) != 0 + : (flags & AMPRPackFormat.ChunkFlagPageAligned) != 0; + + private static byte[] Records(IReadOnlyList records, int size, Action> write) + { + byte[] data = new byte[records.Count * size]; + for (int i = 0; i < records.Count; i++) + { + write(records[i], data.AsSpan(i * size, size)); + } + + return data; + } + + private static void WriteDurable(string path, byte[] data) + { + using FileStream stream = new(path, FileMode.Create, FileAccess.Write, FileShare.None); + stream.Write(data); + stream.Flush(flushToDisk: true); + } + + /// + /// Publish volumes first, then the CRC sidecar and runtime settings (a stale .runtime is removed when the + /// build has none), and the manifest last; on failure restore the previous files (Python _publish_transaction). + /// The build id in every header makes partial combinations fail closed even after a crash. + /// + private static void PublishTransaction(AMPRPackLayout layout, string tempIndex, string indexPath, string tempCrc, string? tempRuntime) + { + List<(string Final, string? Backup)> replacements = []; + int pid = Environment.ProcessId; + try + { + foreach (AMPRPackVolumeWriter writer in layout.Volumes) + { + string? backup = MoveAside(writer.FinalPath, $".{Path.GetFileName(writer.FinalPath)}.backup-{pid}-{writer.PackId}"); + replacements.Add((writer.FinalPath, backup)); + writer.Publish(); + } + + string crcPath = AMPRPackFormat.ChunkCrcPath(indexPath); + replacements.Add((crcPath, MoveAside(crcPath, $".{Path.GetFileName(crcPath)}.backup-{pid}"))); + File.Move(tempCrc, crcPath, overwrite: true); + string runtimePath = AMPRPackFormat.RuntimePath(indexPath); + replacements.Add((runtimePath, MoveAside(runtimePath, $".{Path.GetFileName(runtimePath)}.backup-{pid}"))); + if (tempRuntime is not null) + { + File.Move(tempRuntime, runtimePath, overwrite: true); + } + + File.Move(tempIndex, indexPath, overwrite: true); + } + catch + { + for (int i = replacements.Count - 1; i >= 0; i--) + { + (string final, string? backup) = replacements[i]; + File.Delete(final); + if (backup is not null && File.Exists(backup)) + { + File.Move(backup, final, overwrite: true); + } + } + + throw; + } + + foreach ((_, string? backup) in replacements) + { + if (backup is not null) + { + File.Delete(backup); + } + } + } + + private static string? MoveAside(string path, string backupName) + { + if (!File.Exists(path)) + { + return null; + } + + string backup = Path.Combine(Path.GetDirectoryName(path)!, backupName); + File.Delete(backup); + File.Move(path, backup); + return backup; + } + + private sealed class PackedContent + { + public AMPRStringTable Strings { get; } = new(); + + public List Files { get; } = []; + + public List Chunks { get; } = []; + + public List ChunkCrcs { get; } = []; + } + + // One LZ4 encoder per concurrently running task, reused across blocks and files. + private sealed class EncoderPool : IDisposable + { + private readonly System.Collections.Concurrent.ConcurrentBag _idle = []; + private readonly List _all = []; + + public AMPRCompressedBlock Run(Func work) + { + if (!_idle.TryTake(out LZ4Encoder? encoder)) + { + encoder = new LZ4Encoder(); + lock (_all) + { + _all.Add(encoder); + } + } + + try + { + return work(encoder); + } + finally + { + _idle.Add(encoder); + } + } + + public void Dispose() + { + foreach (LZ4Encoder encoder in _all) + { + encoder.Dispose(); + } + } + } + + // Progress bookkeeping for the packing, finalizing, publishing and complete phases. + private sealed class PackingProgress(Action? callback, long filesTotal, long bytesTotal) + { + private long _filesDone; + private long _bytesDone; + + public void Report(string phase, string currentPath = "") => + callback?.Invoke(new AMPRBuildProgress(phase, _filesDone, filesTotal, _bytesDone, bytesTotal, currentPath)); + + public void AddBytes(long bytes, string currentPath) + { + _bytesDone += bytes; + Report("packing", currentPath); + } + + public void FileDone(string currentPath) + { + _filesDone++; + Report("packing", currentPath); + } + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRPackConfig.cs b/src/MkPFS.Build/AMPRPack/AMPRPackConfig.cs new file mode 100644 index 0000000..98ef3b7 --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRPackConfig.cs @@ -0,0 +1,713 @@ +using System.Text; +using MkPFS.Core.AMPR; +using MkPFS.Core.Util; +using Tomlyn.Model; + +namespace MkPFS.Build.AMPRPack; + +/// A packing group ([groups.<name>], Python GroupConfig). +/// Group name. +/// Pack lanes (1..65535). +/// balanced, hash or round_robin. +/// Volume size limit; 0 means unlimited. +/// Stripe large files across lanes. +/// Smallest file that is striped. +/// Blocks per stripe. +/// Per-group I/O page size; 0 uses the global one. +public sealed record AMPRGroupConfig( + string Name, + long PackCount = 1, + string Assignment = "balanced", + long MaxPackSize = 0, + bool StripeLargeFiles = false, + long StripeThreshold = 256L * 1024 * 1024, + long StripeGroupBlocks = 8, + long IOPageSize = 0); + +/// An include/exclude rule ([[rule]], Python Rule). +/// compress, store or loose. +/// Include globs. +/// Exclude globs. +/// log2 of the block size. +/// Packing group. +/// LZ4 mode, fast or hc. +/// HC level. +/// Fast-mode acceleration. +/// Smallest saving that keeps a compressed block. +/// Smallest saving ratio that keeps a compressed block. +/// Smallest saving when RAW would use the same I/O pages. +/// Smallest saving ratio when RAW would use the same I/O pages. +/// auto, random, mixed or streaming. +/// Hint for the auto layout. +/// Hint for the auto layout and immediate cache admission. +/// Never auto-loose matched files. +public sealed record AMPRRule( + string Action, + IReadOnlyList Include, + IReadOnlyList Exclude, + int BlockShift = 16, + string Group = "default", + string Mode = "hc", + long Level = 12, + long Acceleration = 1, + long MinSavingsBytes = 64, + double MinSavingsRatio = 0.01, + long IONeutralMinSavingsBytes = 8192, + double IONeutralMinSavingsRatio = 0.125, + string Layout = "auto", + bool Streaming = false, + bool Hot = false, + bool ForcePack = false) +{ + /// A loose rule (Python Rule(action="loose", block_shift=...)). + /// Block shift to keep. + /// The rule. + public static AMPRRule Loose(int blockShift) => new("loose", ["**"], [], blockShift); + + /// Whether a relative path is included and not excluded. + /// Path relative to /app0. + /// when the rule applies. + public bool Matches(string relativePath) => + AMPRGlob.Matches(relativePath, Include) && !AMPRGlob.Matches(relativePath, Exclude); + + /// The layout with auto resolved: streaming, then hot (random), else mixed. + /// Layout. + public string ResolvedLayout() => Layout != "auto" ? Layout : Streaming ? "streaming" : Hot ? "random" : "mixed"; +} + +/// +/// Asset-pack build configuration (Python BuildConfig and load_config). Keys are read leniently like +/// the oracle (unknown keys are ignored) and converted with Python semantics; validation order and messages match. +/// +public sealed class AMPRPackConfig +{ + /// ampr_pack tool version; part of the build id. + public const string ToolVersion = "4.0"; + + /// Default manifest name. + public const string DefaultIndexName = "ampr_assets.index"; + + /// Default volume name pattern. + public const string DefaultPackPattern = "ampr_assets-{id:03d}.pak"; + + private static readonly string[] RuntimeKeys = ["decoded_cache_bytes", "physical_cache_bytes", "workers", "latency_reserve_workers"]; + + /// Runtime settings written as <index>.runtime, or . + public AMPRRuntimeSettings? Runtime { get; set; } + + /// Manifest file name. + public string IndexName { get; set; } = DefaultIndexName; + + /// Volume name pattern. + public string PackPattern { get; set; } = DefaultPackPattern; + + /// Action for files no rule matches. + public string DefaultAction { get; set; } = "loose"; + + /// Default block shift. + public int DefaultBlockShift { get; set; } = 16; + + /// Filesystem I/O page size. + public long IOPageSize { get; set; } = 64 * 1024; + + /// Volume payload alignment. + public long PayloadAlignment { get; set; } = 64 * 1024; + + /// Chunk alignment inside a page. + public long ChunkAlignment { get; set; } = 64; + + /// Compression worker threads (not part of the build id). + public long Workers { get; set; } = Math.Max(1, Math.Min(8, Environment.ProcessorCount)); + + /// Default LZ4 mode. + public string CompressionMode { get; set; } = "hc"; + + /// Default HC level. + public long CompressionLevel { get; set; } = 12; + + /// Default fast-mode acceleration. + public long Acceleration { get; set; } = 1; + + /// Default minimum saving in bytes. + public long MinSavingsBytes { get; set; } = 64; + + /// Default minimum saving ratio. + public double MinSavingsRatio { get; set; } = 0.01; + + /// Default minimum saving in bytes when the page count does not change. + public long IONeutralMinSavingsBytes { get; set; } = 8 * 1024; + + /// Default minimum saving ratio when the page count does not change. + public double IONeutralMinSavingsRatio { get; set; } = 0.125; + + /// Deduplicate identical blocks. + public bool Deduplicate { get; set; } = true; + + /// lane or group. + public string DeduplicateScope { get; set; } = "lane"; + + /// Also deduplicate inside streaming files. + public bool DeduplicateStreaming { get; set; } + + /// Store the source mtime instead of the AMPRIDX3 one. + public bool PreserveMTime { get; set; } = true; + + /// Check source sizes against AMPRIDX3. + public bool ValidateIndexMetadata { get; set; } = true; + + /// Never auto-loose explicitly packed files. + public bool SelfContained { get; set; } + + /// Globs that must end up packed. + public IReadOnlyList RequiredPacked { get; set; } = []; + + /// Leave large incompressible files loose. + public bool AutoLooseLargeFiles { get; set; } = true; + + /// Also auto-loose files from hot rules. + public bool AutoLooseHotFiles { get; set; } + + /// Smallest file considered for auto-loose. + public long AutoLooseMinFileSize { get; set; } = 64L * 1024 * 1024; + + /// Blocks sampled per candidate. + public long AutoLooseSampleBlocks { get; set; } = 32; + + /// Byte budget of the sample. + public long AutoLooseSampleBytes { get; set; } = 16L * 1024 * 1024; + + /// Saving ratio below which a file goes loose. + public double AutoLooseMinSavingsRatio { get; set; } = 0.05; + + /// RAW sample ratio above which a file goes loose. + public double AutoLooseMaxRawRatio { get; set; } = 0.90; + + /// Groups in declaration order; default is added last when missing. + public IReadOnlyDictionary Groups { get; set; } = new Dictionary(); + + /// Rules in order (last match wins). + public IReadOnlyList Rules { get; set; } = []; + + /// The rule for files no rule matches (Python _default_rule). + /// Rule. + public AMPRRule DefaultRule() => new( + DefaultAction, + ["**"], + [], + DefaultBlockShift, + Mode: CompressionMode, + Level: CompressionLevel, + Acceleration: Acceleration, + MinSavingsBytes: MinSavingsBytes, + MinSavingsRatio: MinSavingsRatio, + IONeutralMinSavingsBytes: IONeutralMinSavingsBytes, + IONeutralMinSavingsRatio: IONeutralMinSavingsRatio); + + /// The last matching rule, else the default rule (Python select_rule). + /// Path relative to /app0. + /// Rule. + public AMPRRule SelectRule(string relative) + { + AMPRRule selected = DefaultRule(); + foreach (AMPRRule rule in Rules) + { + if (rule.Matches(relative)) + { + selected = rule; + } + } + + return selected; + } + + /// + /// Build-id input: json.dumps(payload, sort_keys=True, separators=(",", ":")) of the content-relevant + /// settings, groups (sorted by name) and rules (Python _canonical_config_bytes). + /// + /// UTF-8 bytes. + public byte[] CanonicalBytes() + { + Dictionary payload = new() + { + ["tool_version"] = ToolVersion, + ["index_name"] = IndexName, + ["pack_pattern"] = PackPattern, + ["default_action"] = DefaultAction, + ["default_block_shift"] = (long)DefaultBlockShift, + ["io_page_size"] = IOPageSize, + ["payload_alignment"] = PayloadAlignment, + ["chunk_alignment"] = ChunkAlignment, + ["compression_mode"] = CompressionMode, + ["compression_level"] = CompressionLevel, + ["acceleration"] = Acceleration, + ["min_savings_bytes"] = MinSavingsBytes, + ["min_savings_ratio"] = MinSavingsRatio, + ["io_neutral_min_savings_bytes"] = IONeutralMinSavingsBytes, + ["io_neutral_min_savings_ratio"] = IONeutralMinSavingsRatio, + ["deduplicate"] = Deduplicate, + ["deduplicate_scope"] = DeduplicateScope, + ["deduplicate_streaming"] = DeduplicateStreaming, + ["auto_loose_large_files"] = AutoLooseLargeFiles, + ["auto_loose_hot_files"] = AutoLooseHotFiles, + ["auto_loose_min_file_size"] = AutoLooseMinFileSize, + ["auto_loose_sample_blocks"] = AutoLooseSampleBlocks, + ["auto_loose_sample_bytes"] = AutoLooseSampleBytes, + ["auto_loose_min_savings_ratio"] = AutoLooseMinSavingsRatio, + ["auto_loose_max_raw_ratio"] = AutoLooseMaxRawRatio, + ["groups"] = SortedGroupNames().Select(name => (object?)GroupPayload(Groups[name])).ToList(), + ["rules"] = Rules.Select(rule => (object?)RulePayload(rule)).ToList(), + }; + return Encoding.UTF8.GetBytes(PythonSortedJson.Compact(payload)); + } + + /// Group names sorted by code point (Python sorted(config.groups)); the index is the packing class. + /// Names. + public List SortedGroupNames() => [.. Groups.Keys.OrderBy(name => name, PythonCodePointComparer.Instance)]; + + /// + /// Load a configuration, or the defaults when is (Python + /// load_config). + /// + /// TOML file. + /// Configuration. + /// The configuration is invalid. + /// A value cannot be converted (Python ValueError). + public static AMPRPackConfig Load(string? path) + { + TomlTable raw = []; + string configDir = Directory.GetCurrentDirectory(); + if (path is not null) + { + string resolved = Path.GetFullPath(path); + configDir = Path.GetDirectoryName(resolved)!; + raw = AMPRToml.Load(resolved); + } + + return FromTable(raw, configDir); + } + + /// + /// Read pattern lines: stripped, without blank lines and # comments (Python _load_pattern_file). + /// The file is decoded as UTF-8 without BOM detection, like Python read_text(encoding="utf-8"). + /// + /// File, or for none. + /// Patterns. + public static List LoadPatternFile(string? path) + { + if (path is null) + { + return []; + } + + string text; + using (StreamReader reader = new(path, new UTF8Encoding(false, throwOnInvalidBytes: true), detectEncodingFromByteOrderMarks: false)) + { + text = reader.ReadToEnd(); + } + + List patterns = []; + foreach (string line in PythonText.SplitLines(text)) + { + string stripped = PythonText.Strip(line); + if (stripped.Length > 0 && !stripped.StartsWith('#')) + { + patterns.Add(stripped); + } + } + + return patterns; + } + + /// Build a configuration from a parsed TOML document. + /// Root table. + /// Directory that include_from/exclude_from are relative to. + /// Configuration. + internal static AMPRPackConfig FromTable(TomlTable raw, string configDir) + { + object? packValue = raw.TryGetValue("pack", out object? p) ? p : new TomlTable(); + if (packValue is not TomlTable pack) + { + throw new AMPRPackException("[pack] must be a table"); + } + + AMPRRuntimeSettings? runtime = raw.TryGetValue("runtime", out object? runtimeValue) ? LoadRuntime(runtimeValue) : null; + + // Python evaluates BuildConfig(...) arguments in this order; the first failing conversion wins. + int defaultBlock = PythonValue.BlockShift(Get(pack, "default_block_size", "64KiB")); + long ioPageSize = PythonValue.Size(Get(pack, "io_page_size", "64KiB")); + AMPRPackConfig config = new() + { + Runtime = runtime, + IndexName = PythonValue.Str(Get(pack, "index_name", DefaultIndexName)), + PackPattern = PythonValue.Str(Get(pack, "pack_pattern", DefaultPackPattern)), + DefaultAction = Action(Get(pack, "default_action", "loose")), + DefaultBlockShift = defaultBlock, + IOPageSize = ioPageSize, + PayloadAlignment = PythonValue.Size(Get(pack, "payload_alignment", ioPageSize)), + ChunkAlignment = PythonValue.Size(Get(pack, "chunk_alignment", "64B")), + Workers = PythonValue.Int(Get(pack, "workers", (long)Math.Max(1, Math.Min(8, Environment.ProcessorCount)))), + CompressionMode = PythonValue.Str(Get(pack, "compression_mode", "hc")).ToLowerInvariant(), + CompressionLevel = PythonValue.Int(Get(pack, "compression_level", 12L)), + Acceleration = PythonValue.Int(Get(pack, "acceleration", 1L)), + MinSavingsBytes = PythonValue.Size(Get(pack, "min_savings_bytes", 64L)), + MinSavingsRatio = PythonValue.Float(Get(pack, "min_savings_ratio", 0.01)), + IONeutralMinSavingsBytes = PythonValue.Size(Get(pack, "io_neutral_min_savings_bytes", "8KiB")), + IONeutralMinSavingsRatio = PythonValue.Float(Get(pack, "io_neutral_min_savings_ratio", 0.125)), + Deduplicate = PythonValue.Bool(Get(pack, "deduplicate", true)), + DeduplicateScope = PythonValue.Str(Get(pack, "deduplicate_scope", "lane")).ToLowerInvariant(), + DeduplicateStreaming = PythonValue.Bool(Get(pack, "deduplicate_streaming", false)), + PreserveMTime = PythonValue.Bool(Get(pack, "preserve_mtime", true)), + ValidateIndexMetadata = PythonValue.Bool(Get(pack, "validate_index_metadata", true)), + SelfContained = PythonValue.Bool(Get(pack, "self_contained", false)), + RequiredPacked = ListOfStrings(Get(pack, "required_packed", new TomlArray()), "pack.required_packed"), + AutoLooseLargeFiles = PythonValue.Bool(Get(pack, "auto_loose_large_files", true)), + AutoLooseHotFiles = PythonValue.Bool(Get(pack, "auto_loose_hot_files", false)), + AutoLooseMinFileSize = PythonValue.Size(Get(pack, "auto_loose_min_file_size", "64MiB")), + AutoLooseSampleBlocks = PythonValue.Int(Get(pack, "auto_loose_sample_blocks", 32L)), + AutoLooseSampleBytes = PythonValue.Size(Get(pack, "auto_loose_sample_bytes", "16MiB")), + AutoLooseMinSavingsRatio = PythonValue.Float(Get(pack, "auto_loose_min_savings_ratio", 0.05)), + AutoLooseMaxRawRatio = PythonValue.Float(Get(pack, "auto_loose_max_raw_ratio", 0.90)), + }; + config.ValidatePackSection(); + config.Groups = LoadGroups(raw, config); + config.Rules = LoadRules(raw, config, configDir); + return config; + } + + private void ValidatePackSection() + { + if (Workers is < 1 or > 256) + { + throw new AMPRPackException("workers must be between 1 and 256"); + } + + if (IOPageSize < 1L << AMPRPackFormat.MinIOPageShift || (IOPageSize & (IOPageSize - 1)) != 0 || IOPageSize > 1L << AMPRPackFormat.MaxIOPageShift) + { + throw new AMPRPackException("io_page_size must be a power of two between 4 KiB and 1 MiB"); + } + + if (PayloadAlignment < AMPRPackFormat.DataHeaderSize + || (PayloadAlignment & (PayloadAlignment - 1)) != 0 + || PayloadAlignment > 1L << AMPRPackFormat.MaxBlockShift) + { + throw new AMPRPackException( + "payload_alignment must be a power of two between 64 B and 1 MiB; the writer automatically raises it to the selected io_page_size"); + } + + if (ChunkAlignment < AMPRPackFormat.PhysicalChunkAlignment + || (ChunkAlignment & (ChunkAlignment - 1)) != 0 + || ChunkAlignment > IOPageSize + || IOPageSize % ChunkAlignment != 0) + { + throw new AMPRPackException("chunk_alignment must be a power of two between 64 B and io_page_size"); + } + + if (CompressionMode is not ("fast" or "hc")) + { + throw new AMPRPackException("compression_mode must be fast or hc"); + } + + if (DeduplicateScope is not ("lane" or "group")) + { + throw new AMPRPackException("deduplicate_scope must be lane or group"); + } + + if (CompressionLevel is < 1 or > 12) + { + throw new AMPRPackException("compression_level must be between 1 and 12"); + } + + if (Acceleration < 1) + { + throw new AMPRPackException("acceleration must be at least 1"); + } + + CheckRatio(MinSavingsRatio, "min_savings_ratio must be in [0, 1)"); + CheckRatio(IONeutralMinSavingsRatio, "io_neutral_min_savings_ratio must be in [0, 1)"); + if (AutoLooseSampleBlocks is < 1 or > 4096) + { + throw new AMPRPackException("auto_loose_sample_blocks must be between 1 and 4096"); + } + + if (AutoLooseSampleBytes < 1) + { + throw new AMPRPackException("auto_loose_sample_bytes must be at least 1 byte"); + } + + CheckRatio(AutoLooseMinSavingsRatio, "auto_loose_min_savings_ratio must be in [0, 1)"); + if (!(AutoLooseMaxRawRatio >= 0.0 && AutoLooseMaxRawRatio <= 1.0)) + { + throw new AMPRPackException("auto_loose_max_raw_ratio must be in [0, 1]"); + } + + try + { + AMPRAssetPath.SafeOutputPath(".", IndexName.Replace('\\', '/')); + } + catch (ArgumentException exc) + { + throw new AMPRPackException($"unsafe index_name: {PythonText.Repr(IndexName)}", exc); + } + + AMPRPackPattern.Parse(PackPattern); + } + + private static Dictionary LoadGroups(TomlTable raw, AMPRPackConfig config) + { + Dictionary groups = new(StringComparer.Ordinal); + object? groupsRaw = raw.TryGetValue("groups", out object? g) ? g : null; + if (PythonValue.Bool(groupsRaw) && groupsRaw is not TomlTable) + { + throw new AMPRPackException("[groups] must be a table"); + } + + if (groupsRaw is TomlTable groupTable) + { + foreach (KeyValuePair entry in groupTable) + { + string name = entry.Key; + if (entry.Value is not TomlTable value) + { + throw new AMPRPackException($"[groups.{name}] must be a table"); + } + + AMPRGroupConfig group = new( + name, + PythonValue.Int(Get(value, "pack_count", 1L)), + Assignment(Get(value, "assignment", "balanced")), + PythonValue.Size(Get(value, "max_pack_size", 0L)), + PythonValue.Bool(Get(value, "stripe_large_files", false)), + PythonValue.Size(Get(value, "stripe_threshold", "256MiB")), + PythonValue.Int(Get(value, "stripe_group_blocks", 8L)), + PythonValue.Size(Get(value, "io_page_size", 0L))); + if (group.PackCount is < 1 or > 0xFFFF) + { + throw new AMPRPackException($"groups.{name}.pack_count must be between 1 and 65535"); + } + + long groupPage = group.IOPageSize != 0 ? group.IOPageSize : config.IOPageSize; + if (groupPage < 1L << AMPRPackFormat.MinIOPageShift + || groupPage > 1L << AMPRPackFormat.MaxIOPageShift + || (groupPage & (groupPage - 1)) != 0 + || groupPage % config.ChunkAlignment != 0) + { + throw new AMPRPackException( + $"groups.{name}.io_page_size must be a power of two between 4 KiB and 1 MiB and a multiple of chunk_alignment"); + } + + long minimumPayloadOffset = (long)AMPRPackFormat.AlignUp(AMPRPackFormat.DataHeaderSize, (ulong)Math.Max(config.PayloadAlignment, groupPage)); + long minimumVolumeSize = minimumPayloadOffset + groupPage; + if (group.MaxPackSize != 0 && group.MaxPackSize < minimumVolumeSize) + { + throw new AMPRPackException( + $"groups.{name}.max_pack_size must be at least one complete I/O page beyond the payload offset ({minimumVolumeSize} bytes)"); + } + + if (group.StripeGroupBlocks < 1) + { + throw new AMPRPackException($"groups.{name}.stripe_group_blocks must be at least 1"); + } + + groups[name] = group; + } + } + + groups.TryAdd("default", new AMPRGroupConfig("default")); + return groups.Count > 256 ? throw new AMPRPackException("the binary format supports at most 256 packing groups") : groups; + } + + private static List LoadRules(TomlTable raw, AMPRPackConfig config, string configDir) + { + object? rulesRaw = raw.TryGetValue("rule", out object? r) ? r : new TomlArray(); + List items = rulesRaw switch + { + TomlTableArray tables => [.. tables], + TomlArray array => [.. array], + _ => throw new AMPRPackException("[[rule]] entries must form an array"), + }; + + List rules = []; + for (int index = 0; index < items.Count; index++) + { + if (items[index] is not TomlTable value) + { + throw new AMPRPackException($"rule {index} must be a table"); + } + + string group = PythonValue.Str(Get(value, "group", "default")); + if (!config.Groups.ContainsKey(group)) + { + throw new AMPRPackException($"rule {index} references unknown group {PythonText.Repr(group)}"); + } + + int shift = PythonValue.BlockShift(Get(value, "block_size", 1L << config.DefaultBlockShift)); + List includeFrom = LoadRulePatternFiles(Get(value, "include_from", new TomlArray()), "include_from", configDir); + List excludeFrom = LoadRulePatternFiles(Get(value, "exclude_from", new TomlArray()), "exclude_from", configDir); + object includeDefault = includeFrom.Count > 0 ? new TomlArray() : new TomlArray { "**" }; + List include = [.. ListOfStrings(Get(value, "include", includeDefault), "include"), .. includeFrom]; + List exclude = [.. ListOfStrings(Get(value, "exclude", new TomlArray()), "exclude"), .. excludeFrom]; + AMPRRule rule = new( + Action(Get(value, "action", "compress")), + include, + exclude, + shift, + group, + PythonValue.Str(Get(value, "compression_mode", config.CompressionMode)).ToLowerInvariant(), + PythonValue.Int(Get(value, "compression_level", config.CompressionLevel)), + PythonValue.Int(Get(value, "acceleration", config.Acceleration)), + PythonValue.Size(Get(value, "min_savings_bytes", config.MinSavingsBytes)), + PythonValue.Float(Get(value, "min_savings_ratio", config.MinSavingsRatio)), + PythonValue.Size(Get(value, "io_neutral_min_savings_bytes", config.IONeutralMinSavingsBytes)), + PythonValue.Float(Get(value, "io_neutral_min_savings_ratio", config.IONeutralMinSavingsRatio)), + Layout(Get(value, "layout", "auto")), + PythonValue.Bool(Get(value, "streaming", false)), + PythonValue.Bool(Get(value, "hot", false)), + PythonValue.Bool(Get(value, "force_pack", false))); + if (rule.Include.Count == 0) + { + throw new AMPRPackException($"rule {index} include list cannot be empty"); + } + + if (rule.Mode is not ("fast" or "hc")) + { + throw new AMPRPackException($"rule {index} compression_mode must be fast or hc"); + } + + if (rule.Level is < 1 or > 12) + { + throw new AMPRPackException($"rule {index} compression_level must be between 1 and 12"); + } + + if (rule.Acceleration < 1) + { + throw new AMPRPackException($"rule {index} acceleration must be at least 1"); + } + + CheckRatio(rule.MinSavingsRatio, $"rule {index} min_savings_ratio must be in [0, 1)"); + CheckRatio(rule.IONeutralMinSavingsRatio, $"rule {index} io_neutral_min_savings_ratio must be in [0, 1)"); + rules.Add(rule); + } + + return rules; + } + + private static List LoadRulePatternFiles(object? value, string field, string configDir) + { + List patterns = []; + string root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(configDir)); + StringComparison comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + foreach (string name in ListOfStrings(value, field)) + { + if (Path.IsPathFullyQualified(name)) + { + throw new AMPRPackException($"{field} entries must be relative to the TOML file"); + } + + string candidate = Path.GetFullPath(Path.Combine(root, name)); + string prefix = Path.EndsInDirectorySeparator(root) ? root : root + Path.DirectorySeparatorChar; + if (!candidate.Equals(root, comparison) && !candidate.StartsWith(prefix, comparison)) + { + throw new AMPRPackException($"{field} entry escapes the TOML directory: {name}"); + } + + try + { + patterns.AddRange(LoadPatternFile(candidate)); + } + catch (Exception exc) when (exc is IOException or UnauthorizedAccessException) + { + throw new AMPRPackException($"cannot read {field} file {name}: {exc.Message}", exc); + } + } + + return patterns; + } + + private static AMPRRuntimeSettings LoadRuntime(object? value) + { + if (value is not TomlTable runtime + || runtime.Count != RuntimeKeys.Length + || !RuntimeKeys.All(runtime.ContainsKey)) + { + throw new AMPRPackException( + "[runtime] requires exactly decoded_cache_bytes, physical_cache_bytes, workers, latency_reserve_workers"); + } + + // RuntimeSettings(parse_size(...), parse_size(...), workers, reserve).validate(): only real ints pass. + object? decodedRaw = runtime["decoded_cache_bytes"]; + object? physicalRaw = runtime["physical_cache_bytes"]; + long decoded = PythonValue.Size(decodedRaw); + long physical = PythonValue.Size(physicalRaw); + if (decodedRaw is bool || physicalRaw is bool || runtime["workers"] is not long workers || runtime["latency_reserve_workers"] is not long reserve) + { + throw new ArgumentException("runtime settings must be integers"); + } + + AMPRRuntimeSettings settings = new(decoded, physical, workers, reserve); + settings.Validate(); + return settings; + } + + private static object? Get(TomlTable table, string key, object? fallback) => + table.TryGetValue(key, out object? value) ? value : fallback; + + private static List ListOfStrings(object? value, string key) => value switch + { + null => [], + string s => [s], + TomlArray array when array.All(item => item is string) => [.. array.Cast()], + _ => throw new AMPRPackException($"{key} must be a string or list of strings"), + }; + + private static string Action(object? value) => Choice(value, ["compress", "store", "loose"], "action must be compress, store or loose"); + + private static string Layout(object? value) => Choice(value, ["auto", "random", "mixed", "streaming"], "layout must be auto, random, mixed or streaming"); + + private static string Assignment(object? value) => Choice(value, ["balanced", "hash", "round_robin"], "assignment must be balanced, hash or round_robin"); + + private static string Choice(object? value, string[] allowed, string message) + { + string result = PythonValue.Str(value).ToLowerInvariant(); + return allowed.Contains(result, StringComparer.Ordinal) ? result : throw new AMPRPackException($"{message}; got {PythonText.Repr(result)}"); + } + + private static void CheckRatio(double value, string message) + { + if (!(value >= 0.0 && value < 1.0)) + { + throw new AMPRPackException(message); + } + } + + private static Dictionary GroupPayload(AMPRGroupConfig group) => new() + { + ["name"] = group.Name, + ["pack_count"] = group.PackCount, + ["assignment"] = group.Assignment, + ["max_pack_size"] = group.MaxPackSize, + ["stripe_large_files"] = group.StripeLargeFiles, + ["stripe_threshold"] = group.StripeThreshold, + ["stripe_group_blocks"] = group.StripeGroupBlocks, + ["io_page_size"] = group.IOPageSize, + }; + + private static Dictionary RulePayload(AMPRRule rule) => new() + { + ["action"] = rule.Action, + ["include"] = rule.Include, + ["exclude"] = rule.Exclude, + ["block_shift"] = (long)rule.BlockShift, + ["group"] = rule.Group, + ["mode"] = rule.Mode, + ["level"] = rule.Level, + ["acceleration"] = rule.Acceleration, + ["min_savings_bytes"] = rule.MinSavingsBytes, + ["min_savings_ratio"] = rule.MinSavingsRatio, + ["io_neutral_min_savings_bytes"] = rule.IONeutralMinSavingsBytes, + ["io_neutral_min_savings_ratio"] = rule.IONeutralMinSavingsRatio, + ["layout"] = rule.Layout, + ["streaming"] = rule.Streaming, + ["hot"] = rule.Hot, + ["force_pack"] = rule.ForcePack, + }; +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRPackLayout.cs b/src/MkPFS.Build/AMPRPack/AMPRPackLayout.cs new file mode 100644 index 0000000..f8b8d7f --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRPackLayout.cs @@ -0,0 +1,350 @@ +using System.Text; +using MkPFS.Core.AMPR; + +namespace MkPFS.Build.AMPRPack; + +/// +/// One AMPRDAT3 data volume being written to a temporary file (Python PackVolumeWriter). The header is +/// written last, once the build id is known. +/// +public sealed class AMPRPackVolumeWriter : IDisposable +{ + private readonly FileStream _stream; + private bool _closed; + + internal AMPRPackVolumeWriter( + int packId, + string name, + string outputDir, + long payloadAlignment, + long chunkAlignment, + long ioPageSize, + long maxPackSize, + uint flags) + { + PackId = packId; + Name = name; + FinalPath = AMPRAssetPath.SafeOutputPath(outputDir, name); + Directory.CreateDirectory(Path.GetDirectoryName(FinalPath)!); + TempPath = Path.Combine(Path.GetDirectoryName(FinalPath)!, $".{Path.GetFileName(FinalPath)}.tmp-{Environment.ProcessId}-{packId}"); + ChunkAlignment = chunkAlignment; + IOPageSize = ioPageSize; + PayloadOffset = Align(AMPRPackFormat.DataHeaderSize, payloadAlignment); + MaxPackSize = maxPackSize; + Flags = flags | AMPRPackFormat.PackFlagIOPageLayout; + _stream = new FileStream(TempPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + WriteZeros(PayloadOffset); + Position = PayloadOffset; + PaddingBytes = PayloadOffset - AMPRPackFormat.DataHeaderSize; + } + + /// Pack id. + public int PackId { get; } + + /// Volume name relative to the output directory. + public string Name { get; } + + /// Published path. + public string FinalPath { get; } + + /// Temporary path until publication. + public string TempPath { get; } + + /// Chunk alignment. + public long ChunkAlignment { get; } + + /// I/O page size of this volume. + public long IOPageSize { get; } + + /// Payload start. + public long PayloadOffset { get; } + + /// Size limit; 0 or less means unlimited. + public long MaxPackSize { get; } + + /// Pack flags. + public uint Flags { get; internal set; } + + /// Current end of written data. + public long Position { get; private set; } + + /// Alignment padding written so far. + public long PaddingBytes { get; private set; } + + /// Where a chunk would be placed and its placement flags (Python placement_for). + /// Stored size (positive). + /// random, mixed or streaming. + /// Start of a streaming extent (page-aligned). + /// Offset and ChunkFlagPage* bits. + public (long Offset, int Flags) PlacementFor(long storedSize, string layout, bool extentStart = false) + { + if (storedSize <= 0) + { + throw new AMPRPackException("cannot place an empty physical chunk"); + } + + long position = Position; + if (extentStart) + { + position = Align(position, IOPageSize); + } + + long offset; + if (layout == "streaming") + { + // Streaming extents stay dense; the runtime merges adjacent ranges. + offset = Align(position, ChunkAlignment); + } + else if (storedSize >= IOPageSize) + { + // Large random/mixed chunks start on a page, so a cold read touches ceil(size / page) pages. + offset = Align(position, IOPageSize); + } + else + { + // Small random/mixed chunks share pages but never straddle one. + offset = Align(position, ChunkAlignment); + long pageEnd = Align(offset + 1, IOPageSize); + if (offset + storedSize > pageEnd) + { + offset = Align(offset, IOPageSize); + } + } + + int placement = 0; + if (storedSize <= IOPageSize && offset / IOPageSize == (offset + storedSize - 1) / IOPageSize) + { + placement |= AMPRPackFormat.ChunkFlagPageContained; + } + + if (offset % IOPageSize == 0) + { + placement |= AMPRPackFormat.ChunkFlagPageAligned; + } + + return (offset, placement); + } + + /// Whether a chunk still fits under (Python can_fit). + /// Stored size. + /// Layout. + /// Start of a streaming extent. + /// when the page-rounded end stays within the limit. + public bool CanFit(long storedSize, string layout, bool extentStart = false) + { + (long offset, _) = PlacementFor(storedSize, layout, extentStart); + return MaxPackSize <= 0 || Align(offset + storedSize, IOPageSize) <= MaxPackSize; + } + + /// Append a chunk with its padding (Python write). + /// Stored bytes. + /// Layout. + /// Start of a streaming extent. + /// Offset and placement flags. + public (long Offset, int Flags) Write(byte[] payload, string layout, bool extentStart = false) + { + (long offset, int placement) = PlacementFor(payload.Length, layout, extentStart); + long padding = offset - Position; + if (padding > 0) + { + WriteZeros(padding); + PaddingBytes += padding; + } + + _stream.Write(payload); + Position = offset + payload.Length; + return (offset, placement); + } + + /// File size after tail padding to a whole page. + /// Size. + public long ProjectedFinalSize() => Align(Position, IOPageSize); + + /// Pad the tail, write the header and close the temporary file (Python finalize). + /// Build id. + /// Pack record without name. + public AMPRPackRecord Finalize(byte[] buildId) + { + if (_closed) + { + throw new AMPRPackException("pack was already finalized"); + } + + long finalSize = ProjectedFinalSize(); + long tail = finalSize - Position; + if (tail > 0) + { + WriteZeros(tail); + PaddingBytes += tail; + Position = finalSize; + } + + long payloadBytes = Position - PayloadOffset; + byte[] header = new AMPRDataHeader((uint)PackId, buildId, (ulong)PayloadOffset, (ulong)payloadBytes, Flags).ToBytes(); + _stream.Seek(0, SeekOrigin.Begin); + _stream.Write(header); + _stream.Flush(flushToDisk: true); + _stream.Dispose(); + _closed = true; + long fileSize = new FileInfo(TempPath).Length; + return fileSize == Position + ? new AMPRPackRecord((ulong)payloadBytes, (ulong)fileSize, 0, 0, Flags, (uint)IOPageSize) + : throw new AMPRPackException($"unexpected size for {Name}"); + } + + /// Move the finished volume into place. + public void Publish() + { + if (!_closed) + { + throw new AMPRPackException("cannot publish an open pack"); + } + + File.Move(TempPath, FinalPath, overwrite: true); + } + + /// Close and delete the temporary file. + public void Abort() + { + Dispose(); + File.Delete(TempPath); + } + + /// + public void Dispose() + { + if (!_closed) + { + _stream.Dispose(); + _closed = true; + } + } + + internal static long Align(long value, long alignment) => + checked((long)AMPRPackFormat.AlignUp((ulong)value, (ulong)alignment)); + + private void WriteZeros(long count) + { + Span zeros = stackalloc byte[4096]; + zeros.Clear(); + while (count > 0) + { + int n = (int)Math.Min(count, zeros.Length); + _stream.Write(zeros[..n]); + count -= n; + } + } +} + +/// Volume allocation per (group, lane) with rollover (Python PackLayout). +public sealed class AMPRPackLayout +{ + private readonly AMPRPackConfig _config; + private readonly AMPRPackPattern _pattern; + private readonly string _outputDir; + private readonly Dictionary<(string Group, int Lane), AMPRPackVolumeWriter> _current = []; + private readonly Dictionary<(string Group, int Lane), int> _volumeNumbers = []; + private readonly HashSet _names = new(StringComparer.Ordinal); + + /// Create a layout. + /// Configuration. + /// Output directory. + public AMPRPackLayout(AMPRPackConfig config, string outputDir) + { + _config = config; + _pattern = AMPRPackPattern.Parse(config.PackPattern); + _outputDir = outputDir; + } + + /// Volumes in pack-id order. + public List Volumes { get; } = []; + + /// The volume to append a chunk to, opening a new one when the current one is full (Python writer_for). + /// Group. + /// Lane. + /// Stored size. + /// Whether the file is striped. + /// Layout. + /// Start of a streaming extent. + /// Writer. + public AMPRPackVolumeWriter WriterFor(AMPRGroupConfig group, int lane, long storedSize, bool striped, string layout, bool extentStart) + { + uint flags = AMPRPackFormat.PackFlagIOPageLayout | (striped ? AMPRPackFormat.PackFlagStriped : 0); + if (!_current.TryGetValue((group.Name, lane), out AMPRPackVolumeWriter? writer)) + { + writer = NewVolume(group, lane, flags); + } + + if (!writer.CanFit(storedSize, layout, extentStart)) + { + writer = NewVolume(group, lane, flags); + if (!writer.CanFit(storedSize, layout, extentStart: true)) + { + throw new AMPRPackException($"one block ({storedSize} bytes) exceeds max pack size for group {group.Name}"); + } + } + + if (striped) + { + writer.Flags |= AMPRPackFormat.PackFlagStriped; + } + + return writer; + } + + /// Delete every temporary volume. + public void Abort() + { + foreach (AMPRPackVolumeWriter writer in Volumes) + { + writer.Abort(); + } + } + + private AMPRPackVolumeWriter NewVolume(AMPRGroupConfig group, int lane, uint flags) + { + int packId = Volumes.Count; + if (packId > 0xFFFF) + { + throw new AMPRPackException("pack count exceeds uint16 format limit"); + } + + int volumeNumber = _volumeNumbers.GetValueOrDefault((group.Name, lane)); + _volumeNumbers[(group.Name, lane)] = volumeNumber + 1; + string name = RenderName(group.Name, lane, volumeNumber, packId); + long pageSize = group.IOPageSize != 0 ? group.IOPageSize : _config.IOPageSize; + AMPRPackVolumeWriter writer = new( + packId, name, _outputDir, Math.Max(_config.PayloadAlignment, pageSize), _config.ChunkAlignment, pageSize, group.MaxPackSize, flags); + Volumes.Add(writer); + _current[(group.Name, lane)] = writer; + return writer; + } + + private string RenderName(string group, int lane, int volume, int packId) + { + string name; + try + { + name = _pattern.Render(group, lane, volume, packId); + } + catch (FormatException exc) + { + throw new AMPRPackException($"invalid pack_pattern: {Core.Util.PythonText.Repr(_config.PackPattern)}", exc); + } + + name = name.Replace('\\', '/'); + // The runtime root is /app0; keep room for it, a slash and NUL within SCE_KERNEL_PATH_MAX (1024). + if (Encoding.UTF8.GetByteCount(name) > 1017) + { + throw new AMPRPackException("pack_pattern produced a name longer than the runtime path limit"); + } + + if (!_names.Add(name)) + { + throw new AMPRPackException($"pack_pattern produced duplicate name: {name}"); + } + + AMPRAssetPath.SafeOutputPath(_outputDir, name); + return name; + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRPackMaintenance.cs b/src/MkPFS.Build/AMPRPack/AMPRPackMaintenance.cs new file mode 100644 index 0000000..99beb0a --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRPackMaintenance.cs @@ -0,0 +1,289 @@ +using MkPFS.Core.AMPR; + +namespace MkPFS.Build.AMPRPack; + +/// Read-only plan of . +/// Packed files. +/// Packed files present under the root. +/// Packed files already absent. +/// Bytes of present files. +/// Relative paths in file-id order. +public sealed record AMPRRemovalPlan(long Files, long PresentFiles, long MissingFiles, long Bytes, IReadOnlyList Paths); + +/// Result of . +/// Files removed. +/// Bytes removed. +/// Empty directories removed. +/// Always : the set was verified before removal. +public sealed record AMPRRemovalResult(long Files, long Bytes, long Directories, bool Verified); + +/// Result of . +/// Written .runtime file. +/// Manifest build id, lower-case hex. +/// Written settings. +public sealed record AMPRRuntimeConfigResult(string Path, string BuildId, AMPRRuntimeSettings Runtime); + +/// +/// Pack-set maintenance that changes files: removing packed sources from /app0 and replacing runtime +/// settings (ampr_pack remove-packed-sources, runtime-config). +/// +public static class AMPRPackMaintenance +{ + private static readonly HashSet ProtectedDirectories = new(StringComparer.Ordinal) { "mods", "save", "sce_module", "sce_sys", "system" }; + + private static readonly HashSet ProtectedNames = new(StringComparer.Ordinal) + { + "ampr_assets.index", "ampr_assets.index.crc", "ampr_assets.index.runtime", "ampr_emu.index", "eboot.bin", "nptitle.dat", "param.sfo", + }; + + private static readonly HashSet ProtectedSuffixes = new(StringComparer.Ordinal) { ".elf", ".prx", ".self", ".sprx" }; + + /// + /// Whether a relative path must never be removed: system and save directories, executables and modules, + /// indexes and pack volumes (Python _removal_path_is_protected). + /// + /// Path relative to /app0. + /// when protected. + public static bool IsProtected(string relative) + { + string[] parts = relative.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0) + { + return false; + } + + string name = parts[^1].ToLowerInvariant(); + int dot = name.LastIndexOf('.'); + // PurePosixPath.suffix: empty for names that only start with a dot. + string suffix = dot > 0 && dot < name.Length - 1 ? name[dot..] : string.Empty; + return ProtectedDirectories.Contains(parts[0].ToLowerInvariant()) + || ProtectedNames.Contains(name) + || ProtectedSuffixes.Contains(suffix) + || (name.StartsWith("ampr_assets-", StringComparison.Ordinal) && name.EndsWith(".pak", StringComparison.Ordinal)); + } + + /// The exact safe source set the manifest packs; nothing is changed (Python packed_source_removal_plan). + /// Manifest path. + /// /app0 source tree. + /// Plan. + /// The manifest packs a protected file or a source is unsafe. + public static AMPRRemovalPlan RemovalPlan(string indexPath, string root) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + root = RequireRoot(root); + List paths = []; + long present = 0; + long missing = 0; + for (int fileId = 1; fileId <= manifest.Files.Count; fileId++) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + continue; + } + + string relative = AMPRAssetPath.Relative(manifest.FilePath(fileId)); + if (IsProtected(relative)) + { + throw new AMPRPackException($"manifest marks a protected game/runtime file as packed: {relative}"); + } + + paths.Add(relative); + if (ValidatedSource(root, relative, record.LogicalSize, allowMissing: true) is null) + { + missing++; + } + else + { + present += (long)record.LogicalSize; + } + } + + return new AMPRRemovalPlan(paths.Count, paths.Count - missing, missing, present, paths); + } + + /// + /// Verify the pack set and every source byte, then permanently delete the sources of packed files + /// (Python remove_packed_sources). Offline verification cannot prove the title never reads a file + /// through an unhooked path; the caller decides that. + /// + /// Manifest path. + /// /app0 source tree. + /// Remove directories left empty, never the root. + /// Counters. + /// A source is missing, protected, unsafe or differs from the packed data. + public static AMPRRemovalResult RemovePackedSources(string indexPath, string root, bool removeEmptyDirs = false) + { + AMPRRemovalPlan initial = RemovalPlan(indexPath, root); + if (initial.MissingFiles > 0) + { + throw new AMPRPackException($"cannot verify before removal: {initial.MissingFiles} packed source files are missing"); + } + + // Check the pack payload and every source byte immediately before the irreversible phase. + AMPRPackTools.Verify(indexPath); + AMPRPackTools.VerifyAgainstRoot(indexPath, root); + + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + root = RequireRoot(root); + List<(string Path, ulong Size)> candidates = []; + for (int fileId = 1; fileId <= manifest.Files.Count; fileId++) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + continue; + } + + string relative = AMPRAssetPath.Relative(manifest.FilePath(fileId)); + if (IsProtected(relative)) + { + throw new AMPRPackException($"refusing to remove protected path: {relative}"); + } + + candidates.Add((ValidatedSource(root, relative, record.LogicalSize, allowMissing: false)!, record.LogicalSize)); + } + + long bytes = 0; + HashSet parents = new(StringComparer.Ordinal); + foreach ((string path, ulong size) in candidates) + { + File.Delete(path); + bytes += (long)size; + parents.Add(Path.GetDirectoryName(path)!); + } + + long removedDirs = 0; + if (removeEmptyDirs) + { + HashSet expanded = new(StringComparer.Ordinal); + foreach (string directory in parents) + { + for (string current = directory; !SamePath(current, root); current = Path.GetDirectoryName(current)!) + { + expanded.Add(current); + } + } + + foreach (string directory in expanded.OrderByDescending(d => d.Split(Path.DirectorySeparatorChar).Length)) + { + try + { + Directory.Delete(directory); + removedDirs++; + } + catch (IOException) + { + // Non-empty directories and directories reused concurrently are kept. + } + catch (UnauthorizedAccessException) + { + } + } + } + + return new AMPRRemovalResult(candidates.Count, bytes, removedDirs, true); + } + + /// Atomically replace <index>.runtime from a TOML [runtime] section without repacking. + /// Manifest path. + /// TOML file with [runtime]. + /// What was written. + /// The configuration has no [runtime]. + public static AMPRRuntimeConfigResult WriteRuntimeConfig(string indexPath, string configPath) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + AMPRRuntimeSettings settings = AMPRPackConfig.Load(configPath).Runtime + ?? throw new AMPRPackException("configuration must contain [runtime]"); + string destination = AMPRPackFormat.RuntimePath(indexPath); + string temporary = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(destination))!, $".runtime-{Guid.NewGuid():N}"); + try + { + using (FileStream handle = new(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + handle.Write(settings.Encode(manifest.BuildId)); + handle.Flush(flushToDisk: true); + } + + File.Move(temporary, destination, overwrite: true); + } + finally + { + File.Delete(temporary); + } + + return new AMPRRuntimeConfigResult(destination, Convert.ToHexStringLower(manifest.BuildId), settings); + } + + private static string RequireRoot(string root) + { + string full = Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)); + if (!Directory.Exists(full)) + { + throw File.Exists(full) + ? new AMPRPackException($"source root is not a directory: {full}") + : new DirectoryNotFoundException($"No such file or directory: '{root}'"); + } + + return full; + } + + // Python _validated_removal_source: a regular, non-link file inside the root with the manifest size. + private static string? ValidatedSource(string root, string relative, ulong expectedSize, bool allowMissing) + { + string candidate = AMPRAssetPath.SafeOutputPath(root, relative); + FileInfo info = new(candidate); + bool isDirectory = Directory.Exists(candidate); + if (!info.Exists && !isDirectory && info.LinkTarget is null) + { + return allowMissing ? null : throw new AMPRPackException($"packed source is missing: {relative}"); + } + + FileSystemInfo entry = isDirectory ? new DirectoryInfo(candidate) : info; + if (entry.LinkTarget is not null || (entry.Attributes & FileAttributes.ReparsePoint) != 0) + { + throw new AMPRPackException($"refusing to remove a symlink/reparse point: {relative}"); + } + + if (isDirectory) + { + throw new AMPRPackException($"packed source is not a regular file: {relative}"); + } + + string parent = Path.GetDirectoryName(Path.GetFullPath(candidate))!; + if (!SamePath(parent, root) && !IsUnder(ResolveDirectory(parent), ResolveDirectory(root))) + { + throw new AMPRPackException($"packed source resolves outside /app0: {relative}"); + } + + return (ulong)info.Length == expectedSize + ? candidate + : throw new AMPRPackException($"packed source size changed for {relative}: manifest={expectedSize}, disk={info.Length}"); + } + + // Directory path with linked components resolved. + private static string ResolveDirectory(string path) + { + string full = Path.GetFullPath(path); + string current = Path.GetPathRoot(full)!; + foreach (string part in full[current.Length..].Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries)) + { + string next = Path.Combine(current, part); + DirectoryInfo info = new(next); + current = info.Exists && info.LinkTarget is not null && info.ResolveLinkTarget(returnFinalTarget: true) is { } target + ? ResolveDirectory(target.FullName) + : next; + } + + return Path.TrimEndingDirectorySeparator(current); + } + + private static StringComparison PathComparison => + OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + private static bool SamePath(string a, string b) => + Path.TrimEndingDirectorySeparator(a).Equals(Path.TrimEndingDirectorySeparator(b), PathComparison); + + private static bool IsUnder(string path, string root) => + SamePath(path, root) || path.StartsWith(Path.TrimEndingDirectorySeparator(root) + Path.DirectorySeparatorChar, PathComparison); +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRPackPattern.cs b/src/MkPFS.Build/AMPRPack/AMPRPackPattern.cs new file mode 100644 index 0000000..1da3aee --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRPackPattern.cs @@ -0,0 +1,521 @@ +using System.Globalization; +using System.Text; +using MkPFS.Core.AMPR; +using MkPFS.Core.Util; + +namespace MkPFS.Build.AMPRPack; + +/// +/// Volume name pattern such as ampr_assets-{id:03d}.pak: a Python str.format string over the fields +/// group (text) and lane, volume, id (integers). Port of ampr_pack +/// _pack_output_glob plus the parts of string.Formatter and the format-spec mini-language it relies on. +/// Float presentation types g/G on integers are not supported (rejected as invalid). +/// +public sealed class AMPRPackPattern +{ + private static readonly string[] AllowedFields = ["group", "lane", "volume", "id"]; + + private readonly List _segments; + + private AMPRPackPattern(string pattern, List segments, string outputGlob) + { + Pattern = pattern; + _segments = segments; + OutputGlob = outputGlob; + } + + /// The pattern text. + public string Pattern { get; } + + /// Glob matching every volume this pattern can produce (fields become *). + public string OutputGlob { get; } + + /// Parse and check a pattern like ampr_pack _pack_output_glob. + /// Pattern. + /// The pattern. + /// The pattern is malformed, uses another field, is empty, or renders an unsafe path. + public static AMPRPackPattern Parse(string pattern) + { + List segments; + try + { + segments = ParseFormat(pattern); + } + catch (FormatException exc) + { + throw new AMPRPackException($"invalid pack_pattern: {PythonText.Repr(pattern)}", exc); + } + + StringBuilder glob = new(); + foreach (Segment segment in segments) + { + if (segment.Field is null) + { + glob.Append(segment.Literal); + continue; + } + + if (!AllowedFields.Contains(segment.Field, StringComparer.Ordinal)) + { + throw new AMPRPackException( + $"pack_pattern uses unsupported field {PythonText.Repr(segment.Field)}; allowed fields are group, lane, volume and id"); + } + + glob.Append('*'); + } + + string outputGlob = glob.ToString().Replace('\\', '/').TrimStart('/'); + if (outputGlob.Length == 0) + { + throw new AMPRPackException("pack_pattern cannot be empty"); + } + + AMPRPackPattern result = new(pattern, segments, outputGlob); + // A concrete rendering catches absolute paths and traversal before any data is written. + try + { + AMPRAssetPath.SafeOutputPath(".", result.Render("g", 0, 0, 0).Replace('\\', '/')); + } + catch (Exception exc) when (exc is ArgumentException or FormatException) + { + throw new AMPRPackException($"unsafe or invalid pack_pattern: {PythonText.Repr(pattern)}", exc); + } + + return result; + } + + /// Python pattern.format(group=..., lane=..., volume=..., id=...). + /// Group name. + /// Lane. + /// Volume within the lane. + /// Pack id. + /// Volume file name. + /// A conversion or format spec is invalid for its value. + public string Render(string group, long lane, long volume, long id) + { + StringBuilder output = new(); + foreach (Segment segment in _segments) + { + if (segment.Field is null) + { + output.Append(segment.Literal); + continue; + } + + object value = segment.Field switch + { + "group" => group, + "lane" => lane, + "volume" => volume, + _ => id, + }; + output.Append(FormatField(value, segment.Conversion, segment.Spec)); + } + + return output.ToString(); + } + + private static string FormatField(object value, char? conversion, string spec) + { + if (conversion is { } c) + { + value = c switch + { + 's' => value is string s ? s : ((long)value).ToString(CultureInfo.InvariantCulture), + 'r' or 'a' => value is string s ? PythonText.Repr(s) : ((long)value).ToString(CultureInfo.InvariantCulture), + _ => throw new FormatException($"Unknown conversion specifier {c}"), + }; + } + + if (spec.Contains('{', StringComparison.Ordinal) || spec.Contains('}', StringComparison.Ordinal)) + { + // Nested replacement fields would need arguments this pattern does not have (Python KeyError). + throw new FormatException("nested replacement field in format spec"); + } + + FormatSpec parsed = FormatSpec.Parse(spec); + return value is string text ? parsed.FormatString(text) : parsed.FormatInteger((long)value); + } + + // string.Formatter().parse: literal text with {{ and }} escapes, then fields "{name[!conv][:spec]}". + private static List ParseFormat(string pattern) + { + List segments = []; + StringBuilder literal = new(); + int i = 0; + while (i < pattern.Length) + { + char c = pattern[i]; + if (c == '}') + { + if (i + 1 < pattern.Length && pattern[i + 1] == '}') + { + literal.Append('}'); + i += 2; + continue; + } + + throw new FormatException("Single '}' encountered in format string"); + } + + if (c != '{') + { + literal.Append(c); + i++; + continue; + } + + if (i + 1 < pattern.Length && pattern[i + 1] == '{') + { + literal.Append('{'); + i += 2; + continue; + } + + if (i + 1 >= pattern.Length) + { + throw new FormatException("Single '{' encountered in format string"); + } + + // Find the matching '}' (format specs may nest braces). + int depth = 1; + int j = i + 1; + while (j < pattern.Length) + { + if (pattern[j] == '{') + { + depth++; + } + else if (pattern[j] == '}' && --depth == 0) + { + break; + } + + j++; + } + + if (j >= pattern.Length) + { + throw new FormatException("expected '}' before end of string"); + } + + segments.Add(new Segment(literal.ToString(), null, null, string.Empty)); + literal.Clear(); + segments.Add(ParseField(pattern[(i + 1)..j])); + i = j + 1; + } + + segments.Add(new Segment(literal.ToString(), null, null, string.Empty)); + return segments; + } + + private static Segment ParseField(string field) + { + int k = 0; + while (k < field.Length) + { + char c = field[k]; + if (c == '{') + { + throw new FormatException("unexpected '{' in field name"); + } + + if (c == '[') + { + while (k < field.Length && field[k] != ']') + { + k++; + } + + continue; + } + + if (c is ':' or '!') + { + break; + } + + k++; + } + + string name = field[..k]; + char? conversion = null; + if (k < field.Length && field[k] == '!') + { + if (k + 1 >= field.Length) + { + throw new FormatException("end of string while looking for conversion specifier"); + } + + conversion = field[k + 1]; + k += 2; + if (k < field.Length && field[k] != ':') + { + throw new FormatException("expected ':' after conversion specifier"); + } + } + + string spec = k < field.Length ? field[(k + 1)..] : string.Empty; + return new Segment(string.Empty, name, conversion, spec); + } + + private sealed record Segment(string Literal, string? Field, char? Conversion, string Spec); + + // Python format-spec mini-language: [[fill]align][sign][z][#][0][width][grouping][.precision][type]. + private sealed record FormatSpec( + char? Fill, char? Align, char Sign, bool Alternate, bool ZeroPad, int Width, char? Grouping, int? Precision, char? Type) + { + public static FormatSpec Parse(string spec) + { + int i = 0; + char? fill = null; + char? align = null; + if (spec.Length >= 2 && spec[1] is '<' or '>' or '=' or '^') + { + fill = spec[0]; + align = spec[1]; + i = 2; + } + else if (spec.Length >= 1 && spec[0] is '<' or '>' or '=' or '^') + { + align = spec[0]; + i = 1; + } + + char sign = '-'; + if (i < spec.Length && spec[i] is '+' or '-' or ' ') + { + sign = spec[i++]; + } + + if (i < spec.Length && spec[i] == 'z') + { + throw new FormatException("Negative zero coercion (z) not allowed in format specifier"); + } + + bool alternate = false; + if (i < spec.Length && spec[i] == '#') + { + alternate = true; + i++; + } + + bool zero = false; + if (i < spec.Length && spec[i] == '0') + { + zero = true; + i++; + } + + int width = ReadNumber(spec, ref i) ?? 0; + char? grouping = null; + if (i < spec.Length && spec[i] is ',' or '_') + { + grouping = spec[i++]; + } + + int? precision = null; + if (i < spec.Length && spec[i] == '.') + { + i++; + precision = ReadNumber(spec, ref i) ?? throw new FormatException("Format specifier missing precision"); + } + + char? type = null; + if (i < spec.Length) + { + type = spec[i++]; + } + + return i == spec.Length + ? new FormatSpec(fill, align, sign, alternate, zero, width, grouping, precision, type) + : throw new FormatException("Invalid format specifier"); + } + + public string FormatString(string value) + { + if (Type is not (null or 's')) + { + throw new FormatException($"Unknown format code '{Type}' for object of type 'str'"); + } + + if (Sign != '-' || Alternate || Align == '=' || Grouping is not null) + { + throw new FormatException("invalid format specifier for a string"); + } + + if (Precision is { } p && p < value.Length) + { + value = value[..p]; + } + + // Since Python 3.10 a leading '0' only sets the fill for strings; the default alignment stays left. + char fill = Fill ?? (ZeroPad ? '0' : ' '); + return Pad(string.Empty, value, fill, Align ?? '<'); + } + + public string FormatInteger(long value) + { + string digits; + string prefix = string.Empty; + ulong magnitude = value < 0 ? (ulong)(-(value + 1)) + 1 : (ulong)value; + switch (Type) + { + case null or 'd' or 'n': + if (Type == 'n' && Grouping is not null) + { + throw new FormatException("Cannot specify ',' with 'n'."); + } + + digits = magnitude.ToString(CultureInfo.InvariantCulture); + break; + case 'b' or 'o' or 'x' or 'X': + if (Grouping == ',') + { + throw new FormatException($"Cannot specify ',' with '{Type}'."); + } + + digits = Type switch + { + 'b' => Convert.ToString((long)magnitude, 2), + 'o' => Convert.ToString((long)magnitude, 8), + 'x' => magnitude.ToString("x", CultureInfo.InvariantCulture), + _ => magnitude.ToString("X", CultureInfo.InvariantCulture), + }; + if (Alternate) + { + prefix = Type switch { 'b' => "0b", 'o' => "0o", 'x' => "0x", _ => "0X" }; + } + + break; + case 'c': + if (Sign != '-' || Alternate || Grouping is not null || value is < 0 or > 0x10FFFF) + { + throw new FormatException("invalid format specifier for 'c'"); + } + + return FormatChar((int)value); + case 'f' or 'F' or 'e' or 'E' or '%': + return FormatFloat(value); + default: + throw new FormatException($"Unknown format code '{Type}' for object of type 'int'"); + } + + if (Precision is not null) + { + throw new FormatException("Precision not allowed in integer format specifier"); + } + + string signText = value < 0 ? "-" : Sign == '+' ? "+" : Sign == ' ' ? " " : string.Empty; + return PadNumber(signText + prefix, digits, Grouping is null ? 0 : Type is 'b' or 'o' or 'x' or 'X' ? 4 : 3); + } + + private string FormatChar(int value) + { + string text = char.ConvertFromUtf32(value); + return Pad(string.Empty, text, Fill ?? (ZeroPad ? '0' : ' '), Align ?? (ZeroPad ? '=' : '>')); + } + + private string FormatFloat(long value) + { + int precision = Precision ?? 6; + double number = Type == '%' ? value * 100.0 : value; + string digits = Type switch + { + 'e' or 'E' => FormatExponent(Math.Abs(number), precision, Type == 'E'), + _ => Math.Abs(number).ToString("F" + precision.ToString(CultureInfo.InvariantCulture), CultureInfo.InvariantCulture), + }; + if (Alternate && precision == 0) + { + digits = Type is 'e' or 'E' ? digits.Insert(1, ".") : digits + "."; + } + + string suffix = Type == '%' ? "%" : string.Empty; + string signText = number < 0 ? "-" : Sign == '+' ? "+" : Sign == ' ' ? " " : string.Empty; + if (Grouping is not null && Type is 'e' or 'E') + { + return PadNumber(signText, digits + suffix, 0); + } + + int dot = digits.IndexOf('.', StringComparison.Ordinal); + string whole = dot < 0 ? digits : digits[..dot]; + string fraction = dot < 0 ? string.Empty : digits[dot..]; + return PadNumber(signText, whole, Grouping is null ? 0 : 3, fraction + suffix); + } + + private static string FormatExponent(double value, int precision, bool upper) + { + string text = value.ToString((upper ? "E" : "e") + precision.ToString(CultureInfo.InvariantCulture), CultureInfo.InvariantCulture); + // .NET writes e+006; Python writes e+06. + int e = text.IndexOfAny(['e', 'E']); + int exponent = int.Parse(text.AsSpan(e + 1), NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + return text[..(e + 1)] + (exponent < 0 ? "-" : "+") + Math.Abs(exponent).ToString("00", CultureInfo.InvariantCulture); + } + + // Numbers align right by default; '=' (or a leading '0' without an explicit alignment) pads between the + // sign/prefix and the digits, and zero padding is grouped like the digits. + private string PadNumber(string head, string digits, int groupSize, string tail = "") + { + char align = Align ?? (ZeroPad ? '=' : '>'); + char fill = Fill ?? (ZeroPad ? '0' : ' '); + if (align == '=' && fill == '0' && Fill is null && groupSize > 0) + { + string grouped = Group(digits, groupSize); + while (head.Length + grouped.Length + tail.Length < Width) + { + digits = "0" + digits; + grouped = Group(digits, groupSize); + } + + return head + grouped + tail; + } + + string body = (groupSize > 0 ? Group(digits, groupSize) : digits) + tail; + return Pad(head, body, fill, align); + } + + private string Group(string digits, int size) + { + StringBuilder builder = new(); + for (int k = 0; k < digits.Length; k++) + { + if (k > 0 && (digits.Length - k) % size == 0) + { + builder.Append(Grouping); + } + + builder.Append(digits[k]); + } + + return builder.ToString(); + } + + private string Pad(string head, string body, char fill, char align) + { + int padding = Width - head.Length - body.Length; + if (padding <= 0) + { + return head + body; + } + + return align switch + { + '<' => head + body + new string(fill, padding), + '^' => new string(fill, padding / 2) + head + body + new string(fill, padding - (padding / 2)), + '=' => head + new string(fill, padding) + body, + _ => new string(fill, padding) + head + body, + }; + } + + private static int? ReadNumber(string spec, ref int i) + { + int start = i; + while (i < spec.Length && char.IsAsciiDigit(spec[i])) + { + i++; + } + + return i == start ? null : int.Parse(spec.AsSpan(start, i - start), CultureInfo.InvariantCulture); + } + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRPackPlanner.cs b/src/MkPFS.Build/AMPRPack/AMPRPackPlanner.cs new file mode 100644 index 0000000..6d9a50b --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRPackPlanner.cs @@ -0,0 +1,348 @@ +using MkPFS.Core.AMPR; +using MkPFS.Core.Compression; +using MkPFS.Core.Util; + +namespace MkPFS.Build.AMPRPack; + +/// +/// Chooses a rule, source and lane for every AMPRIDX3 file (Python plan_files, _source_path, +/// _assign_lanes, _auto_loose_decision). +/// +public static class AMPRPackPlanner +{ + /// Plan every file. + /// AMPRIDX3 rows (file id = row + 1). + /// Resolved /app0 source directory. + /// Configuration. + /// Counters to update. + /// CLI include globs; when given, other files stay loose. + /// CLI exclude globs. + /// Leave missing sources loose instead of failing. + /// Progress callback. + /// One planned file per row, and warnings. + public static (List Files, List Warnings) Plan( + IReadOnlyList entries, + string root, + AMPRPackConfig config, + AMPRBuildStats stats, + IReadOnlyList includePatterns, + IReadOnlyList excludePatterns, + bool allowMissing, + Action? progress = null) + { + List candidates = []; + List warnings = []; + string indexName = config.IndexName.Replace('\\', '/').TrimStart('/'); + string packOutputGlob = AMPRPackPattern.Parse(config.PackPattern).OutputGlob; + using LZ4Encoder encoder = new(); + for (int fileId = 1; fileId <= entries.Count; fileId++) + { + AMPRIndexEntry entry = entries[fileId - 1]; + string relative = AMPRAssetPath.Relative(entry.Path); + progress?.Invoke(new AMPRBuildProgress("planning", fileId - 1, entries.Count, 0, 0, relative)); + AMPRRule rule = config.SelectRule(relative); + if (includePatterns.Count > 0 && !AMPRGlob.Matches(relative, includePatterns)) + { + rule = AMPRRule.Loose(rule.BlockShift); + } + + if (excludePatterns.Count > 0 && AMPRGlob.Matches(relative, excludePatterns)) + { + rule = AMPRRule.Loose(rule.BlockShift); + } + + // Never consume an index or pack left by an earlier build, including custom nested pack names. + string normalized = relative.Replace('\\', '/').TrimStart('/'); + if (normalized == indexName || normalized == indexName + ".crc" || normalized == indexName + ".runtime" + || AMPRGlob.Matches(normalized, [packOutputGlob])) + { + rule = AMPRRule.Loose(rule.BlockShift); + } + + string source = Path.Combine([root, .. relative.Split('/')]); + if (rule.Action == "loose") + { + candidates.Add(new AMPRSelectedFile(fileId, entry, source, relative, rule, (long)entry.Size, entry.MTime, 0)); + continue; + } + + string resolved; + try + { + (source, resolved) = SourcePath(root, relative); + } + catch (Exception exc) when (exc is FileNotFoundException or DirectoryNotFoundException or ArgumentException) + { + if (!allowMissing) + { + throw new AMPRPackException($"source is missing or unsafe for file id {fileId}: {relative}", exc); + } + + warnings.Add($"file id {fileId} left loose because source is missing: {relative}"); + candidates.Add(new AMPRSelectedFile(fileId, entry, source, relative, AMPRRule.Loose(rule.BlockShift), (long)entry.Size, entry.MTime, 0)); + continue; + } + + FileInfo info = new(resolved); + if (!info.Exists) + { + throw new AMPRPackException($"source is not a regular file: {relative}"); + } + + if (config.ValidateIndexMetadata && (ulong)info.Length != entry.Size) + { + throw new AMPRPackException($"AMPRIDX3 size mismatch for {relative}: index={entry.Size}, disk={info.Length}"); + } + + AMPRSelectedFile candidate = new(fileId, entry, source, relative, rule, info.Length, UnixSeconds(info.LastWriteTimeUtc), 0); + AMPRGroupConfig group = config.Groups[rule.Group]; + AutoLooseDecision autoLoose = DecideAutoLoose(candidate, config, encoder, group.IOPageSize != 0 ? group.IOPageSize : config.IOPageSize); + stats.AutoLooseSampledBytes += autoLoose.SampledBytes; + if (autoLoose.UseLoose) + { + string details = $"{relative} size={candidate.Size} sampled={autoLoose.SampledBytes} " + + $"saving={Percent(autoLoose.SavingsRatio)} rawBlocks={Percent(autoLoose.RawRatio)}; "; + if (config.SelfContained) + { + warnings.Add("self-contained: auto-loose suppressed for " + details + "incompressible blocks will be stored RAW"); + } + else + { + stats.FilesAutoLoose++; + stats.AutoLooseLogicalBytes += candidate.Size; + warnings.Add("auto-loose: " + details + "use force_pack=true or self_contained=true to override"); + rule = rule with { Action = "loose", Hot = false, Streaming = false, Layout = "mixed" }; + } + } + + candidates.Add(candidate with { Rule = rule }); + } + + Dictionary lanes = AssignLanes(candidates, config); + List result = [.. candidates.Select(c => c with { Lane = lanes.GetValueOrDefault(c.FileId) })]; + if (config.RequiredPacked.Count > 0) + { + List missing = [.. result + .Where(item => AMPRGlob.Matches(item.Relative, config.RequiredPacked) && item.Rule.Action == "loose") + .Select(item => item.Relative)]; + if (missing.Count > 0) + { + string preview = string.Join(", ", missing.Take(8)); + string suffix = missing.Count <= 8 ? string.Empty : $" (+{missing.Count - 8} more)"; + throw new AMPRPackException($"required_packed matched files that would remain loose: {preview}{suffix}"); + } + } + + progress?.Invoke(new AMPRBuildProgress("planning", entries.Count, entries.Count, 0, 0)); + return (result, warnings); + } + + /// + /// Sample positions for auto-loose: all blocks when few, the middle one for one sample, else an even + /// integer interpolation that includes the first and last block (Python _sample_block_indices). + /// + /// Blocks in the file. + /// Samples wanted. + /// Sorted distinct block indices. + public static List SampleBlockIndices(long totalBlocks, long sampleBlocks) + { + if (totalBlocks <= 0 || sampleBlocks <= 0) + { + return []; + } + + if (totalBlocks <= sampleBlocks) + { + return [.. Enumerable.Range(0, (int)totalBlocks).Select(i => (long)i)]; + } + + if (sampleBlocks == 1) + { + return [totalBlocks / 2]; + } + + SortedSet indices = []; + for (long sample = 0; sample < sampleBlocks; sample++) + { + indices.Add(checked(sample * (totalBlocks - 1)) / (sampleBlocks - 1)); + } + + return [.. indices]; + } + + /// Unix seconds truncated toward zero, like Python int(st_mtime). + /// UTC time. + /// Seconds. + internal static long UnixSeconds(DateTime utc) => (utc.Ticks - DateTime.UnixEpoch.Ticks) / TimeSpan.TicksPerSecond; + + /// Python f"{value:.2%}". + private static string Percent(double value) => PythonText.FormatFixed(value * 100.0, 2) + "%"; + + // Python _source_path: a safe relative path whose resolved target stays under the resolved root. + private static (string Source, string Resolved) SourcePath(string root, string relative) + { + string candidate = AMPRAssetPath.SafeOutputPath(root, relative); + string rootReal = RealPath(root); + if (!File.Exists(candidate) && !Directory.Exists(candidate)) + { + throw new FileNotFoundException("source is missing", candidate); + } + + string resolved = RealPath(candidate); + StringComparison comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + string prefix = Path.EndsInDirectorySeparator(rootReal) ? rootReal : rootReal + Path.DirectorySeparatorChar; + return resolved.Equals(rootReal, comparison) || resolved.StartsWith(prefix, comparison) + ? (candidate, resolved) + : throw new AMPRPackException($"source symlink escapes root: {relative}"); + } + + // Absolute path with every symlink or junction component resolved (Path.resolve()). + private static string RealPath(string path, int depth = 0) + { + if (depth > 40) + { + throw new AMPRPackException($"too many levels of symbolic links: {path}"); + } + + string full = Path.GetFullPath(path); + string current = Path.GetPathRoot(full)!; + foreach (string part in full[current.Length..].Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries)) + { + string next = Path.Combine(current, part); + FileSystemInfo info = Directory.Exists(next) ? new DirectoryInfo(next) : new FileInfo(next); + if (info.Exists && info.LinkTarget is not null && info.ResolveLinkTarget(returnFinalTarget: true) is { } target) + { + next = RealPath(target.FullName, depth + 1); + } + + current = next; + } + + return Path.TrimEndingDirectorySeparator(current); + } + + private static Dictionary AssignLanes(List candidates, AMPRPackConfig config) + { + Dictionary> byGroup = new(StringComparer.Ordinal); + foreach (AMPRSelectedFile item in candidates.Where(c => c.Rule.Action != "loose")) + { + if (!byGroup.TryGetValue(item.Rule.Group, out List? files)) + { + byGroup[item.Rule.Group] = files = []; + } + + files.Add(item); + } + + Dictionary result = []; + foreach ((string groupName, List files) in byGroup) + { + AMPRGroupConfig group = config.Groups[groupName]; + int packCount = (int)group.PackCount; + if (group.Assignment == "balanced") + { + // Largest first, then lower-cased path by code point, then file id; each goes to the lightest lane. + long[] loads = new long[packCount]; + IEnumerable ordered = files + .OrderBy(f => -f.Size) + .ThenBy(f => f.Relative.ToLowerInvariant(), PythonCodePointComparer.Instance) + .ThenBy(f => f.FileId); + foreach (AMPRSelectedFile file in ordered) + { + int lane = 0; + for (int candidate = 1; candidate < packCount; candidate++) + { + if (loads[candidate] < loads[lane]) + { + lane = candidate; + } + } + + result[file.FileId] = lane; + loads[lane] += file.Size; + } + } + else if (group.Assignment == "hash") + { + foreach (AMPRSelectedFile file in files) + { + result[file.FileId] = (int)(AMPRAssetPath.Fnv1a64(AMPRAssetPath.AsciiFoldPathBytes(file.Relative)) % (ulong)packCount); + } + } + else + { + foreach (AMPRSelectedFile file in files) + { + result[file.FileId] = (file.FileId - 1) % packCount; + } + } + } + + return result; + } + + private static AutoLooseDecision DecideAutoLoose(AMPRSelectedFile item, AMPRPackConfig config, LZ4Encoder encoder, long ioPageSize) + { + AMPRRule rule = item.Rule; + if (!config.AutoLooseLargeFiles + || rule.Action != "compress" + || rule.ForcePack + || (rule.Hot && !config.AutoLooseHotFiles) + || item.Size < config.AutoLooseMinFileSize + || item.Size == 0) + { + return new AutoLooseDecision(false, 0, 0, 0, 0); + } + + long blockSize = 1L << rule.BlockShift; + long totalBlocks = (item.Size + blockSize - 1) / blockSize; + long byteLimitedBlocks = Math.Max(1, config.AutoLooseSampleBytes / blockSize); + List indices = SampleBlockIndices(totalBlocks, Math.Min(config.AutoLooseSampleBlocks, byteLimitedBlocks)); + long sampledBytes = 0; + long storedBytes = 0; + long rawBlocks = 0; + using (FileStream handle = File.OpenRead(item.Source)) + { + (long Length, DateTime Written) before = FileStamp(item.Source); + foreach (long blockIndex in indices) + { + long offset = blockIndex * blockSize; + int expected = (int)Math.Min(blockSize, item.Size - offset); + byte[] raw = new byte[expected]; + handle.Seek(offset, SeekOrigin.Begin); + if (handle.ReadAtLeast(raw, expected, throwOnEndOfStream: false) != expected) + { + throw new AMPRPackException($"source was truncated while sampling compression: {item.Relative}"); + } + + AMPRCompressedBlock block = AMPRBlockCompressor.Compress(encoder, raw, rule, ioPageSize); + sampledBytes += raw.Length; + storedBytes += block.Stored.Length; + rawBlocks += block.Codec == AMPRPackFormat.ChunkCodecRaw ? 1 : 0; + } + + if (FileStamp(item.Source) != before) + { + throw new AMPRPackException($"source changed while sampling compression: {item.Relative}"); + } + } + + AutoLooseDecision decision = new(false, sampledBytes, storedBytes, rawBlocks, indices.Count); + bool useLoose = decision.SavingsRatio < config.AutoLooseMinSavingsRatio || decision.RawRatio >= config.AutoLooseMaxRawRatio; + return decision with { UseLoose = useLoose }; + } + + /// Size and write time; Python also compares inode and device, which .NET does not expose portably. + internal static (long Length, DateTime Written) FileStamp(string path) + { + FileInfo info = new(path); + return (info.Length, info.LastWriteTimeUtc); + } + + private sealed record AutoLooseDecision(bool UseLoose, long SampledBytes, long StoredBytes, long RawBlocks, long SampledBlocks) + { + public double SavingsRatio => SampledBytes == 0 ? 0.0 : 1.0 - ((double)StoredBytes / SampledBytes); + + public double RawRatio => SampledBlocks == 0 ? 0.0 : (double)RawBlocks / SampledBlocks; + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRProfiler.cs b/src/MkPFS.Build/AMPRPack/AMPRProfiler.cs new file mode 100644 index 0000000..48fbd1a --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRProfiler.cs @@ -0,0 +1,2696 @@ +using System.Globalization; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using MkPFS.Core.AMPR; +using MkPFS.Core.Compression; +using MkPFS.Core.Util; + +namespace MkPFS.Build.AMPRPack; + +/// One trace run: the APR journal and the index it was recorded with (Python TraceSpec). +/// Run name, used in the report and in warnings. +/// ampr_commands.bin. +/// The run's ampr_emu.index. +public sealed record AMPRTraceSpec(string Name, string Commands, string Index); + +/// Options of ampr_pack_profile.py generate (Python ProfileOptions), with its defaults. +public sealed class AMPRProfileOptions +{ + /// Profile name. + public required string Name { get; set; } + + /// --io-page-size. + public long IOPageSize { get; set; } = AMPRProfiler.DefaultIOPage; + + /// --pack-index-budget. + public long IndexBudget { get; set; } = 96L * 1024 * 1024; + + /// --cache-candidates, ascending. + public IReadOnlyList CacheCandidates { get; set; } = AMPRProfiler.DefaultCacheCandidates; + + /// --lanes; 0 picks 1 to 4. + public int Lanes { get; set; } + + /// --workers written to [pack]. + public int Workers { get; set; } = 8; + + /// --runtime-workers; 0 picks 2 to 8 from the trace. + public int RuntimeWorkers { get; set; } + + /// --strategy: conservative, balanced or aggressive. + public string Strategy { get; set; } = "balanced"; + + /// --pattern-mode: exact, hybrid or directory. + public string PatternMode { get; set; } = "exact"; + + /// --min-reads. + public int MinReads { get; set; } = 1; + + /// --min-requested-bytes. + public long MinRequestedBytes { get; set; } = 1; + + /// --max-rule-files. + public int MaxRuleFiles { get; set; } = 256; + + /// --externalize-paths; 0 keeps every path inline. + public int ExternalizePaths { get; set; } = 128; + + /// --generalize-coverage. + public double GeneralizeCoverage { get; set; } = 0.80; + + /// --generalize-min-files. + public int GeneralizeMinFiles { get; set; } = 4; + + /// --cache-sim / --no-cache-sim. + public bool CacheSimulation { get; set; } = true; + + /// --cache-max-touches; 0 simulates every touch. + public long CacheMaxTouches { get; set; } = AMPRProfiler.DefaultCacheMaxTouches; + + /// --content-root for LZ4 sampling, or . + public string? ContentRoot { get; set; } + + /// --sample-budget; 0 disables sampling. + public long SampleBudget { get; set; } + + /// --sample-blocks-per-file. + public int SampleBlocksPerFile { get; set; } = 4; + + /// --sample-mode: hc or fast. + public string SampleMode { get; set; } = "hc"; + + /// --sample-level. + public int SampleLevel { get; set; } = 12; + + /// --sample-acceleration. + public int SampleAcceleration { get; set; } = 1; + + /// Python _validate_options. + /// An option is out of range. + public void Validate() + { + if (IOPageSize < 4096 || (IOPageSize & (IOPageSize - 1)) != 0) + { + throw new AMPRPackException("io-page-size must be a power of two and at least 4 KiB"); + } + + string? error = this switch + { + { IndexBudget: < 1024 * 1024 } => "pack-index-budget must be at least 1 MiB", + { Lanes: < 0 or > 64 } => "lanes must be between 0 and 64", + { Workers: < 1 or > 256 } => "workers must be between 1 and 256", + { RuntimeWorkers: < 0 or > 16 } => "runtime-workers must be between 0 and 16", + { MinReads: < 1 } => "min-reads must be at least 1", + { MaxRuleFiles: < 1 } => "max-rule-files must be at least 1", + { ExternalizePaths: < 0 } => "externalize-paths must be non-negative", + { GeneralizeCoverage: < 0.0 or > 1.0 } => "generalize-coverage must be in [0,1]", + { GeneralizeMinFiles: < 2 } => "generalize-min-files must be at least 2", + _ => null, + }; + if (error is not null) + { + throw new AMPRPackException(error); + } + } +} + +/// Per-trace statistics (Python TraceSummary). +public sealed class AMPRTraceSummary +{ + /// Run name. + public required string Name { get; init; } + + /// Journal path. + public required string Commands { get; init; } + + /// Index path. + public required string Index { get; init; } + + /// SHA-256 of the journal, lowercase hex. + public required string CommandSha256 { get; init; } + + /// SHA-256 of the index, lowercase hex. + public required string IndexSha256 { get; init; } + + /// Journal records read. + public long Records { get; init; } + + /// APR reads kept. + public long Reads { get; init; } + + /// Parser warnings. + public required List Warnings { get; init; } + + /// Records whose commands could not all be decoded. + public long DecodeErrors { get; init; } + + /// Monotonic time between the first and last record. + public long DurationNs { get; init; } +} + +/// Result of (Python ProfileResult). +public sealed class AMPRProfileResult +{ + internal AMPRProfileResult() + { + } + + /// Profile name. + public required string Name { get; init; } + + /// Trace runs. + public required List Traces { get; init; } + + /// Observed files and what to do with each, by path. + internal List Recommendations { get; init; } = []; + + /// Observed index rows, by relative path, in first-read order. + internal Dictionary IndexEntries { get; init; } = []; + + internal List CacheSimulations { get; init; } = []; + + /// Projected resident size of the pack manifest. + public long ProjectedIndexBytes { get; init; } + + /// The index budget the block sizes were fitted to. + public long IndexBudget { get; init; } + + /// Recommended decoded cache. + public long RecommendedCacheBytes { get; init; } + + /// Recommended physical-page cache. + public long RecommendedPhysicalCacheBytes { get; init; } + + /// Recommended runtime workers. + public int RecommendedRuntimeWorkers { get; init; } + + /// Recommended latency-reserved workers. + public int RecommendedLatencyReserveWorkers { get; init; } + + /// Recommended AMPR internal pool. + public long RecommendedPoolBytes { get; init; } + + /// Warnings from the traces and the budget. + public required List Warnings { get; init; } + + internal List<(string Name, int PackCount, long MaxPackSize)> Groups { get; init; } = []; + + /// Observed files that will be packed (not loose). + public int PackedFiles => Recommendations.Count(r => r.Action != "loose"); + + /// Total size of the observed files that will be packed. + public long PackedFileBytes => Recommendations.Where(r => r.Action != "loose").Sum(r => r.Metrics.FileSize); + + /// Observed files. + public int ObservedFiles => Recommendations.Count; +} + +/// +/// Port of ampr_emu tools/ampr_pack_profile.py 4.1: turns APR traces recorded by the debug +/// emulator build into an ampr_pack TOML profile. Files the traces show the game reading through APR are +/// packed with block sizes and layouts fitted to the observed reads; every other file stays loose. The TOML, the +/// Markdown report, the metrics JSON and the runtime header match the Python output. +/// +public static class AMPRProfiler +{ + /// Upstream tool version written into the outputs. + public const string ToolVersion = "4.1"; + + /// Default --io-page-size. + public const long DefaultIOPage = 64 * 1024; + + /// Default --cache-max-touches. + public const long DefaultCacheMaxTouches = 5_000_000; + + /// Default --cache-candidates. + public static readonly IReadOnlyList DefaultCacheCandidates = [32L << 20, 64L << 20, 128L << 20, 256L << 20, 512L << 20]; + + private const int ChunkRecordBytes = 12; + private const int FileRecordBytes = 48; + private const int PackRecordBytes = 32; + private const long RuntimePipelineSlots = 32; + private const long RuntimePipelineIOBytes = 2 * 1024 * 1024; + private const long RuntimeWorkerScratchBytes = 1 * 1024 * 1024; + private const long RuntimePostCacheReserveBytes = 32 * 1024 * 1024; + private const long K64 = 64 * 1024; + + private static readonly long[] BlockSizes = [16 * 1024, 32 * 1024, 64 * 1024, 128 * 1024, 256 * 1024, 512 * 1024, 1024 * 1024]; + private static readonly HashSet KnownLooseBasenames = + ["eboot.bin", "ampr_emu.index", "ampr_assets.index", "ampr_assets.index.crc", "ampr_assets.index.runtime", "param.sfo", "nptitle.dat"]; + private static readonly HashSet KnownLooseSuffixes = [".prx", ".sprx", ".self", ".elf"]; + private static readonly HashSet KnownStreamStoreSuffixes = + [".bik", ".bk2", ".mp4", ".m4v", ".webm", ".m2v", ".mpg", ".mpeg", ".avi", ".wem", ".opus", ".ogg", ".mp3", ".aac", ".flac"]; + + private static readonly string[] StandardLoosePatterns = + [ + "sce_module/**", "system/**", "mods/**", "save/**", "**/*.prx", "**/*.sprx", "eboot.bin", "ampr_emu.index", + "ampr_assets.index", "ampr_assets.index.crc", "ampr_assets.index.runtime", "ampr_assets-*.pak", + ]; + + internal sealed record ReadEvent( + int TraceOrdinal, long Sequence, int CommandOrdinal, long MonotonicNs, long Priority, string Path, string Relative, + int FileId, long FileSize, long Offset, long Length) + { + public long End => Offset + Length; + } + + internal sealed record CandidateMetrics( + long BlockSize, long TouchedBytes, long TotalBlockTouches, long UniqueBlocks, double Amplification, double AverageBlocksPerRead, + double RepeatRatio, double OffsetAlignmentRatio, double LengthAlignmentRatio, long MetadataBytes, double LocalScore); + + internal sealed class FileMetrics + { + public required string Path { get; init; } + public required string Relative { get; init; } + public long FileSize { get; init; } + public int TraceCount { get; init; } + public required List Events { get; init; } + public int ReadCount { get; init; } + public long RequestedBytes { get; init; } + public long UniqueRequestedBytes { get; init; } + public double CoverageRatio { get; init; } + public double P50 { get; init; } + public double P75 { get; init; } + public double P90 { get; init; } + public double P95 { get; init; } + public double P99 { get; init; } + public double Tiny4kRatio { get; init; } + public double Small16kRatio { get; init; } + public double Small64kRatio { get; init; } + public double Large256kRatio { get; init; } + public double ExactSequentialRatio { get; init; } + public double NearSequentialRatio { get; init; } + public double RandomSeekRatio { get; init; } + public required Dictionary Candidates { get; init; } + public double Confidence { get; init; } + public required string ConfidenceLabel { get; init; } + } + + internal sealed class FileRecommendation + { + public required FileMetrics Metrics { get; init; } + public required string Action { get; set; } + public required string Layout { get; set; } + public long BlockSize { get; set; } + public bool Hot { get; set; } + public required string Group { get; set; } + public required string Reason { get; set; } + public double? SampledRatio { get; set; } + public long SampledBytes { get; set; } + + public ProfileKey Key => new(Action, Layout, BlockSize, Hot, Group); + + public long MetadataBytes => Action == "loose" ? 0 : CeilDiv(Metrics.FileSize, BlockSize) * ChunkRecordBytes; + } + + // Python tuple (action, layout, block_size, hot, group), compared element by element. + internal readonly record struct ProfileKey(string Action, string Layout, long BlockSize, bool Hot, string Group) : IComparable + { + public int CompareTo(ProfileKey other) + { + int c = PythonCodePointComparer.Instance.Compare(Action, other.Action); + if (c == 0) + { + c = PythonCodePointComparer.Instance.Compare(Layout, other.Layout); + } + + if (c == 0) + { + c = BlockSize.CompareTo(other.BlockSize); + } + + if (c == 0) + { + c = Hot.CompareTo(other.Hot); + } + + return c != 0 ? c : PythonCodePointComparer.Instance.Compare(Group, other.Group); + } + } + + internal sealed record CacheSimulation( + long Capacity, long RequestedBytes, long HitBytes, long Misses, long Hits, long SampledTouches, long TotalTouches, long SampledWindows) + { + public double HitRatio => RequestedBytes != 0 ? (double)HitBytes / RequestedBytes : 0.0; + } + + private sealed record RuleEmission(ProfileKey Key, IReadOnlyList Include, IReadOnlyList Comments); + + /// + /// Trace pairs under a folder (Python discover_trace_pairs): the folder itself when it holds + /// ampr_commands.bin and ampr_emu.index, otherwise every subfolder that does, in path order. + /// + /// Folder. + /// Trace runs. + /// is a file. + public static List DiscoverTracePairs(string root) + { + if (File.Exists(root)) + { + throw new AMPRPackException($"expected a directory, got file: {root}"); + } + + string commands = Path.Combine(root, "ampr_commands.bin"); + string index = Path.Combine(root, "ampr_emu.index"); + if (File.Exists(commands) && File.Exists(index)) + { + string name = Path.GetFileName(Path.TrimEndingDirectorySeparator(root)); + return [new AMPRTraceSpec(name.Length > 0 ? name : "trace", commands, index)]; + } + + if (!Directory.Exists(root)) + { + return []; + } + + List pairs = []; + // Like Path.rglob: hidden and system files count too (copies off the console can carry those attributes), + // and names match with the platform's case rules. + EnumerationOptions options = new() { RecurseSubdirectories = true, IgnoreInaccessible = true, AttributesToSkip = 0, MatchCasing = MatchCasing.PlatformDefault }; + List found = [.. Directory.EnumerateFiles(root, "ampr_commands.bin", options) + .Select(f => Path.GetRelativePath(root, f).Split(Path.DirectorySeparatorChar))]; + found.Sort(ComparePathParts); + foreach (string[] parts in found) + { + string file = Path.Combine([root, .. parts]); + string sibling = Path.Combine(Path.GetDirectoryName(file)!, "ampr_emu.index"); + if (File.Exists(sibling)) + { + string name = string.Join('-', parts[..^1]); + pairs.Add(new AMPRTraceSpec(name.Length > 0 ? name : Path.GetFileName(Path.GetDirectoryName(file)!), file, sibling)); + } + } + + return pairs; + } + + /// Analyse the traces and choose what to pack (Python build_profile). + /// Trace runs, in order. + /// Options. + /// The profile. + /// A journal is not a supported AMPRCMD1 file. + public static AMPRProfileResult Build(IReadOnlyList traces, AMPRProfileOptions options) + { + (List events, List summaries, Dictionary indexEntries, List warnings) = + LoadTraceEvents(traces); + List files = BuildFileMetrics(events); + List recommendations = InitialRecommendations(files, options); + SampleCompressibility(recommendations, options, warnings); + int provisionalLanes = ChooseLanes(recommendations, options.Lanes); + int provisionalPackCount = BuildGroups(recommendations, provisionalLanes).Sum(g => g.PackCount); + long mandatoryIndexBytes = EstimateIndexBytes([], provisionalPackCount, indexEntries); + long chunkBudget = Math.Max(0, options.IndexBudget - mandatoryIndexBytes); + EnforceIndexBudget(recommendations, chunkBudget, options.Strategy); + int lanes = ChooseLanes(recommendations, options.Lanes); + List<(string Name, int PackCount, long MaxPackSize)> groups = BuildGroups(recommendations, lanes); + int packCount = groups.Sum(g => g.PackCount); + long projectedIndex = EstimateIndexBytes(recommendations, packCount, indexEntries); + if (mandatoryIndexBytes > options.IndexBudget) + { + warnings.Add($"mandatory AMPRPAK4 file/string tables {HumanSize(mandatoryIndexBytes)} exceed index budget " + + $"{HumanSize(options.IndexBudget)} before any packed chunks"); + } + + if (projectedIndex > options.IndexBudget) + { + warnings.Add($"projected pack index {HumanSize(projectedIndex)} still exceeds budget {HumanSize(options.IndexBudget)}"); + } + + Dictionary byPath = []; + foreach (FileRecommendation recommendation in recommendations) + { + byPath[recommendation.Metrics.Relative] = recommendation; + } + + List simulations = options.CacheSimulation ? SimulateCache(events, byPath, options.CacheCandidates, options.CacheMaxTouches) : []; + long cacheSize = ChooseCacheSize(simulations); + long physicalPageCache = ChoosePhysicalPageCacheSize(recommendations); + int runtimeWorkers = ChooseRuntimeWorkers(events, recommendations, options.RuntimeWorkers); + int latencyReserveWorkers = ChooseLatencyReserveWorkers(events, recommendations, runtimeWorkers); + long amprIndexBytes = summaries.Where(s => s.Index.Length > 0).Select(s => new FileInfo(s.Index).Length).DefaultIfEmpty(0).Max(); + long poolSize = RoundPoolSize(projectedIndex + amprIndexBytes + cacheSize + physicalPageCache + (RuntimePipelineSlots * RuntimePipelineIOBytes) + + (runtimeWorkers * RuntimeWorkerScratchBytes) + RuntimePostCacheReserveBytes); + return new AMPRProfileResult + { + Name = options.Name, + Traces = summaries, + Recommendations = recommendations, + IndexEntries = indexEntries, + ProjectedIndexBytes = projectedIndex, + IndexBudget = options.IndexBudget, + CacheSimulations = simulations, + RecommendedCacheBytes = cacheSize, + RecommendedPhysicalCacheBytes = physicalPageCache, + RecommendedRuntimeWorkers = runtimeWorkers, + RecommendedLatencyReserveWorkers = latencyReserveWorkers, + RecommendedPoolBytes = poolSize, + Warnings = warnings, + Groups = groups, + }; + } + + /// + /// The TOML profile (Python render_toml). With , exact include lists longer + /// than go to <stem>.rule-NNN.include.txt + /// sidecars (added to the dictionary) referenced by include_from; without it every path is inline. + /// + /// Profile. + /// Options it was built with. + /// Receives sidecar names and contents, or . + /// Sidecar name prefix (the TOML file name without extension). + /// TOML text. + public static string RenderToml(AMPRProfileResult result, AMPRProfileOptions options, Dictionary? externalLists = null, string profileStem = "profile") + { + List emissions = DirectoryGeneralization(result.Recommendations, result.IndexEntries.Keys, options); + string io = SizeLiteral(options.IOPageSize); + List lines = + [ + $"# Generated by ampr_pack_profile.py {ToolVersion} from APR traces.", + "# IMPORTANT: traces contain only APR reads observed in recorded scenarios.", + "# Adapt this config to the real game by adding remaining files/directories", + "# or deliberately keep them loose. A trace cannot prove complete coverage.", + $"# Projected resident pack index: {HumanSize(result.ProjectedIndexBytes)}.", + $"# Recommended decoded cache: {HumanSize(result.RecommendedCacheBytes)}.", + $"# Recommended physical-page cache: {HumanSize(result.RecommendedPhysicalCacheBytes)}.", + $"# Recommended runtime workers: {result.RecommendedRuntimeWorkers}.", + $"# Recommended latency-reserved workers: {result.RecommendedLatencyReserveWorkers}.", + $"# Recommended AMPR internal pool: {HumanSize(result.RecommendedPoolBytes)}.", + "", + "[pack]", + "index_name = \"ampr_assets.index\"", + "pack_pattern = \"ampr_assets-{group}-lane{lane:02d}-vol{volume:02d}-{id:03d}.pak\"", + "default_action = \"loose\"", + "default_block_size = \"64KiB\"", + $"io_page_size = \"{io}\"", + $"payload_alignment = \"{io}\"", + "chunk_alignment = \"64B\"", + $"workers = {options.Workers}", + "compression_mode = \"hc\"", + "compression_level = 12", + "acceleration = 1", + "deduplicate = true", + "deduplicate_scope = \"lane\"", + "deduplicate_streaming = false", + "min_savings_bytes = 64", + "min_savings_ratio = 0.01", + "io_neutral_min_savings_bytes = \"8KiB\"", + "io_neutral_min_savings_ratio = 0.125", + "auto_loose_large_files = true", + "auto_loose_hot_files = false", + "auto_loose_min_file_size = \"64MiB\"", + "auto_loose_sample_blocks = 32", + "auto_loose_sample_bytes = \"16MiB\"", + "auto_loose_min_savings_ratio = 0.05", + "auto_loose_max_raw_ratio = 0.90", + "preserve_mtime = true", + "validate_index_metadata = true", + "", + "[runtime]", + $"decoded_cache_bytes = {result.RecommendedCacheBytes}", + $"physical_cache_bytes = {result.RecommendedPhysicalCacheBytes}", + $"workers = {result.RecommendedRuntimeWorkers}", + $"latency_reserve_workers = {result.RecommendedLatencyReserveWorkers}", + "", + ]; + foreach ((string name, int packCount, long maxPackSize) in result.Groups) + { + lines.AddRange( + [ + $"[groups.{name}]", + $"pack_count = {packCount}", + $"assignment = {TomlString("balanced")}", + $"max_pack_size = {TomlString(SizeLiteral(maxPackSize))}", + "stripe_large_files = false", + $"io_page_size = {TomlString(io)}", + "", + ]); + } + + for (int emissionIndex = 1; emissionIndex <= emissions.Count; emissionIndex++) + { + RuleEmission emission = emissions[emissionIndex - 1]; + ProfileKey key = emission.Key; + lines.AddRange(emission.Comments.Select(c => $"# {c}")); + lines.Add("[[rule]]"); + lines.Add($"action = {TomlString(key.Action)}"); + bool isExactList = emission.Include.All(p => p.IndexOfAny(['*', '?', '[']) < 0); + bool externalize = externalLists is not null && options.ExternalizePaths > 0 && emission.Include.Count > options.ExternalizePaths && isExactList; + if (externalize) + { + string listName = $"{profileStem}.rule-{emissionIndex:000}.include.txt"; + externalLists![listName] = string.Concat(emission.Include.Select(p => p + "\n")); + lines.Add($"include_from = [{TomlString(listName)}]"); + } + else if (emission.Include.Count == 1) + { + lines.Add($"include = [{TomlString(emission.Include[0])}]"); + } + else + { + lines.Add("include = ["); + lines.AddRange(emission.Include.Select(p => $" {TomlString(p)},")); + lines.Add("]"); + } + + lines.Add($"block_size = {TomlString(SizeLiteral(key.BlockSize))}"); + lines.Add($"group = {TomlString(key.Group)}"); + lines.Add($"layout = {TomlString(key.Layout)}"); + if (key.Hot) + { + lines.Add("hot = true"); + if (key.BlockSize <= 32 * 1024) + { + lines.AddRange( + [ + "min_savings_bytes = 16", + "min_savings_ratio = 0.0025", + "io_neutral_min_savings_bytes = \"2KiB\"", + "io_neutral_min_savings_ratio = 0.125", + ]); + } + } + + if (key.Layout == "streaming") + { + lines.Add("streaming = true"); + } + + lines.Add(""); + } + + lines.AddRange(["# Safety exclusions. The last matching rule wins.", "[[rule]]", "action = \"loose\"", "include = ["]); + lines.AddRange(StandardLoosePatterns.Select(p => $" {TomlString(p)},")); + lines.AddRange(["]", ""]); + return string.Join('\n', lines); + } + + /// Untraced files added. + /// Files added. + /// Their total size. + /// Their chunk records in the pack manifest. + public readonly record struct UntracedAddition(int Files, long Bytes, long IndexBytes); + + /// + /// plus a + /// MkPFS extension (not in ampr_pack_profile.py): every file of the traced games' indexes that no trace read, but + /// whose extension a packed traced file has, is packed too, with the layout, block size and group most common for + /// that extension among the traced files. A trace session rarely visits every level or mode, and archives of the + /// same type are read the same way. Files without an extension, executables and modules, sce_sys, + /// sce_module and fakelib are never added. The rules go before the safety exclusions. + /// + /// Profile. + /// Options it was built with. + /// The trace runs; their indexes list the game's files. + /// Receives sidecars, as for the upstream rules, or . + /// Sidecar name prefix. + /// What was added. + /// TOML text. + public static string RenderToml( + AMPRProfileResult result, AMPRProfileOptions options, IReadOnlyList traces, Dictionary? externalLists, + string profileStem, out UntracedAddition added) + { + string toml = RenderToml(result, options, externalLists, profileStem); + + // The most common packing of each extension among the packed traced files (ties: smaller block first). + Dictionary byType = []; + foreach (IGrouping type in result.Recommendations + .Where(r => r.Action != "loose") + .GroupBy(r => Suffix(r.Metrics.Relative[(r.Metrics.Relative.LastIndexOf('/') + 1)..]).ToLowerInvariant()) + .Where(g => g.Key.Length > 0)) + { + byType[type.Key] = type + .GroupBy(r => new ProfileKey(r.Action, r.Layout, r.BlockSize, false, r.Group)) + .OrderByDescending(g => g.Count()).ThenBy(g => g.Key.BlockSize).ThenBy(g => g.Key) + .First().Key; + } + + Dictionary files = []; + foreach (AMPRTraceSpec spec in traces) + { + foreach (AMPRTraceIndexEntry entry in AMPRCommandLog.LoadIndex(spec.Index).Values) + { + try + { + string relative = CanonicalRelative(entry.Path); + files[relative] = Math.Max(files.GetValueOrDefault(relative), entry.Size); + } + catch (AMPRPackException) + { + // Unsafe paths are reported by the trace reader already. + } + } + } + + Dictionary> rules = []; + long bytes = 0; + long indexBytes = 0; + foreach ((string relative, long size) in files.OrderBy(f => f.Key, PythonCodePointComparer.Instance)) + { + string name = relative[(relative.LastIndexOf('/') + 1)..]; + string top = relative.Split('/')[0]; + if (result.IndexEntries.ContainsKey(relative) || KnownAction(relative) == "loose" + || top is "sce_sys" or "sce_module" or "fakelib" or "fakelib2" + || !byType.TryGetValue(Suffix(name).ToLowerInvariant(), out ProfileKey key)) + { + continue; + } + + if (!rules.TryGetValue(key, out List? list)) + { + rules[key] = list = []; + } + + list.Add(relative); + bytes += size; + indexBytes += CeilDiv(size, key.BlockSize) * ChunkRecordBytes; + } + + added = new UntracedAddition(rules.Values.Sum(l => l.Count), bytes, indexBytes); + if (rules.Count == 0) + { + return toml; + } + + StringBuilder text = new(); + int ruleNumber = 0; + foreach ((ProfileKey key, List paths) in rules.OrderBy(r => r.Key)) + { + ruleNumber++; + string types = string.Join(", ", paths.Select(p => Suffix(p[(p.LastIndexOf('/') + 1)..]).ToLowerInvariant()).Distinct().Order(StringComparer.Ordinal)); + text.Append($"# MkPFS: {paths.Count} untraced {types} files ({HumanSize(paths.Sum(p => files[p]))}), packed like most traced {types} files\n"); + text.Append("[[rule]]\n").Append($"action = {TomlString(key.Action)}\n"); + if (externalLists is not null && options.ExternalizePaths > 0 && paths.Count > options.ExternalizePaths) + { + string listName = $"{profileStem}.untraced-{ruleNumber:000}.include.txt"; + externalLists[listName] = string.Concat(paths.Select(p => p + "\n")); + text.Append($"include_from = [{TomlString(listName)}]\n"); + } + else + { + text.Append("include = [\n"); + foreach (string path in paths) + { + text.Append($" {TomlString(path)},\n"); + } + + text.Append("]\n"); + } + + text.Append($"block_size = {TomlString(SizeLiteral(key.BlockSize))}\n").Append($"group = {TomlString(key.Group)}\n") + .Append($"layout = {TomlString(key.Layout)}\n"); + if (key.Layout == "streaming") + { + text.Append("streaming = true\n"); + } + + text.Append('\n'); + } + + const string safety = "# Safety exclusions. The last matching rule wins.\n"; + int at = toml.LastIndexOf(safety, StringComparison.Ordinal); + return toml.Insert(at, text.ToString()); + } + + /// The pack configuration of with every path inline, without writing it. + /// Profile. + /// Options it was built with. + /// Configuration. + public static AMPRPackConfig ToConfig(AMPRProfileResult result, AMPRProfileOptions options) => ToConfig(RenderToml(result, options)); + + /// Parse rendered profile TOML that has every path inline. + /// TOML text from . + /// Configuration. + public static AMPRPackConfig ToConfig(string toml) => + AMPRPackConfig.FromTable(AMPRToml.Parse(toml, "trace profile"), Directory.GetCurrentDirectory()); + + /// The Markdown report (Python render_report). + /// Profile. + /// Report text. + public static string RenderReport(AMPRProfileResult result) + { + List selected = [.. result.Recommendations.Where(r => r.Action != "loose")]; + long totalReads = selected.Sum(r => (long)r.Metrics.ReadCount); + long submitted = selected.Sum(r => r.Metrics.RequestedBytes); + long fileSize = selected.Sum(r => r.Metrics.FileSize); + List lines = + [ + $"# AMPR trace-derived pack profile: {result.Name}", + "", + "## Summary", + "", + $"- Trace files: **{result.Traces.Count}**", + $"- Selected files: **{selected.Count}**", + $"- Read commands: **{Thousands(totalReads)}**", + $"- Submitted bytes: **{HumanSize(submitted)}**", + $"- Selected logical file size: **{HumanSize(fileSize)}**", + $"- Projected pack index: **{HumanSize(result.ProjectedIndexBytes)}** of {HumanSize(result.IndexBudget)} budget", + $"- Recommended decoded cache: **{HumanSize(result.RecommendedCacheBytes)}**", + $"- Recommended physical-page cache: **{HumanSize(result.RecommendedPhysicalCacheBytes)}**", + $"- Runtime workers: **{result.RecommendedRuntimeWorkers}**", + $"- Latency-reserved workers: **{result.RecommendedLatencyReserveWorkers}**", + $"- Recommended AMPR internal pool: **{HumanSize(result.RecommendedPoolBytes)}**", + "", + "The recommendation describes access behaviour, not entropy. Unless content sampling was enabled, `compress` relies on the packer's per-block RAW fallback.", + "", + "## Profile distribution", + "", + "| Dimension | Distribution |", + "|---|---|", + "| Actions | " + Distribution(selected.Select(r => r.Action), StringKey, k => k) + " |", + "| Layouts | " + Distribution(selected.Select(r => r.Layout), StringKey, k => k) + " |", + "| Block sizes | " + Distribution(selected.Select(r => r.BlockSize), Comparer.Default, SizeLiteral) + " |", + "", + "## Decoded-cache model", + "", + "The model uses byte-weighted LRU with second-touch admission; streaming rules bypass the main cache, matching the runtime policy.", + "", + "| Capacity | Hit bytes | Hit ratio | Block hits | Misses | Touches |", + "|---:|---:|---:|---:|---:|---:|", + ]; + if (result.CacheSimulations.Count > 0) + { + CacheSimulation first = result.CacheSimulations[0]; + if (first.SampledTouches < first.TotalTouches) + { + lines.AddRange( + [ + $"Cache simulation sampled {Thousands(first.SampledTouches)} of {Thousands(first.TotalTouches)} block touches in {first.SampledWindows} contiguous windows.", + "", + ]); + } + + lines.AddRange(result.CacheSimulations.Select(s => + $"| {HumanSize(s.Capacity)} | {HumanSize(s.HitBytes)} | {Percent(s.HitRatio, 1)} | {Thousands(s.Hits)} | {Thousands(s.Misses)} | " + + $"{Thousands(s.SampledTouches)}/{Thousands(s.TotalTouches)} |")); + } + else + { + lines.Add("| disabled | — | — | — | — | — |"); + } + + lines.AddRange( + [ + "", + "## Highest-impact files", + "", + "| Path | Reads | Submitted | p50 | Seq. | Repeat 64K | Recommendation | Amplification | Index | Confidence |", + "|---|---:|---:|---:|---:|---:|---|---:|---:|---|", + ]); + foreach (FileRecommendation recommendation in selected + .OrderByDescending(r => (r.Metrics.RequestedBytes, r.Metrics.ReadCount)).Take(80)) + { + FileMetrics metrics = recommendation.Metrics; + CandidateMetrics candidate = metrics.Candidates[recommendation.BlockSize]; + lines.Add( + $"| `{metrics.Relative.Replace("|", "\\|", StringComparison.Ordinal)}` | {Thousands(metrics.ReadCount)} | {HumanSize(metrics.RequestedBytes)} | " + + $"{HumanSize(metrics.P50)} | {Percent(metrics.ExactSequentialRatio + metrics.NearSequentialRatio, 0)} | " + + $"{Percent(metrics.Candidates[K64].RepeatRatio, 0)} | " + + $"{recommendation.Action}/{recommendation.Layout}/{SizeLiteral(recommendation.BlockSize)}{(recommendation.Hot ? "/hot" : "")} | " + + $"{PythonText.FormatFixed(candidate.Amplification, 2)}x | {HumanSize(recommendation.MetadataBytes)} | {metrics.ConfidenceLabel} |"); + } + + lines.AddRange(["", "## Trace inputs", ""]); + foreach (AMPRTraceSummary trace in result.Traces) + { + lines.AddRange( + [ + $"### {trace.Name}", + "", + $"- Commands: `{trace.Commands}`", + $"- Index: `{trace.Index}`", + $"- Records: {Thousands(trace.Records)}; reads: {Thousands(trace.Reads)}; duration: {PythonText.FormatFixed(trace.DurationNs / 1e9, 1)} s", + $"- Sequence/parser warnings: {trace.Warnings.Count}; decode issues: {trace.DecodeErrors}", + $"- Command SHA-256: `{trace.CommandSha256}`", + $"- Index SHA-256: `{trace.IndexSha256}`", + "", + ]); + } + + if (result.Warnings.Count > 0) + { + lines.AddRange(["## Warnings", ""]); + lines.AddRange(result.Warnings.Take(200).Select(w => $"- {w}")); + if (result.Warnings.Count > 200) + { + lines.Add($"- {result.Warnings.Count - 200} additional warnings omitted"); + } + + lines.Add(""); + } + + lines.AddRange( + [ + "## Operational guidance", + "", + "1. Treat this profile as a starting point: traces contain only observed APR reads. Review the real game and add required files/directories from unrecorded levels, modes, languages and DLC, or deliberately keep them loose.", + "2. Build packs, run `ampr_pack.py verify`, then test with loose source files still available.", + "3. Measure LZ4 ratio. Archives that remain almost entirely RAW and show little cache reuse may be better left loose.", + "4. Re-run this profiler when the game or resource layout changes.", + "", + ]); + return string.Join('\n', lines); + } + + /// + /// The metrics document (Python json.dumps(result_as_json(...), ensure_ascii=False, indent=2) plus a + /// newline): profile settings, traces, cache simulations, warnings and every observed file. + /// + /// Profile. + /// Include all seven block-size candidates per file (--full-metrics). + /// JSON text. + public static string RenderMetricsJson(AMPRProfileResult result, bool fullCandidates = false) + { + PythonJsonObject groups = []; + foreach ((string name, int packCount, long maxPackSize) in result.Groups) + { + groups.Add(name, new PythonJsonObject + { + { "pack_count", packCount }, + { "assignment", "balanced" }, + { "max_pack_size", maxPackSize }, + { "stripe_large_files", false }, + }); + } + + PythonJsonObject document = new() + { + { "tool_version", ToolVersion }, + { "name", result.Name }, + { "traces", result.Traces.Select(t => new PythonJsonObject + { + { "name", t.Name }, + { "commands", t.Commands }, + { "index", t.Index }, + { "command_sha256", t.CommandSha256 }, + { "index_sha256", t.IndexSha256 }, + { "records", t.Records }, + { "reads", t.Reads }, + { "warnings", t.Warnings }, + { "decode_errors", t.DecodeErrors }, + { "duration_ns", t.DurationNs }, + }).ToList() }, + { "projected_index_bytes", result.ProjectedIndexBytes }, + { "index_budget", result.IndexBudget }, + { "recommended_cache_bytes", result.RecommendedCacheBytes }, + { "recommended_physical_cache_bytes", result.RecommendedPhysicalCacheBytes }, + { "recommended_runtime_workers", result.RecommendedRuntimeWorkers }, + { "recommended_latency_reserve_workers", result.RecommendedLatencyReserveWorkers }, + { "recommended_pool_bytes", result.RecommendedPoolBytes }, + { "groups", groups }, + { "cache_simulations", result.CacheSimulations.Select(c => new PythonJsonObject + { + { "capacity", c.Capacity }, + { "requested_bytes", c.RequestedBytes }, + { "hit_bytes", c.HitBytes }, + { "misses", c.Misses }, + { "hits", c.Hits }, + { "sampled_touches", c.SampledTouches }, + { "total_touches", c.TotalTouches }, + { "sampled_windows", c.SampledWindows }, + { "hit_ratio", c.HitRatio }, + }).ToList() }, + { "warnings", result.Warnings }, + { "files", result.Recommendations.OrderByDescending(r => r.Metrics.RequestedBytes).Select(r => RecommendationJson(r, fullCandidates)).ToList() }, + }; + return PythonSortedJson.Indented(document) + "\n"; + } + + // Python _recommendation_json. + private static PythonJsonObject RecommendationJson(FileRecommendation recommendation, bool fullCandidates) + { + FileMetrics m = recommendation.Metrics; + CandidateMetrics selected = m.Candidates[recommendation.BlockSize]; + PythonJsonObject json = new() + { + { "path", m.Path }, + { "relative", m.Relative }, + { "file_size", m.FileSize }, + { "read_count", m.ReadCount }, + { "requested_bytes", m.RequestedBytes }, + { "unique_requested_bytes", m.UniqueRequestedBytes }, + { "coverage_ratio", m.CoverageRatio }, + { "request_percentiles", new PythonJsonObject { { "p50", m.P50 }, { "p75", m.P75 }, { "p90", m.P90 }, { "p95", m.P95 }, { "p99", m.P99 } } }, + { "sequential_ratio", m.ExactSequentialRatio + m.NearSequentialRatio }, + { "random_seek_ratio", m.RandomSeekRatio }, + { "repeat_ratio_64k", m.Candidates[K64].RepeatRatio }, + { "confidence", m.Confidence }, + { "confidence_label", m.ConfidenceLabel }, + { "recommendation", new PythonJsonObject + { + { "action", recommendation.Action }, + { "layout", recommendation.Layout }, + { "block_size", recommendation.BlockSize }, + { "hot", recommendation.Hot }, + { "group", recommendation.Group }, + { "reason", recommendation.Reason }, + { "metadata_bytes", recommendation.MetadataBytes }, + { "amplification", selected.Amplification }, + { "average_blocks_per_read", selected.AverageBlocksPerRead }, + { "sampled_ratio", recommendation.SampledRatio }, + { "sampled_bytes", recommendation.SampledBytes }, + } }, + }; + if (fullCandidates) + { + PythonJsonObject candidates = []; + foreach (CandidateMetrics c in m.Candidates.Values.OrderBy(c => c.BlockSize)) + { + candidates.Add(c.BlockSize.ToString(CultureInfo.InvariantCulture), new PythonJsonObject + { + { "block_size", c.BlockSize }, + { "touched_bytes", c.TouchedBytes }, + { "total_block_touches", c.TotalBlockTouches }, + { "unique_blocks", c.UniqueBlocks }, + { "amplification", c.Amplification }, + { "average_blocks_per_read", c.AverageBlocksPerRead }, + { "repeat_ratio", c.RepeatRatio }, + { "offset_alignment_ratio", c.OffsetAlignmentRatio }, + { "length_alignment_ratio", c.LengthAlignmentRatio }, + { "metadata_bytes", c.MetadataBytes }, + { "local_score", c.LocalScore }, + }); + } + + json.Add("candidates", candidates); + } + + return json; + } + + /// One row of the batch summary.json (Python _build_batch_profile). + /// Profile. + /// JSON object. + public static PythonJsonObject BatchSummaryRow(AMPRProfileResult result) + { + CacheSimulation? first = result.CacheSimulations.Count > 0 ? result.CacheSimulations[0] : null; + return new PythonJsonObject + { + { "name", result.Name }, + { "files", result.Recommendations.Count }, + { "reads", result.Recommendations.Sum(r => (long)r.Metrics.ReadCount) }, + { "submitted_bytes", result.Recommendations.Sum(r => r.Metrics.RequestedBytes) }, + { "selected_file_bytes", result.Recommendations.Sum(r => r.Metrics.FileSize) }, + { "projected_index_bytes", result.ProjectedIndexBytes }, + { "recommended_cache_bytes", result.RecommendedCacheBytes }, + { "recommended_physical_cache_bytes", result.RecommendedPhysicalCacheBytes }, + { "recommended_runtime_workers", result.RecommendedRuntimeWorkers }, + { "recommended_latency_reserve_workers", result.RecommendedLatencyReserveWorkers }, + { "recommended_pool_bytes", result.RecommendedPoolBytes }, + { "cache_sampled_touches", first?.SampledTouches ?? 0 }, + { "cache_total_touches", first?.TotalTouches ?? 0 }, + { "warnings", result.Warnings.Count }, + }; + } + + /// + /// Extract the trace inputs of a ZIP support bundle (Python _extract_trace_archive): only + /// ampr_commands.bin and ampr_emu.index members, under <tempRoot>/traces. + /// + /// ZIP file. + /// Scratch folder. + /// The folder to scan for trace pairs. + /// Not a ZIP, an unsafe member, or no trace inputs. + public static string ExtractTraceArchive(string path, string tempRoot) + { + ZipArchive archive; + try + { + archive = ZipFile.OpenRead(path); + } + catch (InvalidDataException) + { + throw new AMPRPackException($"unsupported archive (only ZIP is accepted): {path}"); + } + + string destination = Path.Combine(tempRoot, "traces"); + Directory.CreateDirectory(destination); + int extracted = 0; + using (archive) + { + foreach (ZipArchiveEntry member in archive.Entries) + { + string name = member.FullName; + string[] parts = [.. name.Split('/').Where(p => p.Length > 0 && p != ".")]; + if (name.StartsWith('/') || parts.Contains("..")) + { + throw new AMPRPackException($"unsafe archive member: {PythonText.Repr(name)}"); + } + + // Support bundles also carry multi-gigabyte decoded logs; the profiler needs only these two. + if (name.EndsWith('/') || parts.Length == 0 || parts[^1] is not ("ampr_commands.bin" or "ampr_emu.index")) + { + continue; + } + + // On Windows a part can still hold "..\" or a drive ("C:"); the target must stay inside the folder. + string target = Path.GetFullPath(Path.Combine([destination, .. parts])); + if (!target.StartsWith(Path.GetFullPath(destination) + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + { + throw new AMPRPackException($"unsafe archive member: {PythonText.Repr(name)}"); + } + + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + member.ExtractToFile(target, overwrite: true); + extracted++; + } + } + + return extracted > 0 ? destination : throw new AMPRPackException($"archive contains no AMPR trace inputs: {path}"); + } + + /// The runtime header with the recommended emulator defines (Python render_runtime_header). + /// Profile. + /// C header text. + public static string RenderRuntimeHeader(AMPRProfileResult result) + { + List lines = + [ + "/*", + $" * Generated by ampr_pack_profile.py {ToolVersion}.", + $" * Profile: {result.Name}", + " * Inputs:", + .. result.Traces.Select(t => $" * {t.Name}: commands={t.CommandSha256} index={t.IndexSha256}"), + " */", + "#pragma once", + "", + $"#define AMPR_EMU_INTERNAL_AMM_POOL_SIZE 0x{result.RecommendedPoolBytes:x}ull", + $"#define AMPR_EMU_PACK_DECODED_CACHE_BYTES 0x{result.RecommendedCacheBytes:x}ull", + $"#define AMPR_EMU_PACK_PHYSICAL_CACHE_BYTES 0x{result.RecommendedPhysicalCacheBytes:x}ull", + $"#define AMPR_EMU_PACK_WORKERS {result.RecommendedRuntimeWorkers}u", + $"#define AMPR_EMU_PACK_LATENCY_RESERVE_WORKERS {result.RecommendedLatencyReserveWorkers}u", + "", + ]; + return string.Join('\n', lines); + } + + /// Python _human_size: N B, or two decimals with KiB to TiB. + /// Size. + /// Text. + public static string HumanSize(double value) + { + double size = value; + foreach (string suffix in (string[])["B", "KiB", "MiB", "GiB", "TiB"]) + { + if (Math.Abs(size) < 1024.0 || suffix == "TiB") + { + return suffix == "B" + ? $"{(long)Math.Round(size, MidpointRounding.ToEven)} B" + : $"{PythonText.FormatFixed(size, 2)} {suffix}"; + } + + size /= 1024.0; + } + + return $"{PythonText.FormatFixed(size, 2)} TiB"; + } + + /// Python _size_literal: the largest exact GiB, MiB or KiB unit, else bytes. + /// Size. + /// TOML size literal. + public static string SizeLiteral(long value) + { + foreach ((long divisor, string suffix) in (ReadOnlySpan<(long, string)>)[(1L << 30, "GiB"), (1L << 20, "MiB"), (1024, "KiB")]) + { + if (value != 0 && value % divisor == 0) + { + return $"{value / divisor}{suffix}"; + } + } + + return $"{value}B"; + } + + // ---- trace loading ------------------------------------------------------------------------------------------- + + private static void AppendTraceWarning(List warnings, string message, int limit = 200) + { + if (warnings.Count < limit) + { + warnings.Add(message); + } + else if (warnings.Count == limit) + { + warnings.Add("additional trace warnings omitted"); + } + } + + // Python _canonical_relative: "/app0/" stripped, PurePosixPath parts (no empty or "." parts), ".." rejected. + private static string CanonicalRelative(string path) + { + string value = path.Replace('\\', '/'); + if (value.Length >= 6 && AsciiLower(value[..6]) == "/app0/") + { + value = value[6..]; + } + else if (AsciiLower(value) == "/app0") + { + value = string.Empty; + } + + string[] parts = [.. value.TrimStart('/').Split('/').Where(p => p.Length > 0 && p != ".")]; + if (parts.Contains("..")) + { + throw new AMPRPackException($"unsafe indexed path: {PythonText.Repr(path)}"); + } + + return parts.Length == 0 ? "." : string.Join('/', parts); + } + + private static string AsciiLower(string value) => string.Create(value.Length, value, static (span, source) => + { + for (int i = 0; i < source.Length; i++) + { + char c = source[i]; + span[i] = c is >= 'A' and <= 'Z' ? (char)(c + 32) : c; + } + }); + + private sealed class GatherState + { + public int FileId { get; init; } + public long NextFileOffset { get; set; } + } + + private static (List Events, AMPRTraceSummary Summary, Dictionary Entries, List Warnings) StreamOneTrace( + AMPRTraceSpec spec, int traceOrdinal, Dictionary index) + { + List events = []; + Dictionary pathEntries = []; + List warnings = []; + Dictionary gatherState = []; + long records = 0; + long readCount = 0; + long decodeErrors = 0; + ulong? previousSequence = null; + ulong minMonotonic = 0; + ulong maxMonotonic = 0; + + using (FileStream handle = new(spec.Commands, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20)) + { + long fileOffset = 0; + byte[] rawHeader = new byte[AMPRCommandLog.HeaderSize]; + while (true) + { + int got = handle.ReadAtLeast(rawHeader, rawHeader.Length, throwOnEndOfStream: false); + if (got == 0) + { + break; + } + + if (got != AMPRCommandLog.HeaderSize) + { + AppendTraceWarning(warnings, $"truncated header at 0x{fileOffset:x}: {got}/{AMPRCommandLog.HeaderSize}"); + break; + } + + if (!rawHeader.AsSpan(0, 8).SequenceEqual(AMPRCommandLog.Magic)) + { + throw new AMPRPackException($"{spec.Name}: invalid command-log magic at 0x{fileOffset:x}"); + } + + AMPRCommandLog.Header header = AMPRCommandLog.ReadHeader(rawHeader); + if (header.Version != AMPRCommandLog.Version) + { + throw new AMPRPackException($"{spec.Name}: unsupported command-log version {header.Version}"); + } + + if (header.HeaderBytes < AMPRCommandLog.HeaderSize) + { + throw new AMPRPackException($"{spec.Name}: invalid header size {header.HeaderBytes} at 0x{fileOffset:x}"); + } + + if (header.RecordBytes < header.HeaderBytes || header.RecordBytes - header.HeaderBytes != header.PayloadBytes) + { + throw new AMPRPackException($"{spec.Name}: inconsistent record sizes at 0x{fileOffset:x}"); + } + + int extraBytes = header.HeaderBytes - AMPRCommandLog.HeaderSize; + if (extraBytes > 0) + { + byte[] extra = new byte[extraBytes]; + if (handle.ReadAtLeast(extra, extraBytes, throwOnEndOfStream: false) != extraBytes) + { + AppendTraceWarning(warnings, "truncated extended header"); + break; + } + } + + // A truncated journal: read what is left (Python read(n) returns short) instead of allocating + // the declared size. + long available = handle.Length - handle.Position; + byte[] payload = new byte[Math.Min(header.PayloadBytes, available)]; + int payloadRead = handle.ReadAtLeast(payload, payload.Length, throwOnEndOfStream: false); + if (payloadRead != header.PayloadBytes) + { + AppendTraceWarning(warnings, $"truncated payload for seq={header.Sequence}: {payloadRead}/{header.PayloadBytes}"); + break; + } + + records++; + ulong sequence = header.Sequence; + if (previousSequence is null) + { + if (sequence != 1) + { + AppendTraceWarning(warnings, $"sequence starts at {sequence}, expected 1"); + } + } + else if (sequence != previousSequence.Value + 1) + { + AppendTraceWarning(warnings, $"sequence gap/non-monotonic: previous={previousSequence.Value}, current={sequence}"); + } + + previousSequence = sequence; + if (AMPRCommandLog.Fnv1a64(payload) != header.PayloadHash) + { + AppendTraceWarning(warnings, $"payload hash mismatch for seq={sequence}"); + } + + if (header.MonotonicNs != 0) + { + minMonotonic = minMonotonic == 0 ? header.MonotonicNs : Math.Min(minMonotonic, header.MonotonicNs); + maxMonotonic = Math.Max(maxMonotonic, header.MonotonicNs); + } + + if (header.Domain != AMPRCommandLog.DomainApr) + { + fileOffset += header.RecordBytes; + continue; + } + + int offset = 0; + int ordinal = 0; + while (offset < payload.Length) + { + AMPRCommand command; + try + { + command = AMPRCommandLog.Decode(payload, offset); + } + catch (InvalidDataException exc) + { + decodeErrors++; + AppendTraceWarning(warnings, $"seq={sequence} decode error at 0x{offset:x}: {exc.Message}"); + break; + } + + long commandBytes = command.Dwords * 4L; + if (commandBytes <= 0 || offset + commandBytes > payload.Length) + { + decodeErrors++; + AppendTraceWarning(warnings, $"seq={sequence} decoder returned invalid size {commandBytes}"); + break; + } + + void Emit(int fileId, long fileReadOffset, long length) + { + if (!index.TryGetValue(fileId, out AMPRTraceIndexEntry? entry)) + { + AppendTraceWarning(warnings, $"fileId {fileId} absent from AMPRIDX3 at seq={sequence}"); + return; + } + + string relative; + try + { + relative = CanonicalRelative(entry.Path); + } + catch (AMPRPackException exc) + { + AppendTraceWarning(warnings, exc.Message); + return; + } + + if (length <= 0 || fileReadOffset < 0) + { + return; + } + + if (fileReadOffset >= entry.Size) + { + AppendTraceWarning(warnings, $"read beyond {relative}: offset=0x{fileReadOffset:x} size=0x{entry.Size:x}"); + return; + } + + if (length > entry.Size - fileReadOffset) + { + AppendTraceWarning(warnings, $"clamped read for {relative}: offset=0x{fileReadOffset:x} length=0x{length:x} size=0x{entry.Size:x}"); + length = entry.Size - fileReadOffset; + } + + pathEntries.TryAdd(relative, entry); + events.Add(new ReadEvent(traceOrdinal, (long)sequence, ordinal, (long)header.MonotonicNs, header.Priority, entry.Path, relative, + fileId, entry.Size, fileReadOffset, length)); + readCount++; + } + + switch (command.Name) + { + case "AprReadFile": + gatherState[header.Priority] = new GatherState { FileId = (int)command.FileId, NextFileOffset = command.FileOffset + command.Length }; + Emit((int)command.FileId, command.FileOffset, command.Length); + break; + case "AprReadGather" or "AprReadGatherScatter": + if (gatherState.TryGetValue(header.Priority, out GatherState? state)) + { + Emit(state.FileId, command.FileOffset, command.Length); + state.NextFileOffset = command.FileOffset + command.Length; + } + else + { + AppendTraceWarning(warnings, $"{command.Name} without active state at seq={sequence}"); + } + + break; + case "AprReadScatter": + if (gatherState.TryGetValue(header.Priority, out GatherState? scatter)) + { + long start = scatter.NextFileOffset; + Emit(scatter.FileId, start, command.Length); + scatter.NextFileOffset = start + command.Length; + } + else + { + AppendTraceWarning(warnings, $"AprReadScatter without active state at seq={sequence}"); + } + + break; + case "AprResetGatherScatterState": + gatherState.Remove(header.Priority); + break; + default: + break; + } + + offset += (int)commandBytes; + ordinal++; + } + + if (header.CommandCount != 0 && ordinal != header.CommandCount) + { + decodeErrors++; + AppendTraceWarning(warnings, $"seq={sequence} header command_count={header.CommandCount}, decoded={ordinal}"); + } + + fileOffset += header.RecordBytes; + } + } + + AMPRTraceSummary summary = new() + { + Name = spec.Name, + Commands = spec.Commands, + Index = spec.Index, + CommandSha256 = Sha256(spec.Commands), + IndexSha256 = Sha256(spec.Index), + Records = records, + Reads = readCount, + Warnings = [.. warnings], + DecodeErrors = decodeErrors, + DurationNs = minMonotonic != 0 && maxMonotonic >= minMonotonic ? (long)(maxMonotonic - minMonotonic) : 0, + }; + return (events, summary, pathEntries, warnings); + } + + private static (List, List, Dictionary, List) LoadTraceEvents(IReadOnlyList traces) + { + List allEvents = []; + List summaries = []; + Dictionary pathEntries = []; + List warnings = []; + for (int ordinal = 0; ordinal < traces.Count; ordinal++) + { + AMPRTraceSpec spec = traces[ordinal]; + Dictionary index = AMPRCommandLog.LoadIndex(spec.Index); + (List events, AMPRTraceSummary summary, Dictionary traceEntries, List traceWarnings) = + StreamOneTrace(spec, ordinal, index); + allEvents.AddRange(events); + summaries.Add(summary); + foreach ((string relative, AMPRTraceIndexEntry entry) in traceEntries) + { + if (pathEntries.TryGetValue(relative, out AMPRTraceIndexEntry? existing) && existing.Size != entry.Size) + { + warnings.Add($"{spec.Name}: indexed size changed for {relative}: {existing.Size} -> {entry.Size}; using the larger entry"); + if (entry.Size > existing.Size) + { + pathEntries[relative] = entry; + } + } + else + { + pathEntries.TryAdd(relative, entry); + } + } + + warnings.AddRange(traceWarnings.Select(w => $"{spec.Name}: {w}")); + } + + return ([.. allEvents.OrderBy(e => (e.TraceOrdinal, e.Sequence, e.CommandOrdinal))], summaries, pathEntries, warnings); + } + + // ---- per-file metrics ---------------------------------------------------------------------------------------- + + private static double Percentile(List values, double pct) + { + if (values.Count == 0) + { + return 0.0; + } + + if (values.Count == 1) + { + return values[0]; + } + + List ordered = [.. values.Order()]; + double position = (ordered.Count - 1) * pct / 100.0; + int lower = (int)Math.Floor(position); + int upper = (int)Math.Ceiling(position); + if (lower == upper) + { + return ordered[lower]; + } + + double fraction = position - lower; + return (ordered[lower] * (1.0 - fraction)) + (ordered[upper] * fraction); + } + + private static long UnionLength(IEnumerable<(long Start, long End)> ranges) + { + long total = 0; + long? currentStart = null; + long currentEnd = 0; + foreach ((long start, long end) in ranges.Order()) + { + if (end <= start) + { + continue; + } + + if (currentStart is null) + { + (currentStart, currentEnd) = (start, end); + continue; + } + + if (start <= currentEnd) + { + currentEnd = Math.Max(currentEnd, end); + } + else + { + total += currentEnd - currentStart.Value; + (currentStart, currentEnd) = (start, end); + } + } + + return currentStart is null ? total : total + (currentEnd - currentStart.Value); + } + + private static long CeilDiv(long value, long divisor) => (long)Math.Ceiling((double)value / divisor); + + private static CandidateMetrics CandidateFor(List events, long fileSize, long blockSize, string layout, double randomness) + { + long requestedBytes = events.Sum(e => e.Length); + long touchedBytes = 0; + long totalTouches = 0; + List<(long, long)> blockRanges = new(events.Count); + long offsetAligned = 0; + long lengthAligned = 0; + foreach (ReadEvent e in events) + { + long first = e.Offset / blockSize; + long lastExclusive = (e.End + blockSize - 1) / blockSize; + long start = first * blockSize; + long end = Math.Min(lastExclusive * blockSize, fileSize); + touchedBytes += Math.Max(0, end - start); + totalTouches += Math.Max(0, lastExclusive - first); + blockRanges.Add((first, lastExclusive)); + offsetAligned += e.Offset % blockSize == 0 ? 1 : 0; + lengthAligned += e.Length % blockSize == 0 ? 1 : 0; + } + + long uniqueBlocks = UnionLength(blockRanges); + double amplification = requestedBytes != 0 ? (double)touchedBytes / requestedBytes : 1.0; + double averageBlocks = events.Count != 0 ? (double)totalTouches / events.Count : 0.0; + double repeatRatio = totalTouches != 0 ? Math.Max(0.0, 1.0 - ((double)uniqueBlocks / totalTouches)) : 0.0; + long metadataBytes = CeilDiv(fileSize, blockSize) * ChunkRecordBytes; + + double logAmp = Math.Log2(Math.Max(1.0, amplification)); + double logOps = Math.Log2(Math.Max(1.0, averageBlocks)); + double sizeLog = Math.Log2((double)blockSize / K64); + double score; + if (layout == "random") + { + score = (3.8 * logAmp) + (0.12 * logOps); + if (blockSize > K64) + { + score += 0.55 * Math.Log2((double)blockSize / K64); + } + + if (blockSize < 16 * 1024) + { + score += 10.0; + } + } + else if (layout == "streaming") + { + score = (0.65 * logAmp) + (0.95 * logOps); + if (blockSize < 128 * 1024) + { + score += 0.35 * Math.Log2(128.0 * 1024 / blockSize); + } + + if (blockSize > 512 * 1024) + { + score += 0.15 * Math.Log2((double)blockSize / (512 * 1024)); + } + } + else + { + score = (2.15 * logAmp) + (0.38 * logOps); + score += Math.Abs(sizeLog) * 0.06; + if (blockSize > 128 * 1024) + { + score += 0.28 * Math.Log2((double)blockSize / (128 * 1024)); + } + + if (blockSize > K64) + { + score += randomness * 0.42 * Math.Log2((double)blockSize / K64); + } + + if (blockSize < 32 * 1024) + { + score += 0.20 * Math.Log2(32.0 * 1024 / blockSize); + } + } + + score += (double)metadataBytes / Math.Max(fileSize, 1) * 3.0; + return new CandidateMetrics(blockSize, touchedBytes, totalTouches, uniqueBlocks, amplification, averageBlocks, repeatRatio, + events.Count != 0 ? (double)offsetAligned / events.Count : 0.0, events.Count != 0 ? (double)lengthAligned / events.Count : 0.0, + metadataBytes, score); + } + + private static string ClassifyLayout(FileMetricsInput m) + { + double sequential = m.ExactSequentialRatio + m.NearSequentialRatio; + bool microIndex = m.P50 <= 256 && m.ReadCount >= 64 && (m.Repeat64k >= 0.10 || sequential >= 0.70); + if (microIndex) + { + return "random"; + } + + if (m.P50 >= K64 && sequential >= 0.76 && m.RandomSeekRatio <= 0.20 && m.Repeat64k <= 0.18) + { + return "streaming"; + } + + if (m.P90 >= 256 * 1024 && m.Large256kRatio >= 0.65 && sequential >= 0.86 && m.Repeat64k <= 0.12) + { + return "streaming"; + } + + return m.Tiny4kRatio >= 0.50 || m.Small16kRatio >= 0.70 || (m.Small64kRatio >= 0.82 && m.RandomSeekRatio >= 0.30) + || (m.Repeat64k >= 0.28 && m.P50 <= K64) + ? "random" + : "mixed"; + } + + private readonly record struct FileMetricsInput( + double P50, double P90, double Tiny4kRatio, double Small16kRatio, double Small64kRatio, double Large256kRatio, + double ExactSequentialRatio, double NearSequentialRatio, double RandomSeekRatio, double Repeat64k, int ReadCount); + + private static FileMetricsInput LayoutInput(FileMetrics m) => new( + m.P50, m.P90, m.Tiny4kRatio, m.Small16kRatio, m.Small64kRatio, m.Large256kRatio, m.ExactSequentialRatio, m.NearSequentialRatio, + m.RandomSeekRatio, m.Candidates[K64].RepeatRatio, m.ReadCount); + + private static List BuildFileMetrics(List events) + { + Dictionary> grouped = []; + foreach (ReadEvent e in events) + { + if (!grouped.TryGetValue(e.Relative, out List? list)) + { + grouped[e.Relative] = list = []; + } + + list.Add(e); + } + + List results = []; + foreach (string relative in grouped.Keys.Order(PythonCodePointComparer.Instance)) + { + List fileEvents = [.. grouped[relative].OrderBy(e => (e.TraceOrdinal, e.Sequence, e.CommandOrdinal))]; + List lengths = [.. fileEvents.Select(e => e.Length)]; + long requested = lengths.Sum(); + long fileSize = fileEvents.Max(e => e.FileSize); + long uniqueRequested = UnionLength(fileEvents.Select(e => (e.Offset, e.End))); + + long transitionBytes = 0; + long exactBytes = 0; + long nearBytes = 0; + long randomBytes = 0; + Dictionary previousByTrace = []; + foreach (ReadEvent e in fileEvents) + { + previousByTrace.TryGetValue(e.TraceOrdinal, out ReadEvent? previous); + previousByTrace[e.TraceOrdinal] = e; + if (previous is null) + { + continue; + } + + long weight = e.Length; + transitionBytes += weight; + if (e.Offset == previous.End) + { + exactBytes += weight; + continue; + } + + long gap = e.Offset - previous.End; + long nearLimit = Math.Max(DefaultIOPage, Math.Min(previous.Length, 512 * 1024)); + if (gap >= 0 && gap <= nearLimit) + { + nearBytes += weight; + } + else if (e.Offset < previous.Offset) + { + randomBytes += weight; + } + else if (e.Offset < previous.End) + { + // overlap: counted by Python's overlap ratio, which nothing here uses + } + else + { + randomBytes += weight; + } + } + + double Share(Func predicate) => (double)lengths.Where(predicate).Sum() / requested; + double Ratio(long part) => transitionBytes != 0 ? (double)part / transitionBytes : 0.0; + double tiny4k = Share(l => l <= 4 * 1024); + double small16k = Share(l => l <= 16 * 1024); + double small64k = Share(l => l <= K64); + double large256k = Share(l => l >= 256 * 1024); + double exactRatio = Ratio(exactBytes); + double nearRatio = Ratio(nearBytes); + double randomRatio = Ratio(randomBytes); + + CandidateMetrics provisional64 = CandidateFor(fileEvents, fileSize, K64, "mixed", randomRatio); + double p50 = Percentile(lengths, 50); + double p90 = Percentile(lengths, 90); + string layout = ClassifyLayout(new FileMetricsInput( + p50, p90, tiny4k, small16k, small64k, large256k, exactRatio, nearRatio, randomRatio, provisional64.RepeatRatio, fileEvents.Count)); + Dictionary candidates = BlockSizes.ToDictionary(size => size, size => CandidateFor(fileEvents, fileSize, size, layout, randomRatio)); + + double evidenceReads = Math.Min(1.0, Math.Log10(fileEvents.Count + 1) / 3.0); + double evidenceBytes = Math.Min(1.0, requested / (64.0 * 1024 * 1024)); + int traceCount = fileEvents.Select(e => e.TraceOrdinal).Distinct().Count(); + double multiTrace = Math.Min(1.0, Math.Max(0, traceCount - 1) / 2.0); + double confidence = (0.45 * evidenceReads) + (0.35 * evidenceBytes) + (0.20 * multiTrace); + string confidenceLabel = confidence >= 0.72 ? "high" : confidence >= 0.42 ? "medium" : "low"; + + results.Add(new FileMetrics + { + Path = fileEvents[0].Path, + Relative = relative, + FileSize = fileSize, + TraceCount = traceCount, + Events = fileEvents, + ReadCount = fileEvents.Count, + RequestedBytes = requested, + UniqueRequestedBytes = uniqueRequested, + CoverageRatio = fileSize != 0 ? (double)uniqueRequested / fileSize : 0.0, + P50 = p50, + P75 = Percentile(lengths, 75), + P90 = p90, + P95 = Percentile(lengths, 95), + P99 = Percentile(lengths, 99), + Tiny4kRatio = tiny4k, + Small16kRatio = small16k, + Small64kRatio = small64k, + Large256kRatio = large256k, + ExactSequentialRatio = exactRatio, + NearSequentialRatio = nearRatio, + RandomSeekRatio = randomRatio, + Candidates = candidates, + Confidence = confidence, + ConfidenceLabel = confidenceLabel, + }); + } + + return results; + } + + // ---- recommendations ----------------------------------------------------------------------------------------- + + // Python PurePosixPath.suffix: from the last dot, unless the name starts with it or ends with it. + private static string Suffix(string name) + { + int i = name.LastIndexOf('.'); + return i > 0 && i < name.Length - 1 ? name[i..] : string.Empty; + } + + private static string KnownAction(string relative) + { + string name = relative[(relative.LastIndexOf('/') + 1)..]; + string lowerName = name.ToLowerInvariant(); + string suffix = Suffix(name).ToLowerInvariant(); + if (KnownLooseBasenames.Contains(lowerName) || KnownLooseSuffixes.Contains(suffix)) + { + return "loose"; + } + + return KnownStreamStoreSuffixes.Contains(suffix) ? "store" : "compress"; + } + + private static List InitialRecommendations(List files, AMPRProfileOptions options) + { + if (files.Count == 0) + { + return []; + } + + long readHotThreshold = Math.Max(32, (long)Percentile([.. files.Select(f => (long)f.ReadCount)], 90)); + long byteHotThreshold = Math.Max(16L * 1024 * 1024, (long)Percentile([.. files.Select(f => f.RequestedBytes)], 90)); + List recommendations = []; + foreach (FileMetrics metrics in files) + { + if (metrics.ReadCount < options.MinReads || metrics.RequestedBytes < options.MinRequestedBytes) + { + continue; + } + + string layout = ClassifyLayout(LayoutInput(metrics)); + string action = KnownAction(metrics.Relative); + if (action == "store") + { + layout = "streaming"; + } + + double repeat64k = metrics.Candidates[K64].RepeatRatio; + bool hot = layout != "streaming" + && ((metrics.ReadCount >= 16 && repeat64k >= 0.22) || metrics.ReadCount >= readHotThreshold + || (metrics.ReadCount >= 8 && metrics.RequestedBytes >= byteHotThreshold)); + + IEnumerable allowed = layout switch + { + "random" => BlockSizes.Where(s => s <= 128 * 1024), + "mixed" => BlockSizes.Where(s => s is >= 32 * 1024 and <= 256 * 1024), + _ => BlockSizes.Where(s => s >= K64), + }; + long selected = 0; + (double Score, double Distance, long Size)? best = null; + foreach (long size in allowed) + { + (double, double, long) key = (metrics.Candidates[size].LocalScore, Math.Abs(Math.Log2((double)size / K64)), size); + if (best is null || key.CompareTo(best.Value) < 0) + { + best = key; + selected = size; + } + } + + if (metrics.P50 <= 256 && metrics.ReadCount >= 64) + { + selected = 16 * 1024; + layout = "random"; + hot = true; + } + + string group = action == "loose" ? "loose" : action == "store" || layout == "streaming" ? "stream" : layout == "random" ? "metadata" : "bulk"; + CandidateMetrics chosen = metrics.Candidates[selected]; + string reason = $"{layout}; p50={HumanSize(metrics.P50)}, sequential={Percent(metrics.ExactSequentialRatio + metrics.NearSequentialRatio, 0)}, " + + $"repeat64={Percent(repeat64k, 0)}, amp={PythonText.FormatFixed(chosen.Amplification, 2)}x"; + recommendations.Add(new FileRecommendation + { + Metrics = metrics, + Action = action, + Layout = layout, + BlockSize = selected, + Hot = hot, + Group = group, + Reason = reason, + }); + } + + return recommendations; + } + + private static void EnforceIndexBudget(List recommendations, long budget, string strategy) + { + List packed = [.. recommendations.Where(r => r.Action != "loose")]; + if (packed.Count == 0) + { + return; + } + + double strategyWeight = strategy switch + { + "conservative" => 1.35, + "balanced" => 1.0, + "aggressive" => 0.70, + _ => throw new AMPRPackException($"unknown strategy: {strategy}"), + }; + PriorityQueue<(int Index, long NextSize), (double Ratio, int Generation, int Index, long NextSize)> heap = new(Comparer<(double, int, int, long)>.Default); + int[] generations = new int[packed.Count]; + + void Push(int index) + { + FileRecommendation recommendation = packed[index]; + int currentPos = Array.IndexOf(BlockSizes, recommendation.BlockSize); + if (currentPos < 0 || currentPos + 1 >= BlockSizes.Length) + { + return; + } + + long nextSize = BlockSizes[currentPos + 1]; + CandidateMetrics current = recommendation.Metrics.Candidates[recommendation.BlockSize]; + CandidateMetrics next = recommendation.Metrics.Candidates[nextSize]; + long saved = current.MetadataBytes - next.MetadataBytes; + if (saved <= 0) + { + return; + } + + double scoreCost = Math.Max(0.0, next.LocalScore - current.LocalScore); + double importance = 1.0; + if (recommendation.Hot) + { + importance *= 2.5; + } + + if (recommendation.Layout == "random") + { + importance *= 1.8; + } + else if (recommendation.Layout == "streaming") + { + importance *= 0.55; + } + + importance *= 0.75 + (0.5 * recommendation.Metrics.Confidence); + double ratio = scoreCost * importance * strategyWeight / saved; + heap.Enqueue((index, nextSize), (ratio, generations[index], index, nextSize)); + } + + for (int index = 0; index < packed.Count; index++) + { + Push(index); + } + + long total = packed.Sum(r => r.MetadataBytes); + while (total > budget && heap.TryDequeue(out (int Index, long NextSize) item, out (double, int Generation, int, long) priority)) + { + if (priority.Generation != generations[item.Index]) + { + continue; + } + + FileRecommendation recommendation = packed[item.Index]; + if (item.NextSize <= recommendation.BlockSize) + { + continue; + } + + total -= recommendation.MetadataBytes; + recommendation.BlockSize = item.NextSize; + total += recommendation.MetadataBytes; + generations[item.Index]++; + Push(item.Index); + } + } + + private static List SamplePositions(long fileSize, long blockSize, int count) + { + if (fileSize <= 0 || count <= 0) + { + return []; + } + + if (fileSize <= blockSize || count == 1) + { + return [0]; + } + + long maxStart = Math.Max(0, fileSize - blockSize); + SortedSet positions = []; + for (int index = 0; index < count; index++) + { + positions.Add((long)Math.Round((double)maxStart * index / (count - 1), MidpointRounding.ToEven)); + } + + return [.. positions.Select(p => p - (p % blockSize)).Order()]; + } + + private static void SampleCompressibility(List recommendations, AMPRProfileOptions options, List warnings) + { + if (options.ContentRoot is null || options.SampleBudget <= 0) + { + return; + } + + using LZ4Encoder codec = new(); + string root = Path.GetFullPath(options.ContentRoot); + long remaining = options.SampleBudget; + foreach (FileRecommendation recommendation in recommendations.Where(r => r.Action == "compress") + .OrderByDescending(r => (r.Metrics.RequestedBytes, r.Metrics.ReadCount))) + { + if (remaining <= 0) + { + break; + } + + string source = Path.GetFullPath(Path.Combine(root, recommendation.Metrics.Relative)); + string relativeToRoot = Path.GetRelativePath(root, source); + if (relativeToRoot == ".." || relativeToRoot.StartsWith(".." + Path.DirectorySeparatorChar, StringComparison.Ordinal) || Path.IsPathRooted(relativeToRoot)) + { + warnings.Add($"content sample escaped root: {source}"); + continue; + } + + if (!File.Exists(source)) + { + continue; + } + + long rawTotal = 0; + long storedTotal = 0; + try + { + using FileStream handle = File.OpenRead(source); + foreach (long position in SamplePositions(recommendation.Metrics.FileSize, recommendation.BlockSize, options.SampleBlocksPerFile)) + { + if (remaining <= 0) + { + break; + } + + int length = (int)Math.Min(recommendation.BlockSize, remaining); + handle.Seek(position, SeekOrigin.Begin); + byte[] buffer = new byte[length]; + int got = handle.ReadAtLeast(buffer, length, throwOnEndOfStream: false); + if (got == 0) + { + continue; + } + + ReadOnlySpan raw = buffer.AsSpan(0, got); + byte[] compressed = options.SampleMode == "hc" + ? codec.CompressHC(raw, Math.Clamp(options.SampleLevel, 1, 12)) + : codec.CompressFast(raw, Math.Max(1, options.SampleAcceleration)); + rawTotal += got; + storedTotal += Math.Min(got, compressed.Length); + remaining -= got; + } + } + catch (IOException exc) + { + warnings.Add($"cannot sample {recommendation.Metrics.Relative}: {exc.Message}"); + continue; + } + + if (rawTotal == 0) + { + continue; + } + + double ratio = (double)storedTotal / rawTotal; + recommendation.SampledRatio = ratio; + recommendation.SampledBytes = rawTotal; + if (ratio >= 0.94) + { + if (recommendation.Metrics.FileSize >= 64L * 1024 * 1024) + { + recommendation.Action = "loose"; + recommendation.Hot = false; + recommendation.Reason += $"; sampled LZ4 ratio={Percent(ratio, 1)}, auto-loose"; + } + else + { + recommendation.Action = "store"; + recommendation.Layout = recommendation.Layout == "streaming" ? "streaming" : "mixed"; + recommendation.Group = recommendation.Layout == "streaming" ? "stream" : "bulk"; + recommendation.Hot = recommendation.Layout != "streaming" && recommendation.Hot; + recommendation.Reason += $"; sampled LZ4 ratio={Percent(ratio, 1)}, store"; + } + } + else + { + recommendation.Reason += $"; sampled LZ4 ratio={Percent(ratio, 1)}"; + } + } + } + + private static long EstimateIndexBytes(IReadOnlyList recommendations, int packCount, Dictionary indexEntries) + { + long chunkBytes = recommendations.Where(r => r.Action != "loose").Sum(r => r.MetadataBytes); + long fileBytes = (long)indexEntries.Count * FileRecordBytes; + long stringBytes = indexEntries.Keys.Sum(p => (long)Encoding.UTF8.GetByteCount(p) + 1); + return chunkBytes + fileBytes + ((long)packCount * PackRecordBytes) + stringBytes + 4096; + } + + private static int ChooseLanes(List recommendations, int requested) + { + if (requested > 0) + { + return requested; + } + + List packed = [.. recommendations.Where(r => r.Action != "loose")]; + long totalSize = packed.Sum(r => r.Metrics.FileSize); + int fileCount = packed.Count; + return fileCount <= 1 ? 1 + : totalSize < 2L * 1024 * 1024 * 1024 && fileCount < 64 ? 2 + : totalSize < 16L * 1024 * 1024 * 1024 && fileCount < 1024 ? 3 + : 4; + } + + private static List<(string Name, int PackCount, long MaxPackSize)> BuildGroups(List recommendations, int lanes) + { + List<(string, int, long)> result = []; + foreach (string groupName in (string[])["metadata", "bulk", "stream"]) + { + List items = [.. recommendations.Where(r => r.Action != "loose" && r.Group == groupName)]; + if (items.Count == 0) + { + continue; + } + + long totalSize = items.Sum(i => i.Metrics.FileSize); + (int packCount, long maxPackSize) = groupName switch + { + "bulk" => (Math.Min(lanes, Math.Max(1, items.Count)), 16L << 30), + "metadata" => (Math.Min(Math.Max(1, Math.Min(lanes, totalSize < 8L << 30 ? 2 : 4)), items.Count), 8L << 30), + _ => (Math.Min(Math.Max(1, Math.Min(lanes, 2)), items.Count), 16L << 30), + }; + result.Add((groupName, packCount, maxPackSize)); + } + + return result; + } + + // ---- decoded-cache model ------------------------------------------------------------------------------------- + + private static IEnumerable<(long BlockIndex, long RawSize, long Overlap)> BlockOverlaps(ReadEvent e, long blockSize) + { + long first = e.Offset / blockSize; + long last = (e.End - 1) / blockSize; + for (long blockIndex = first; blockIndex <= last; blockIndex++) + { + long blockStart = blockIndex * blockSize; + long blockEnd = Math.Min(blockStart + blockSize, e.FileSize); + long overlap = Math.Max(0, Math.Min(e.End, blockEnd) - Math.Max(e.Offset, blockStart)); + if (overlap != 0) + { + yield return (blockIndex, blockEnd - blockStart, overlap); + } + } + } + + private static long BlockTouchCount(ReadEvent e, long blockSize) => + e.Length <= 0 || blockSize <= 0 ? 0 : ((e.End - 1) / blockSize) - (e.Offset / blockSize) + 1; + + private sealed class AccessStream + { + public List Keys { get; } = []; + public List RawSizes { get; } = []; + public List Overlaps { get; } = []; + public List Hot { get; } = []; + public List ResetBefore { get; } = []; + public long TotalTouches { get; set; } + public long SampledWindows { get; set; } + } + + private static AccessStream BuildAccessStream(List events, Dictionary byPath, long maxTouches) + { + List<(ReadEvent Event, FileRecommendation Recommendation, long Touches)> eligible = []; + List cumulativeEnds = []; + long totalTouches = 0; + Dictionary fileOrdinals = []; + foreach (ReadEvent e in events) + { + if (!byPath.TryGetValue(e.Relative, out FileRecommendation? recommendation) || recommendation.Action == "loose" || recommendation.Layout == "streaming") + { + continue; + } + + long touches = BlockTouchCount(e, recommendation.BlockSize); + if (touches <= 0) + { + continue; + } + + fileOrdinals.TryAdd(e.Relative, (ulong)fileOrdinals.Count + 1); + totalTouches += touches; + eligible.Add((e, recommendation, touches)); + cumulativeEnds.Add(totalTouches); + } + + AccessStream stream = new(); + if (eligible.Count == 0) + { + return stream; + } + + long AppendSlice(ReadEvent e, FileRecommendation recommendation, long firstTouch, long touchCount, bool reset) + { + long appended = 0; + ulong fileOrdinal = fileOrdinals[e.Relative]; + long stopTouch = firstTouch + touchCount; + long localIndex = 0; + foreach ((long blockIndex, long rawSize, long overlap) in BlockOverlaps(e, recommendation.BlockSize)) + { + if (localIndex >= stopTouch) + { + break; + } + + if (localIndex++ < firstTouch) + { + continue; + } + + if (blockIndex >= 1L << 48) + { + throw new AMPRPackException($"cache simulation block index is too large: {e.Relative} block={blockIndex}"); + } + + stream.Keys.Add((fileOrdinal << 48) | (ulong)blockIndex); + stream.RawSizes.Add(rawSize); + stream.Overlaps.Add(overlap); + stream.Hot.Add(recommendation.Hot); + stream.ResetBefore.Add(reset && appended == 0); + appended++; + } + + return appended; + } + + if (maxTouches <= 0 || totalTouches <= maxTouches) + { + bool first = true; + foreach ((ReadEvent e, FileRecommendation recommendation, long touches) in eligible) + { + AppendSlice(e, recommendation, 0, touches, first); + first = false; + } + + stream.SampledWindows = 1; + } + else + { + long windowCount = Math.Min(64, Math.Max(1, maxTouches / 4096)); + long windowBudget = Math.Max(1, maxTouches / windowCount); + long sampledWindows = 0; + for (long windowIndex = 0; windowIndex < windowCount; windowIndex++) + { + long segmentStart = (long)((Int128)totalTouches * windowIndex / windowCount); + long segmentEnd = (long)((Int128)totalTouches * (windowIndex + 1) / windowCount); + long segmentSize = Math.Max(0, segmentEnd - segmentStart); + if (segmentSize == 0) + { + continue; + } + + long budget = Math.Min(windowBudget, segmentSize); + long startTouch = segmentStart + ((segmentSize - budget) / 2); + long endTouch = startTouch + budget; + int eventIndex = BisectRight(cumulativeEnds, startTouch); + bool windowFirst = true; + while (eventIndex < eligible.Count) + { + long eventStart = eventIndex != 0 ? cumulativeEnds[eventIndex - 1] : 0; + if (eventStart >= endTouch) + { + break; + } + + (ReadEvent e, FileRecommendation recommendation, long touches) = eligible[eventIndex]; + long localStart = Math.Max(0, startTouch - eventStart); + long localEnd = Math.Min(touches, endTouch - eventStart); + if (localEnd > localStart && AppendSlice(e, recommendation, localStart, localEnd - localStart, windowFirst) != 0) + { + windowFirst = false; + } + + eventIndex++; + } + + if (!windowFirst) + { + sampledWindows++; + } + } + + stream.SampledWindows = sampledWindows; + } + + stream.TotalTouches = totalTouches; + return stream; + } + + private static int BisectRight(List values, long x) + { + int lo = 0; + int hi = values.Count; + while (lo < hi) + { + int mid = (lo + hi) / 2; + if (x < values[mid]) + { + hi = mid; + } + else + { + lo = mid + 1; + } + } + + return lo; + } + + private static List SimulateCache(List events, Dictionary byPath, IReadOnlyList capacities, long maxTouches) + { + AccessStream stream = BuildAccessStream(events, byPath, maxTouches); + List simulations = []; + int sampledTouches = stream.Keys.Count; + foreach (long capacity in capacities) + { + LinkedList<(ulong Key, long Size)> order = []; + Dictionary> cache = []; + HashSet seenOnce = []; + long resident = 0; + long requested = 0; + long hitBytes = 0; + long hits = 0; + long misses = 0; + for (int index = 0; index < sampledTouches; index++) + { + if (stream.ResetBefore[index]) + { + order.Clear(); + cache.Clear(); + seenOnce.Clear(); + resident = 0; + } + + ulong key = stream.Keys[index]; + long rawSize = stream.RawSizes[index]; + long overlap = stream.Overlaps[index]; + requested += overlap; + if (cache.TryGetValue(key, out LinkedListNode<(ulong Key, long Size)>? node)) + { + order.Remove(node); + order.AddLast(node); + hitBytes += overlap; + hits++; + continue; + } + + misses++; + bool admit = stream.Hot[index] || seenOnce.Contains(key); + if (!admit) + { + seenOnce.Add(key); + continue; + } + + seenOnce.Remove(key); + if (rawSize > capacity) + { + continue; + } + + while (resident + rawSize > capacity && order.First is { } victim) + { + order.RemoveFirst(); + cache.Remove(victim.Value.Key); + resident -= victim.Value.Size; + } + + if (resident + rawSize <= capacity) + { + cache[key] = order.AddLast((key, rawSize)); + resident += rawSize; + } + } + + simulations.Add(new CacheSimulation(capacity, requested, hitBytes, misses, hits, sampledTouches, stream.TotalTouches, stream.SampledWindows)); + } + + return simulations; + } + + private static long ChooseCacheSize(List simulations) + { + if (simulations.Count == 0) + { + return 64L * 1024 * 1024; + } + + double bestRatio = simulations.Max(s => s.HitRatio); + if (bestRatio < 0.08) + { + return simulations.Where(s => s.Capacity >= 64L * 1024 * 1024).Select(s => (long?)s.Capacity).Min() ?? simulations[0].Capacity; + } + + long Nearest(long target) => simulations.MinBy(s => (Math.Abs(s.Capacity - target), s.Capacity))!.Capacity; + if (bestRatio < 0.15) + { + return Nearest(128L * 1024 * 1024); + } + + if (bestRatio < 0.25) + { + return Nearest(256L * 1024 * 1024); + } + + CacheSimulation? previous = null; + foreach (CacheSimulation simulation in simulations) + { + if (simulation.Capacity < 64L * 1024 * 1024) + { + previous = simulation; + continue; + } + + bool withinBest = bestRatio - simulation.HitRatio <= 0.03; + double marginal = previous is not null ? simulation.HitRatio - previous.HitRatio : simulation.HitRatio; + if (withinBest || (simulation.HitRatio >= 0.30 && marginal < 0.05)) + { + return simulation.Capacity; + } + + previous = simulation; + } + + return simulations[^1].Capacity; + } + + // Python >= 3.12 sum() of floats: Neumaier compensated summation. + private static double PythonFloatSum(IEnumerable values) + { + double result = 0.0; + double compensation = 0.0; + bool first = true; + foreach (double x in values) + { + if (first) + { + result = 0 + x; + first = false; + continue; + } + + double t = result + x; + compensation += Math.Abs(result) >= Math.Abs(x) ? result - t + x : x - t + result; + result = t; + } + + return compensation != 0 && double.IsFinite(compensation) ? result + compensation : result; + } + + private static long ChoosePhysicalPageCacheSize(List recommendations) + { + List eligible = [.. recommendations.Where(r => r.Action != "loose" && r.Layout != "streaming")]; + if (eligible.Count == 0) + { + return 8L * 1024 * 1024; + } + + long requested = eligible.Sum(i => i.Metrics.RequestedBytes); + if (requested <= 0) + { + return 8L * 1024 * 1024; + } + + double weightedRepeat = PythonFloatSum(eligible.Select(i => i.Metrics.RequestedBytes * i.Metrics.Candidates[K64].RepeatRatio)) / requested; + double weightedSmall = PythonFloatSum(eligible.Select(i => i.Metrics.RequestedBytes * i.Metrics.Small64kRatio)) / requested; + double weightedRandom = PythonFloatSum(eligible.Select(i => i.Metrics.RequestedBytes * i.Metrics.RandomSeekRatio)) / requested; + double pressure = weightedRepeat + (0.5 * weightedSmall * weightedRandom); + if (requested >= 16L * 1024 * 1024 * 1024 && pressure >= 0.20) + { + return 64L * 1024 * 1024; + } + + return requested >= 2L * 1024 * 1024 * 1024 || pressure >= 0.10 ? 32L * 1024 * 1024 : 16L * 1024 * 1024; + } + + private static int ChooseRuntimeWorkers(List events, List recommendations, int requested) + { + if (requested > 0) + { + return requested; + } + + HashSet selectedPaths = [.. recommendations.Where(r => r.Action != "loose").Select(r => r.Metrics.Relative)]; + List selectedEvents = [.. events.Where(e => selectedPaths.Contains(e.Relative))]; + if (selectedEvents.Count == 0) + { + return 2; + } + + Dictionary bytesByPath = []; + long switches = 0; + string? previous = null; + foreach (ReadEvent e in selectedEvents) + { + bytesByPath[e.Relative] = bytesByPath.GetValueOrDefault(e.Relative) + e.Length; + if (previous is not null && previous != e.Relative) + { + switches++; + } + + previous = e.Relative; + } + + long totalBytes = bytesByPath.Values.Sum(); + double dominantShare = totalBytes != 0 ? (double)bytesByPath.Values.Max() / totalBytes : 1.0; + double switchRatio = (double)switches / Math.Max(1, selectedEvents.Count - 1); + int fileCount = bytesByPath.Count; + return fileCount >= 1000 && switchRatio >= 0.70 ? 8 + : fileCount >= 128 && switchRatio >= 0.50 ? 6 + : dominantShare >= 0.85 && fileCount <= 8 ? 2 + : 4; + } + + private static int ChooseLatencyReserveWorkers(List events, List recommendations, int runtimeWorkers) + { + if (runtimeWorkers <= 1) + { + return 0; + } + + HashSet selected = [.. recommendations.Where(r => r.Action != "loose").Select(r => r.Metrics.Relative)]; + List selectedEvents = [.. events.Where(e => selected.Contains(e.Relative))]; + if (selectedEvents.Count == 0) + { + return 0; + } + + double total = selectedEvents.Count; + double smallRatio = selectedEvents.Count(e => e.Length <= K64) / total; + double tinyRatio = selectedEvents.Count(e => e.Length <= 16 * 1024) / total; + double bulkRatio = selectedEvents.Count(e => e.Length >= 256 * 1024) / total; + return smallRatio >= 0.45 || tinyRatio >= 0.20 || (smallRatio >= 0.25 && bulkRatio >= 0.20) ? 1 : 0; + } + + private static long RoundPoolSize(long required) + { + const long quantum = 128L * 1024 * 1024; + return Math.Max(256L * 1024 * 1024, CeilDiv(required, quantum) * quantum); + } + + // ---- rule emission ------------------------------------------------------------------------------------------- + + private static List ProfileComment(IEnumerable recommendations) + { + List items = [.. recommendations]; + long reads = items.Sum(r => (long)r.Metrics.ReadCount); + long requested = items.Sum(r => r.Metrics.RequestedBytes); + string confidence = string.Join(", ", items.GroupBy(r => r.Metrics.ConfidenceLabel) + .OrderBy(g => g.Key, PythonCodePointComparer.Instance).Select(g => $"{g.Key}:{g.Count()}")); + return [$"observed files={items.Count}, reads={reads}, submitted={HumanSize(requested)}", "confidence=" + confidence]; + } + + private static List DirectoryGeneralization(List recommendations, IEnumerable allIndexPaths, AMPRProfileOptions options) + { + Dictionary byPath = []; + foreach (FileRecommendation recommendation in recommendations.Where(r => r.Action != "loose")) + { + byPath[recommendation.Metrics.Relative] = recommendation; + } + + List emissions = []; + + void AppendPaths(ProfileKey key, List paths) + { + if (options.ExternalizePaths != 0 && paths.Count > options.ExternalizePaths) + { + emissions.Add(new RuleEmission(key, paths, ProfileComment(paths.Select(p => byPath[p])))); + return; + } + + for (int start = 0; start < paths.Count; start += options.MaxRuleFiles) + { + List subset = paths.GetRange(start, Math.Min(options.MaxRuleFiles, paths.Count - start)); + emissions.Add(new RuleEmission(key, subset, ProfileComment(subset.Select(p => byPath[p])))); + } + } + + void AppendByProfile(IEnumerable items) + { + Dictionary> byProfile = []; + foreach (FileRecommendation recommendation in items) + { + if (!byProfile.TryGetValue(recommendation.Key, out List? list)) + { + byProfile[recommendation.Key] = list = []; + } + + list.Add(recommendation); + } + + foreach (ProfileKey key in byProfile.Keys.Order()) + { + AppendPaths(key, [.. byProfile[key].Select(r => r.Metrics.Relative).Order(PythonCodePointComparer.Instance)]); + } + } + + if (options.PatternMode == "exact") + { + AppendByProfile(byPath.Values); + return emissions; + } + + Dictionary totalByPrefix = []; + foreach (string relative in allIndexPaths) + { + foreach (string prefix in Prefixes(relative)) + { + totalByPrefix[prefix] = totalByPrefix.GetValueOrDefault(prefix) + 1; + } + } + + // Insertion-ordered counters, like Python's dict and Counter. + Dictionary> profileByPrefix = []; + Dictionary<(string, ProfileKey), List> pathsByPrefixProfile = []; + foreach ((string relative, FileRecommendation recommendation) in byPath) + { + foreach (string prefix in Prefixes(relative)) + { + if (!profileByPrefix.TryGetValue(prefix, out List<(ProfileKey Key, int Count)>? counts)) + { + profileByPrefix[prefix] = counts = []; + } + + int at = counts.FindIndex(c => c.Key == recommendation.Key); + if (at < 0) + { + counts.Add((recommendation.Key, 1)); + } + else + { + counts[at] = (recommendation.Key, counts[at].Count + 1); + } + + if (!pathsByPrefixProfile.TryGetValue((prefix, recommendation.Key), out List? paths)) + { + pathsByPrefixProfile[(prefix, recommendation.Key)] = paths = []; + } + + paths.Add(relative); + } + } + + double threshold = options.PatternMode == "directory" ? Math.Min(options.GeneralizeCoverage, 0.50) : options.GeneralizeCoverage; + List<(int Savings, int Depth, string Prefix, ProfileKey Key, HashSet Covered)> candidates = []; + foreach ((string prefix, List<(ProfileKey Key, int Count)> counts) in profileByPrefix) + { + int total = totalByPrefix.GetValueOrDefault(prefix); + if (total <= 0) + { + continue; + } + + (ProfileKey key, int matching) = counts.MaxBy(c => c.Count); + int observed = counts.Sum(c => c.Count); + double coverage = (double)observed / total; + double purity = (double)matching / observed; + if (matching >= options.GeneralizeMinFiles && coverage >= threshold && purity >= 0.90) + { + HashSet covered = [.. pathsByPrefixProfile[(prefix, key)]]; + candidates.Add((covered.Count - 1, prefix.Split('/').Length, prefix, key, covered)); + } + } + + HashSet coveredPaths = []; + foreach ((_, _, string prefix, ProfileKey key, HashSet covered) in candidates.OrderByDescending(c => (c.Savings, c.Depth))) + { + int remaining = covered.Count(p => !coveredPaths.Contains(p)); + if (remaining < options.GeneralizeMinFiles) + { + continue; + } + + emissions.Add(new RuleEmission(key, [$"{prefix}/**"], + [ + $"generalised from {remaining} observed files under {prefix}", + $"indexed coverage={Percent((double)remaining / Math.Max(1, totalByPrefix[prefix]), 0)}", + ])); + coveredPaths.UnionWith(covered); + } + + AppendByProfile(byPath.Where(p => !coveredPaths.Contains(p.Key)).Select(p => p.Value)); + return emissions; + } + + // "a/b/c" -> "a", "a/b" (PurePosixPath parents below the full path). + private static IEnumerable Prefixes(string relative) + { + string[] parts = relative.Split('/'); + for (int depth = 1; depth < parts.Length; depth++) + { + yield return string.Join('/', parts[..depth]); + } + } + + // ---- formatting ---------------------------------------------------------------------------------------------- + + private static readonly PythonCodePointComparer StringKey = PythonCodePointComparer.Instance; + + private static string Distribution(IEnumerable values, IComparer order, Func format) where T : notnull => + string.Join(", ", values.GroupBy(v => v).OrderBy(g => g.Key, order).Select(g => $"{format(g.Key)}: {g.Count()}")); + + private static string Percent(double value, int decimals) => PythonText.FormatFixed(value * 100.0, decimals) + "%"; + + private static string Thousands(long value) => value.ToString("#,0", CultureInfo.InvariantCulture); + + private static string TomlString(string value) => PythonSortedJson.Indented(value); + + private static string Sha256(string path) + { + using FileStream stream = File.OpenRead(path); + return Convert.ToHexStringLower(SHA256.HashData(stream)); + } + + // Python Path ordering of discovered journals: by path components, code point order. + private static int ComparePathParts(string[] a, string[] b) + { + for (int i = 0; i < Math.Min(a.Length, b.Length); i++) + { + int c = PythonCodePointComparer.Instance.Compare(a[i], b[i]); + if (c != 0) + { + return c; + } + } + + return a.Length.CompareTo(b.Length); + } +} diff --git a/src/MkPFS.Build/AMPRPack/AMPRToml.cs b/src/MkPFS.Build/AMPRPack/AMPRToml.cs new file mode 100644 index 0000000..307ab53 --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/AMPRToml.cs @@ -0,0 +1,70 @@ +using System.Text; +using MkPFS.Core.AMPR; +using Tomlyn; +using Tomlyn.Model; +using Tomlyn.Parsing; +using Tomlyn.Serialization; +using Tomlyn.Syntax; + +namespace MkPFS.Build.AMPRPack; + +/// +/// TOML 1.0 reader for pack configurations. Syntax and semantic validation (duplicate keys and tables, dotted +/// keys redefined by headers) runs first because the table converter alone is lenient; Python tomllib +/// rejects the same documents, with different message text. +/// +internal static class AMPRToml +{ + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// Read a TOML file (UTF-8, like tomllib.load). + /// File path. + /// Root table. + /// The file is not valid UTF-8 or not valid TOML. + public static TomlTable Load(string path) + { + string text; + try + { + text = StrictUtf8.GetString(File.ReadAllBytes(path)); + } + catch (DecoderFallbackException exc) + { + throw new AMPRPackException($"{path} is not valid UTF-8", exc); + } + + return Parse(text, path); + } + + /// Parse TOML text. + /// Document. + /// Name used in diagnostics. + /// Root table. + /// The text is not valid TOML. + public static TomlTable Parse(string text, string sourceName) + { + DocumentSyntax document = SyntaxParser.Parse(text, sourceName, true); + if (document.HasErrors) + { + DiagnosticMessage first = document.Diagnostics.First(d => d.Kind == DiagnosticMessageKind.Error); + throw new AMPRPackException( + $"{first.Message} (at line {first.Span.Start.Line + 1}, column {first.Span.Start.Column + 1})"); + } + + try + { + return TomlSerializer.Deserialize(text, AMPRTomlContext.Default) + ?? throw new AMPRPackException("configuration root must be a TOML table"); + } + catch (TomlException exc) + { + throw new AMPRPackException(exc.Message, exc); + } + } +} + +/// Source-generated Tomlyn metadata (Native AOT). +[TomlSerializable(typeof(TomlTable))] +internal sealed partial class AMPRTomlContext : TomlSerializerContext +{ +} diff --git a/src/MkPFS.Build/AMPRPack/PythonValue.cs b/src/MkPFS.Build/AMPRPack/PythonValue.cs new file mode 100644 index 0000000..7c6442f --- /dev/null +++ b/src/MkPFS.Build/AMPRPack/PythonValue.cs @@ -0,0 +1,192 @@ +using System.Globalization; +using MkPFS.Core.AMPR; +using MkPFS.Core.Util; +using Tomlyn; +using Tomlyn.Model; + +namespace MkPFS.Build.AMPRPack; + +/// +/// Python built-in conversions (str, int, float, bool, parse_size) applied to TOML +/// values, as ampr_pack's load_config does. TOML values are , , +/// , , , , +/// and . Where Python raises TypeError or +/// AttributeError (a traceback, exit 1) this raises with the same text. +/// +internal static class PythonValue +{ + /// Python type name for messages. + public static string TypeName(object? value) => value switch + { + null => "NoneType", + string => "str", + bool => "bool", + long => "int", + double => "float", + TomlDateTime => "datetime", + TomlTable => "dict", + TomlArray or TomlTableArray => "list", + _ => value.GetType().Name, + }; + + /// Python str(value). + public static string Str(object? value) => value switch + { + string s => s, + _ => Repr(value), + }; + + /// Python repr(value). + public static string Repr(object? value) => value switch + { + null => "None", + string s => PythonText.Repr(s), + bool b => b ? "True" : "False", + long l => l.ToString(CultureInfo.InvariantCulture), + double d => PythonText.FloatRepr(d), + TomlTable t => "{" + string.Join(", ", t.Select(kv => PythonText.Repr(kv.Key) + ": " + Repr(kv.Value))) + "}", + TomlTableArray a => "[" + string.Join(", ", a.Select(Repr)) + "]", + TomlArray a => "[" + string.Join(", ", a.Select(Repr)) + "]", + _ => value.ToString() ?? string.Empty, + }; + + /// Python int(value). + public static long Int(object? value) + { + switch (value) + { + case bool b: + return b ? 1 : 0; + case long l: + return l; + case double d when double.IsNaN(d): + throw new ArgumentException("cannot convert float NaN to integer"); + case double d when double.IsInfinity(d): + throw new ArgumentException("cannot convert float infinity to integer"); + case double d: + return (long)Math.Truncate(d); + case string s: + string text = PythonText.Strip(s); + if (IsPythonNumber(text, allowFraction: false) + && long.TryParse(text.Replace("_", string.Empty, StringComparison.Ordinal), NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out long parsed)) + { + return parsed; + } + + throw new ArgumentException($"invalid literal for int() with base 10: {PythonText.Repr(s)}"); + default: + throw new ArgumentException($"int() argument must be a string, a bytes-like object or a real number, not '{TypeName(value)}'"); + } + } + + /// Python float(value). + public static double Float(object? value) + { + switch (value) + { + case bool b: + return b ? 1.0 : 0.0; + case long l: + return l; + case double d: + return d; + case string s: + string text = PythonText.Strip(s); + string lower = text.TrimStart('+', '-').ToLowerInvariant(); + if (lower is "inf" or "infinity" or "nan") + { + double special = lower == "nan" ? double.NaN : double.PositiveInfinity; + return text.StartsWith('-') ? -special : special; + } + + if (IsPythonNumber(text, allowFraction: true) + && double.TryParse(text.Replace("_", string.Empty, StringComparison.Ordinal), NumberStyles.Float, CultureInfo.InvariantCulture, out double parsed)) + { + return parsed; + } + + throw new ArgumentException($"could not convert string to float: {PythonText.Repr(s)}"); + default: + throw new ArgumentException($"float() argument must be a string or a real number, not '{TypeName(value)}'"); + } + } + + /// Python bool(value) (truthiness). + public static bool Bool(object? value) => value switch + { + null => false, + bool b => b, + long l => l != 0, + double d => d != 0.0, + string s => s.Length > 0, + TomlTable t => t.Count > 0, + TomlArray a => a.Count > 0, + TomlTableArray a => a.Count > 0, + _ => true, + }; + + /// + /// ampr_pack parse_size(value): integers pass through, strings are parsed. A TOML boolean is treated as + /// 0 or 1 (Python keeps the bool; only the canonical JSON would differ). + /// + public static long Size(object? value) => value switch + { + bool b => b ? 1 : 0, + long l => AMPRSize.Parse(l), + string s => AMPRSize.Parse(s), + _ => throw new ArgumentException($"'{TypeName(value)}' object has no attribute 'strip'"), + }; + + /// ampr_pack block_shift(value). + public static int BlockShift(object? value) => AMPRSize.BlockShift(Size(value)); + + // Python int()/float() literals: digits with single underscores between digits, optional sign, + // and for floats an optional fraction and exponent. + private static bool IsPythonNumber(string text, bool allowFraction) + { + int i = 0; + if (i < text.Length && text[i] is '+' or '-') + { + i++; + } + + bool digits = false; + bool lastDigit = false; + bool seenDot = false; + bool seenExp = false; + for (; i < text.Length; i++) + { + char c = text[i]; + if (char.IsAsciiDigit(c)) + { + digits = true; + lastDigit = true; + } + else if (c == '_' && lastDigit && i + 1 < text.Length && char.IsAsciiDigit(text[i + 1])) + { + lastDigit = false; + } + else if (allowFraction && c == '.' && !seenDot && !seenExp) + { + seenDot = true; + lastDigit = false; + } + else if (allowFraction && c is 'e' or 'E' && digits && !seenExp) + { + seenExp = true; + lastDigit = false; + digits = false; + if (i + 1 < text.Length && text[i + 1] is '+' or '-') + { + i++; + } + } + else + { + return false; + } + } + + return digits; + } +} diff --git a/src/MkPFS.Build/AmprIndex.cs b/src/MkPFS.Build/AmprIndex.cs index 80fab43..4ce1660 100644 --- a/src/MkPFS.Build/AmprIndex.cs +++ b/src/MkPFS.Build/AmprIndex.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Text; +using MkPFS.Core.AMPR; using MkPFS.Core.Diagnostics; using MkPFS.Core.Util; @@ -18,6 +19,26 @@ public static class AmprIndex /// Marker that signals an emulation build (relative to the source root). public const string FakelibMarker = "fakelib/libSceAmpr.sprx"; + /// + /// The AMPR Emu marker present in : , else + /// fakelib2/libSceAmpr.sprx (ShadowMountPlus's exclusive overlay; Python MkPFS only checks fakelib/). + /// + /// Game folder. + /// The marker, relative with /, or . + public static string? FindMarker(string sourceRoot) + { + foreach (string marker in (string[])[FakelibMarker, "fakelib2/libSceAmpr.sprx"]) + { + string path = Path.Combine(sourceRoot, marker); + if (File.Exists(path) || Directory.Exists(path)) + { + return marker; + } + } + + return null; + } + /// Header size (<8sIIQQQII). public const int HeaderSize = 48; @@ -37,20 +58,36 @@ public static class AmprIndex /// /app0/<rel>. public readonly record struct Row(long Size, long MTime, string Path); - /// Hash and collision key: forward slashes, lower case (Python ampr_key_for). + /// + /// Sort and collision key: forward slashes, only ASCII A..Z folded (ampr_emu build_ampr_index.py + /// key_for). Python MkPFS lower-cases every letter, which the console resolver does not do. + /// /// Path. /// Key. - public static string KeyFor(string path) => path.Replace('\\', '/').ToLowerInvariant(); + public static string KeyFor(string path) => + string.Create(path.Length, path, static (span, source) => + { + for (int i = 0; i < source.Length; i++) + { + char c = source[i]; + span[i] = c == '\\' ? '/' : c is >= 'A' and <= 'Z' ? (char)(c + 0x20) : c; + } + }); - /// FNV-1a 64 over the key's code points; 0 becomes 1 so empty slots stay distinct. + /// + /// FNV-1a 64 over the UTF-8 bytes of ; 0 becomes 1 so empty slots stay distinct. This is the + /// hash the ampr_emu resolver computes for a lookup; Python MkPFS hashes lower-cased code points instead, so a + /// non-ASCII path in its index is never found on the console. The offset basis is ampr_emu's + /// 1469598103934665603, not the standard FNV one that the pack manifest uses. + /// /// Path. /// Hash. public static ulong PathHash(string path) { ulong hash = 1469598103934665603UL; - foreach (Rune rune in KeyFor(path).EnumerateRunes()) + foreach (byte b in AMPRAssetPath.AsciiFoldPathBytes(path)) { - hash ^= (ulong)rune.Value; + hash ^= b; hash = unchecked(hash * 1099511628211UL); } @@ -94,12 +131,25 @@ public static int HashSlotCount(int entryCount) } string indexPath = Path.Combine(sourceRoot, IndexName); - string marker = Path.Combine(sourceRoot, "fakelib", "libSceAmpr.sprx"); - if (!File.Exists(marker) && !Directory.Exists(marker)) + if (FindMarker(sourceRoot) is not { } marker) { return null; } + // An AMPR pack set addresses files by their row in this index; rebuilding renumbers the rows (the new + // ampr_assets-*.pak files sort first) and the emulator then fails every packed read (not in Python MkPFS). + bool packSet = File.Exists(indexPath) && File.Exists(Path.Combine(sourceRoot, AMPRPack.AMPRPackConfig.DefaultIndexName)); + if (packSet && !forceRegen) + { + log.Warning($"{IndexName} kept: the AMPR packs in this folder (ampr_assets.index) are bound to it; --ampr-force-regen rebuilds it and breaks them"); + return null; + } + + if (packSet) + { + log.Warning($"Rebuilding {IndexName} although ampr_assets.index is present; the AMPR packs will stop working until they are rebuilt"); + } + if (createIfMissing && !forceRegen && (File.Exists(indexPath) || Directory.Exists(indexPath))) { if (Validate(indexPath, sourceRoot)) @@ -111,7 +161,7 @@ public static int HashSlotCount(int entryCount) log.Warning($"{IndexName} failed validation; regenerating..."); } - log.Info($"Detected {FakelibMarker}; generating {IndexName}..."); + log.Info($"Detected {marker}; generating {IndexName}..."); int count; try { @@ -134,8 +184,12 @@ public static int HashSlotCount(int entryCount) public static int Build(string root, string outputPath) { outputPath = Path.GetFullPath(outputPath); + return Write(ScanRows(root, outputPath), outputPath); + } + + private static int Write(List rows, string outputPath) + { string tmp = outputPath + ".tmp"; - List rows = ScanRows(root, outputPath); if (rows.Count == 0) { return 0; @@ -156,6 +210,31 @@ public static int Build(string root, string outputPath) return rows.Count; } + /// + /// Write the index for the tree a game will have after (for example + /// fakelib) is taken from instead of : the files under + /// that folder are listed with the overlay's sizes and times, everything else with the source's. + /// + /// Source tree. + /// Tree holding the replacement folder. + /// Folder name relative to both roots. + /// Index path. + /// Records written (0 and no file when there are no files). + public static int BuildWithOverlay(string root, string overlayRoot, string overlayDirectory, string outputPath) + { + outputPath = Path.GetFullPath(outputPath); + string prefix = KeyFor($"/app0/{overlayDirectory}/"); + List rows = [.. ScanRows(root, outputPath).Where(r => !KeyFor(r.Path).StartsWith(prefix, StringComparison.Ordinal))]; + string overlay = Path.Combine(Path.GetFullPath(overlayRoot), overlayDirectory); + if (Directory.Exists(overlay)) + { + HashSet seen = [.. rows.Select(r => KeyFor(r.Path))]; + rows.AddRange(ScanRows(overlayRoot, outputPath, overlay).Where(r => seen.Add(KeyFor(r.Path)))); + } + + return Write(rows, outputPath); + } + /// Serialize rows already in record order. /// Rows. /// Index bytes. @@ -306,15 +385,16 @@ public static bool Validate(string indexPath, string sourceRoot) live.All(row => indexed.TryGetValue(KeyFor(row.Path), out long size) && size == row.Size); } - // Files the index lists, in walk order: the index and its temp file skipped, case-insensitive duplicates + // Files the index lists, in walk order: the index and its temp file, the emulator's trace and log, and paths with + // tabs or line breaks skipped (as build_ampr_index.py and the runtime scan do); case-insensitive duplicates // dropped (first wins). - private static List ScanRows(string root, string indexPath) + private static List ScanRows(string root, string indexPath, string? walkFrom = null) { root = Path.GetFullPath(root); string tmp = indexPath + ".tmp"; List rows = []; HashSet seen = new(StringComparer.Ordinal); - foreach (string file in WalkFiles(root)) + foreach (string file in WalkFiles(walkFrom ?? root)) { if (string.Equals(file, indexPath, PathComparison) || string.Equals(file, tmp, PathComparison)) { @@ -323,7 +403,8 @@ private static List ScanRows(string root, string indexPath) string indexed = "/app0/" + Path.GetRelativePath(root, file).Replace('\\', '/'); string key = KeyFor(indexed); - if (key == $"/app0/{IndexName}" || key == $"/app0/{IndexName}.tmp" || !seen.Add(key)) + if (key == $"/app0/{IndexName}" || key == $"/app0/{IndexName}.tmp" || SkippedRootFiles.Contains(key) || + indexed.AsSpan().IndexOfAny('\t', '\n', '\r') >= 0 || !seen.Add(key)) { continue; } @@ -335,6 +416,9 @@ private static List ScanRows(string root, string indexPath) return rows; } + // Written into /app0 by debug emulator builds; never indexed. + private static readonly HashSet SkippedRootFiles = new(StringComparer.Ordinal) { "/app0/ampr_commands.bin", "/app0/apr_emu.log" }; + private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; private static void TryDelete(string path) @@ -370,7 +454,7 @@ private static void TryDelete(string path) } }); - // Python os.walk top-down with directories and files sorted by name.lower(): ignored names skipped, + // Python os.walk top-down with directories and files sorted by key_for(name): ignored names skipped, // directory links listed but not descended, file links kept when their target is a file. private static IEnumerable WalkFiles(string root) { @@ -382,7 +466,7 @@ private static IEnumerable WalkFiles(string root) string dir = pending.Pop(); List entries = [.. new DirectoryInfo(dir).EnumerateFileSystemInfos("*", options) .Where(e => !NameRules.IsIgnoredName(e.Name))]; - foreach (FileSystemInfo file in entries.Where(e => e is FileInfo).OrderBy(e => e.Name.ToLowerInvariant(), CodePointOrder)) + foreach (FileSystemInfo file in entries.Where(e => e is FileInfo).OrderBy(e => KeyFor(e.Name), CodePointOrder)) { if (file.LinkTarget is null || file.ResolveLinkTarget(returnFinalTarget: true) is FileInfo { Exists: true }) { @@ -392,7 +476,7 @@ private static IEnumerable WalkFiles(string root) // Push in reverse so subdirectories are visited in sorted order. foreach (FileSystemInfo sub in entries.Where(e => e is DirectoryInfo && e.LinkTarget is null) - .OrderByDescending(e => e.Name.ToLowerInvariant(), CodePointOrder)) + .OrderByDescending(e => KeyFor(e.Name), CodePointOrder)) { pending.Push(sub.FullName); } diff --git a/src/MkPFS.Build/AmprLibs.cs b/src/MkPFS.Build/AmprLibs.cs index 278580c..ed1dd0b 100644 --- a/src/MkPFS.Build/AmprLibs.cs +++ b/src/MkPFS.Build/AmprLibs.cs @@ -34,6 +34,14 @@ public static class AmprLibs /// Libraries copied when present, in copy order. public static IReadOnlyList LibraryNames { get; } = [AmprLibrary, "libScePlayGo.sprx"]; + /// + /// when carries AMPR Emu in fakelib/ or in + /// fakelib2/, which ShadowMountPlus mounts instead of fakelib/ when both exist. + /// + /// Game folder. + /// Whether the emulator library is present. + public static bool HasEmulator(string sourceRoot) => AmprIndex.FindMarker(sourceRoot) is not null; + /// when sce_sys holds a PlayGo chunk file (APR title). /// Game folder. /// Whether the title uses PlayGo. @@ -57,6 +65,12 @@ public static int Inject(string sourceRoot, string libsDir, IMkPFSLog log) } string fakelib = Path.Combine(sourceRoot, "fakelib"); + if (Directory.Exists(Path.Combine(sourceRoot, "fakelib2"))) + { + // ShadowMountPlus mounts fakelib2 alone when it exists, so libraries in fakelib would never load. + log.Warning("fakelib2/ exists and ShadowMountPlus mounts it instead of fakelib/; put the AMPR Emu libraries in fakelib2/ yourself"); + } + int copied = 0; try { @@ -100,7 +114,7 @@ public static int Inject(string sourceRoot, string libsDir, IMkPFSLog log) public static void Prepare(string sourceRoot, AmprOptions options, IMkPFSLog log) { bool apr = options.ForceAprTitle || IsAprTitle(sourceRoot); - bool hasLibrary = File.Exists(Path.Combine(sourceRoot, AmprIndex.FakelibMarker)); + bool hasLibrary = HasEmulator(sourceRoot); if (options.LibsDir is { } libs) { if (apr) diff --git a/src/MkPFS.Build/Exfat/ExfatImageWriter.cs b/src/MkPFS.Build/Exfat/ExfatImageWriter.cs index 0e469c0..447107e 100644 --- a/src/MkPFS.Build/Exfat/ExfatImageWriter.cs +++ b/src/MkPFS.Build/Exfat/ExfatImageWriter.cs @@ -35,15 +35,18 @@ public sealed class ExfatImageWriter private readonly long _bitmapClusters; private readonly long _upcaseClusters; private readonly long _clusterCount; + private readonly long _usedClusters; private readonly long _fatLengthSectors; private readonly long _heapOffsetSectors; - private ExfatImageWriter(Node root, int clusterSize) + private ExfatImageWriter(Node root, int clusterSize, long freeBytes) { _root = root; ClusterSize = clusterSize; long sectorsPerCluster = clusterSize / BytesPerSector; - (_bitmapClusters, _upcaseClusters, _clusterCount) = AssignClusters(root, clusterSize); + long freeClusters = Sizes.CeilDiv(Math.Max(0, freeBytes), clusterSize); + (_bitmapClusters, _upcaseClusters, _usedClusters) = AssignClusters(root, clusterSize, freeClusters); + _clusterCount = _usedClusters + freeClusters; if (_clusterCount > uint.MaxValue - 16) { throw new InvalidOperationException("source tree is too large for an exFAT volume with this cluster size"); @@ -60,12 +63,20 @@ private ExfatImageWriter(Node root, int clusterSize) /// Total image size in bytes. public long ImageSize { get; } + /// Free space left in the volume, in bytes (whole clusters). + public long FreeBytes => (_clusterCount - _usedClusters) * ClusterSize; + /// Scan and compute the layout. /// Directory whose contents become the volume root. /// Cluster size in bytes, or for . + /// + /// Free space to leave in the volume (rounded up to clusters). 0, the default, gives a tight image with no free + /// cluster, like Python MkPFS; anything that writes to the mounted image (a game, or AMPR Emu's debug log and trace + /// on a read-write mount) needs some. + /// /// Writer ready to emit the image. - public static ExfatImageWriter Plan(string sourceRoot, int? clusterSize = null) => - new(ScanTree(sourceRoot), clusterSize ?? DefaultClusterSize); + public static ExfatImageWriter Plan(string sourceRoot, int? clusterSize = null, long freeBytes = 0) => + new(ScanTree(sourceRoot), clusterSize ?? DefaultClusterSize, freeBytes); /// /// Write a complete image (Python write_exfat_image). An existing directory as @@ -75,15 +86,16 @@ public static ExfatImageWriter Plan(string sourceRoot, int? clusterSize = null) /// Image path or existing directory. /// Cluster size, or for the default. /// Optional progress, phase exfat. + /// Free space to leave in the volume (see ). /// The path written. - public static string Write(string sourceRoot, string outputPath, int? clusterSize = null, IProgressSink? progress = null) + public static string Write(string sourceRoot, string outputPath, int? clusterSize = null, IProgressSink? progress = null, long freeBytes = 0) { if (Directory.Exists(outputPath)) { outputPath = Path.Combine(outputPath, GameParams.DefaultImageBasename(sourceRoot) + ".exfat"); } - ExfatImageWriter writer = Plan(sourceRoot, clusterSize); + ExfatImageWriter writer = Plan(sourceRoot, clusterSize, freeBytes); string tempPath = outputPath + ".tmp"; try { @@ -164,6 +176,13 @@ public IEnumerable> Chunks() { yield return chunk; } + + // 4. Free clusters, zero-filled. + byte[] zeros = new byte[FileReadChunk]; + for (long left = FreeBytes; left > 0; left -= zeros.Length) + { + yield return zeros.AsMemory(0, (int)Math.Min(zeros.Length, left)); + } } private IEnumerable> EmitChildren(Node node, byte[] buffer) @@ -253,16 +272,16 @@ private static long DirectoryEntryCount(Node node, bool isRoot) private static long NodeClusters(Node node, bool isRoot, int clusterSize) => node.IsDir ? Math.Max(1, Sizes.CeilDiv(DirectoryEntryCount(node, isRoot) * 32, clusterSize)) : Sizes.CeilDiv(node.Size, clusterSize); - private static (long Bitmap, long Upcase, long Total) AssignClusters(Node root, int clusterSize) + private static (long Bitmap, long Upcase, long Used) AssignClusters(Node root, int clusterSize, long freeClusters) { long upcaseClusters = Sizes.CeilDiv(ExfatUpcase.Table.Length, clusterSize); long contentClusters = NodeClusters(root, isRoot: true, clusterSize) + Accumulate(root) + upcaseClusters; - // The bitmap must cover itself plus all content. + // The bitmap must cover itself, all content and the free clusters. long bitmapClusters = 1; while (true) { - long need = Sizes.CeilDiv(Sizes.CeilDiv(bitmapClusters + contentClusters, 8), clusterSize); + long need = Sizes.CeilDiv(Sizes.CeilDiv(bitmapClusters + contentClusters + freeClusters, 8), clusterSize); if (need == bitmapClusters) { break; @@ -495,14 +514,14 @@ void Chain(long first, long count) } } - // Every heap cluster is allocated (tight image). + // Every used heap cluster is allocated; the free ones come last (a tight image has none). private byte[] BuildBitmap() { byte[] bitmap = new byte[AlignUp(Sizes.CeilDiv(_clusterCount, 8), ClusterSize)]; - bitmap.AsSpan(0, (int)(_clusterCount / 8)).Fill(0xFF); - if (_clusterCount % 8 != 0) + bitmap.AsSpan(0, (int)(_usedClusters / 8)).Fill(0xFF); + if (_usedClusters % 8 != 0) { - bitmap[_clusterCount / 8] = (byte)((1 << (int)(_clusterCount % 8)) - 1); + bitmap[_usedClusters / 8] = (byte)((1 << (int)(_usedClusters % 8)) - 1); } return bitmap; diff --git a/src/MkPFS.Build/MkPFS.Build.csproj b/src/MkPFS.Build/MkPFS.Build.csproj index df54737..913dbf8 100644 --- a/src/MkPFS.Build/MkPFS.Build.csproj +++ b/src/MkPFS.Build/MkPFS.Build.csproj @@ -4,6 +4,10 @@ + + + + net10.0 enable diff --git a/src/MkPFS.Cli/Commands/AmprCommand.cs b/src/MkPFS.Cli/Commands/AmprCommand.cs new file mode 100644 index 0000000..c8525f5 --- /dev/null +++ b/src/MkPFS.Cli/Commands/AmprCommand.cs @@ -0,0 +1,603 @@ +using System.CommandLine; +using System.Diagnostics; +using System.Globalization; +using MkPFS.Build; +using MkPFS.Build.PFS; +using MkPFS.Build.AMPRPack; +using MkPFS.Cli.Output; +using MkPFS.Core.AMPR; +using MkPFS.Core.Diagnostics; +using MkPFS.Core.Util; + +namespace MkPFS.Cli.Commands; + +/// +/// ampr: AMPR Emu seekable LZ4 asset packs (port of ampr_emu tools/ampr_pack.py). Subcommands, options, +/// JSON (json.dumps(indent=2, ensure_ascii=False, sort_keys=True)) and error: ... lines with exit code 2 +/// match the oracle. +/// +internal static class AmprCommand +{ + /// Printed to stderr when a pack run packs nothing (an addition to the oracle output). + internal const string NothingPackedWarning = + "warning: no files were packed; without --config or --traces every file stays loose"; + + /// Printed by ampr game before it starts. + internal const string ExperimentalWarning = + "warning: AMPR packing is experimental; keep the original game until the packed one has been played on the PS5"; + + /// Printed to stderr when files were packed but the root has no AMPR Emu (an addition to the oracle output). + internal const string NoEmulatorWarning = + "warning: --root has no fakelib/libSceAmpr.sprx or fakelib2/libSceAmpr.sprx; only AMPR Emu reads these packs, " + + "and only in titles that load libSceAmpr. Keep the original files until the game runs from the packs"; + + public static Command Create(CliContext ctx) + { + Command command = new("ampr", "Build and manage AMPR Emu seekable LZ4 asset packs for /app0 (experimental)"); + command.Subcommands.Add(Pack(ctx)); + command.Subcommands.Add(Game(ctx)); + command.Subcommands.Add(Unpack(ctx)); + command.Subcommands.Add(Verify(ctx)); + command.Subcommands.Add(RemoveSources(ctx)); + command.Subcommands.Add(List(ctx)); + command.Subcommands.Add(Inspect(ctx)); + command.Subcommands.Add(RuntimeConfig(ctx)); + command.Subcommands.Add(AmprProfileCommand.Create(ctx)); + return command; + } + + private static Option Required(string name, string description) => new(name) { Description = description, Required = true }; + + private static Option Repeated(string name, string description, string? helpName = null) => + new(name) { Description = description, HelpName = helpName, AllowMultipleArgumentsPerToken = false, DefaultValueFactory = _ => [] }; + + private static Command Pack(CliContext ctx) + { + Option root = Required("--root", "deployed /app0 directory"); + Option amprIndex = Required("--ampr-index", "AMPRIDX3 file"); + Option output = Required("--output", "output directory"); + Option config = new("--config") { Description = "TOML pack configuration" }; + Option traces = new("--traces") { Description = TracesDescription }; + Option untracedTypes = new("--pack-untraced-types") { Description = UntracedTypesDescription }; + Option include = Repeated("--include", "additional include glob"); + Option exclude = Repeated("--exclude", "force-loose glob"); + Option includeFrom = new("--include-from") { Description = "newline-delimited include globs" }; + Option excludeFrom = new("--exclude-from") { Description = "newline-delimited exclude globs" }; + Option workers = new("--workers") { Description = "compression workers" }; + Option selfContained = new("--self-contained") { Description = "do not auto-loose explicitly packed files; store incompressible blocks RAW" }; + Option requirePacked = Repeated("--require-packed", "fail if a matching logical file would remain loose (repeatable)", "GLOB"); + Option allowMissing = new("--allow-missing") { Description = "leave missing selected files loose" }; + Option noProgress = new("--no-progress") { Description = "suppress build progress on stderr; final JSON still goes to stdout" }; + Command command = new("pack", "build pack index and data volumes") + { + root, amprIndex, output, config, traces, untracedTypes, include, exclude, includeFrom, excludeFrom, workers, selfContained, requirePacked, allowMissing, noProgress, + }; + command.SetAction(parse => Run(ctx, () => + { + string? configPath = parse.GetValue(config); + string? tracesDir = parse.GetValue(traces); + CheckRuleOptions(configPath, tracesDir, parse.GetValue(untracedTypes)); + AMPRPackConfig loaded = tracesDir is not null + ? TraceRules(Resolve(ctx, tracesDir), parse.GetValue(untracedTypes), ctx.Err.WriteLine) + : AMPRPackConfig.Load(configPath is null ? null : Resolve(ctx, configPath)); + if (parse.GetValue(workers) is { } count) + { + loaded.Workers = count is >= 1 and <= 256 ? count : throw new AMPRPackException("--workers must be between 1 and 256"); + } + + if (parse.GetValue(selfContained)) + { + loaded.SelfContained = true; + } + + if (parse.GetValue(requirePacked) is { Length: > 0 } required) + { + loaded.RequiredPacked = [.. loaded.RequiredPacked, .. required]; + } + + List includes = [.. parse.GetValue(include)!, .. PatternFile(ctx, parse.GetValue(includeFrom))]; + List excludes = [.. parse.GetValue(exclude)!, .. PatternFile(ctx, parse.GetValue(excludeFrom))]; + Action? progress = parse.GetValue(noProgress) || !ctx.ProgressEnabled ? null : ProgressReporter(ctx); + AMPRBuildResult result = AMPRPackBuilder.Build( + Resolve(ctx, parse.GetValue(root)!), + Resolve(ctx, parse.GetValue(amprIndex)!), + Resolve(ctx, parse.GetValue(output)!), + loaded, + includes, + excludes, + parse.GetValue(allowMissing), + progress); + AMPRBuildStats stats = result.Stats; + PrintJson(ctx, new Dictionary + { + ["files_total"] = stats.FilesTotal, + ["files_packed"] = stats.FilesPacked, + ["files_loose"] = stats.FilesLoose, + ["loose_paths"] = stats.LoosePaths, + ["files_auto_loose"] = stats.FilesAutoLoose, + ["auto_loose_logical_bytes"] = stats.AutoLooseLogicalBytes, + ["auto_loose_sampled_bytes"] = stats.AutoLooseSampledBytes, + ["chunks"] = stats.Chunks, + ["chunks_lz4"] = stats.ChunksLZ4, + ["chunks_raw"] = stats.ChunksRaw, + ["chunks_shared"] = stats.ChunksShared, + ["logical_bytes"] = stats.LogicalBytes, + ["stored_bytes"] = stats.StoredBytes, + ["padding_bytes"] = stats.PaddingBytes, + ["io_pages_touched"] = stats.IOPagesTouched, + ["io_page_safe_chunks"] = stats.IOPageSafeChunks, + ["dense_streaming_chunks"] = stats.DenseStreamingChunks, + ["index"] = result.IndexPath, + ["crc"] = AMPRPackFormat.ChunkCrcPath(result.IndexPath), + ["compression_ratio"] = stats.LogicalBytes != 0 ? (double)stats.StoredBytes / stats.LogicalBytes : 1.0, + ["warnings"] = result.Warnings, + }); + if (stats.FilesPacked == 0) + { + // Not in ampr_pack.py: a run without rules leaves every file loose and is easy to start by mistake. + ctx.Err.WriteLine(NothingPackedWarning); + } + + // Not in ampr_pack.py: sets the console cannot load, and packs in a folder without AMPR Emu. + foreach (string message in result.RuntimeLimitWarnings()) + { + ctx.Err.WriteLine($"warning: {message}"); + } + + string rootDir = Resolve(ctx, parse.GetValue(root)!); + if (stats.FilesPacked > 0 && !AmprLibs.HasEmulator(rootDir)) + { + ctx.Err.WriteLine(NoEmulatorWarning); + } + })); + return command; + } + + private static Command Game(CliContext ctx) + { + Option root = Required("--root", "unpacked game folder (/app0); only read"); + Option output = Required("--output", "new or empty folder for the playable game"); + Option fakelib = new("--fakelib") { Description = "folder with AMPR Emu 0.4.2.1+ libSceAmpr.sprx and other fakelib libraries to add" }; + Option config = new("--config") { Description = "TOML pack profile for the game (or --traces)" }; + Option traces = new("--traces") { Description = TracesDescription }; + Option untracedTypes = new("--pack-untraced-types") { Description = UntracedTypesDescription }; + Option workers = new("--workers") { Description = "compression workers" }; + Option selfContained = new("--self-contained") { Description = "do not auto-loose explicitly packed files; store incompressible blocks RAW" }; + Option exfat = new("--exfat") { Description = "also build an exFAT image of the output (file, or folder for .exfat)" }; + Option exfatFree = new("--exfat-free-space") { Description = "free space to leave inside the exFAT image, e.g. 2GiB (default: 0); a debug run of AMPR Emu writes its log there" }; + Option skipVerify = new("--skip-verify") { Description = "do not verify the packs and loose files after the build" }; + Option noProgress = new("--no-progress") { Description = "suppress progress on stderr" }; + Command command = new("game", "build a folder that runs from packs as is: libraries, ampr_emu.index, packs and loose files (not in ampr_pack.py)") + { + root, output, fakelib, config, traces, untracedTypes, workers, selfContained, exfat, exfatFree, skipVerify, noProgress, + }; + command.SetAction(parse => Run(ctx, () => + { + ctx.Log.Warning(ExperimentalWarning); + string? configPath = parse.GetValue(config); + string? tracesDir = parse.GetValue(traces); + CheckRuleOptions(configPath, tracesDir, parse.GetValue(untracedTypes)); + AMPRPackConfig loaded = tracesDir is not null ? TraceRules(Resolve(ctx, tracesDir), parse.GetValue(untracedTypes), message => ctx.Log.Info(message)) + : configPath is not null ? Config(ctx, Resolve(ctx, configPath)) + : throw new AMPRPackException(NoRulesError); + if (parse.GetValue(workers) is { } count) + { + loaded.Workers = count is >= 1 and <= 256 ? count : throw new AMPRPackException("--workers must be between 1 and 256"); + } + + if (parse.GetValue(selfContained)) + { + loaded.SelfContained = true; + } + + bool progress = !parse.GetValue(noProgress) && ctx.ProgressEnabled; + AMPRGameOptions options = new() + { + Config = loaded, + LibsDir = parse.GetValue(fakelib) is { } libs ? Resolve(ctx, libs) : null, + Verify = !parse.GetValue(skipVerify), + ExfatImage = parse.GetValue(exfat) is { } image ? Resolve(ctx, image) : null, + ExfatFreeBytes = parse.GetValue(exfatFree) is { } free + ? parse.GetValue(exfat) is not null ? AMPRSize.Parse(free) : throw new AMPRPackException("--exfat-free-space needs --exfat") + : 0, + }; + try + { + AMPRGameBuilder.Build( + Resolve(ctx, parse.GetValue(root)!), + Resolve(ctx, parse.GetValue(output)!), + options, + ctx.Log, + progress ? ProgressReporter(ctx) : null, + ctx.CreateProgress(progress)); + } + catch (BuildException ex) + { + throw new AMPRPackException(ex.Message); + } + })); + return command; + } + + /// ampr game without rules: nothing would be packed. + internal const string NoRulesError = + "ampr game needs the rules to pack with: --config for the game, or --traces of APR traces"; + + private static void CheckRuleOptions(string? configPath, string? tracesDir, bool untracedTypes) + { + if (untracedTypes && tracesDir is null) + { + throw new AMPRPackException("--pack-untraced-types needs --traces"); + } + + if (tracesDir is not null && configPath is not null) + { + throw new AMPRPackException("--traces cannot be used with --config"); + } + } + + private const string TracesDescription = + "folder with APR traces of the debug emulator build (ampr_commands.bin and ampr_emu.index, one run per subfolder): " + + "pack the files the game read through APR, as ampr_pack_profile.py would, and keep the rest loose"; + + internal const string UntracedTypesDescription = + "with --traces: also pack the files no trace read whose type (extension) a traced file has, packed like most traced " + + "files of that type (not for files without an extension, executables, modules, sce_sys or fakelib)"; + + // Not in ampr_pack.py: the rules ampr_pack_profile.py would write for the traces, without the intermediate TOML. + private static AMPRPackConfig TraceRules(string directory, bool untracedTypes, Action report) + { + List runs = AMPRProfiler.DiscoverTracePairs(directory); + if (runs.Count == 0) + { + throw new AMPRPackException($"no traces found in {directory}: each run needs ampr_commands.bin and its ampr_emu.index side by side"); + } + + AMPRProfileOptions options = new() { Name = Path.GetFileName(Path.TrimEndingDirectorySeparator(directory)) }; + AMPRProfileResult profile = AMPRProfiler.Build(runs, options); + string runText = runs.Count == 1 ? "1 trace run" : $"{runs.Count} trace runs"; + string fileText = profile.PackedFiles == 1 ? "1 file" : $"{profile.PackedFiles} files"; + string toml; + if (untracedTypes) + { + toml = AMPRProfiler.RenderToml(profile, options, runs, null, "profile", out AMPRProfiler.UntracedAddition added); + report($"Rules: from {runText} in {directory}; {fileText} the game read through APR are packed " + + $"({AMPRProfiler.HumanSize(profile.PackedFileBytes)}), plus {added.Files} untraced files of the same types " + + $"({AMPRProfiler.HumanSize(added.Bytes)}); every other file stays loose"); + if (profile.ProjectedIndexBytes + added.IndexBytes > options.IndexBudget) + { + report($"warning: the pack manifest grows to about {AMPRProfiler.HumanSize(profile.ProjectedIndexBytes + added.IndexBytes)}, " + + $"over the {AMPRProfiler.HumanSize(options.IndexBudget)} the emulator keeps resident"); + } + } + else + { + toml = AMPRProfiler.RenderToml(profile, options); + report($"Rules: from {runText} in {directory}; {fileText} the game read through APR are packed " + + $"({AMPRProfiler.HumanSize(profile.PackedFileBytes)}), every other file stays loose"); + } + + if (profile.Warnings.Count > 0) + { + report($"warning: the traces have {profile.Warnings.Count} warnings; `mkpfs ampr profile generate --report` lists them"); + } + + return AMPRProfiler.ToConfig(toml); + } + + private static AMPRPackConfig Config(CliContext ctx, string path) + { + ctx.Log.Info($"Rules: TOML profile {path}"); + return AMPRPackConfig.Load(path); + } + + private static Command Unpack(CliContext ctx) + { + Option index = Required("--index", "pack manifest"); + Option output = Required("--output", "output directory"); + Option file = Repeated("--file", "file glob"); + Option overwrite = new("--overwrite") { Description = "replace existing files" }; + Option noMTime = new("--no-preserve-mtime") { Description = "do not restore file modification times" }; + Command command = new("unpack", "extract packed files") { index, output, file, overwrite, noMTime }; + command.SetAction(parse => Run(ctx, () => + { + AMPRExtractResult result = AMPRPackTools.Extract( + Resolve(ctx, parse.GetValue(index)!), Resolve(ctx, parse.GetValue(output)!), parse.GetValue(file), parse.GetValue(overwrite), !parse.GetValue(noMTime)); + PrintJson(ctx, new Dictionary { ["files"] = result.Files, ["bytes"] = result.Bytes }); + })); + return command; + } + + private static Command Verify(CliContext ctx) + { + Option index = Required("--index", "pack manifest"); + Option file = Repeated("--file", "file glob"); + Option root = new("--root") { Description = "also reconstruct packed files and compare byte-for-byte with this /app0 root" }; + Command command = new("verify", "validate every selected packed block") { index, file, root }; + command.SetAction(parse => Run(ctx, () => + { + string indexPath = Resolve(ctx, parse.GetValue(index)!); + AMPRVerifyResult verify = AMPRPackTools.Verify(indexPath, parse.GetValue(file)); + Dictionary result = new() + { + ["files"] = verify.Files, + ["physical_chunks"] = verify.PhysicalChunks, + ["stored_bytes"] = verify.StoredBytes, + ["raw_bytes"] = verify.RawBytes, + }; + if (parse.GetValue(root) is { } sourceRoot) + { + AMPRSourceCompareResult compare = AMPRPackTools.VerifyAgainstRoot(indexPath, Resolve(ctx, sourceRoot), parse.GetValue(file)); + result["source_compare"] = new Dictionary { ["files"] = compare.Files, ["chunks"] = compare.Chunks, ["bytes"] = compare.Bytes }; + } + + PrintJson(ctx, result); + })); + return command; + } + + private static Command RemoveSources(CliContext ctx) + { + Option index = Required("--index", "pack manifest"); + Option root = Required("--root", "/app0 source directory"); + Option confirm = new("--confirm") { Description = "perform permanent removal; without it, print a read-only plan" }; + Option removeEmptyDirs = new("--remove-empty-dirs") { Description = "remove directories left empty by source removal, never the /app0 root" }; + Command command = new("remove-sources", "verify, then remove local /app0 sources represented by PACK records") { index, root, confirm, removeEmptyDirs }; + command.Aliases.Add("remove-packed-sources"); + command.SetAction(parse => Run(ctx, () => + { + string indexPath = Resolve(ctx, parse.GetValue(index)!); + string rootPath = Resolve(ctx, parse.GetValue(root)!); + if (parse.GetValue(confirm)) + { + AMPRRemovalResult removed = AMPRPackMaintenance.RemovePackedSources(indexPath, rootPath, parse.GetValue(removeEmptyDirs)); + PrintJson(ctx, new Dictionary + { + ["files"] = removed.Files, + ["bytes"] = removed.Bytes, + ["directories"] = removed.Directories, + ["verified"] = removed.Verified, + }); + return; + } + + AMPRRemovalPlan plan = AMPRPackMaintenance.RemovalPlan(indexPath, rootPath); + PrintJson(ctx, new Dictionary + { + ["files"] = plan.Files, + ["present_files"] = plan.PresentFiles, + ["missing_files"] = plan.MissingFiles, + ["bytes"] = plan.Bytes, + ["paths"] = plan.Paths, + ["dry_run"] = true, + }); + })); + return command; + } + + private static Command List(CliContext ctx) + { + Option index = Required("--index", "pack manifest"); + Option file = Repeated("--file", "file glob"); + Option json = new("--json") { Description = "print JSON" }; + Command command = new("list", "list logical files and placement") { index, file, json }; + command.SetAction(parse => Run(ctx, () => + { + List rows = AMPRPackTools.List(Resolve(ctx, parse.GetValue(index)!), parse.GetValue(file)); + if (parse.GetValue(json)) + { + PrintJson(ctx, rows.Select(row => new Dictionary + { + ["file_id"] = row.FileId, + ["path"] = row.Path, + ["packed"] = row.Packed, + ["logical_size"] = row.LogicalSize, + ["stored_size"] = row.StoredSize, + ["block_size"] = row.BlockSize, + ["chunks"] = row.Chunks, + ["codecs"] = row.Codecs, + ["packs"] = row.Packs, + ["io_page_sizes"] = row.IOPageSizes, + ["layout"] = row.Layout, + ["streaming"] = row.Streaming, + ["random_access"] = row.RandomAccess, + ["hot"] = row.Hot, + }).ToList()); + return; + } + + foreach (AMPRListEntry row in rows) + { + ctx.Out.WriteLine(string.Create( + CultureInfo.InvariantCulture, + $"{row.FileId,8} {(row.Packed ? "PACK" : "LOOSE"),-5} {row.LogicalSize,12} {row.StoredSize,12} {row.BlockSize,8} {row.Path}")); + } + })); + return command; + } + + private static Command Inspect(CliContext ctx) + { + Option index = Required("--index", "pack manifest"); + Command command = new("inspect", "show manifest-level summary") { index }; + command.SetAction(parse => Run(ctx, () => + { + AMPRInspectResult result = AMPRPackTools.Inspect(Resolve(ctx, parse.GetValue(index)!)); + PrintJson(ctx, new Dictionary + { + ["build_id"] = result.BuildId, + ["runtime"] = RuntimeJson(result.Runtime), + ["files"] = result.Files, + ["packed_files"] = result.PackedFiles, + ["loose_files"] = result.LooseFiles, + ["chunks"] = result.Chunks, + ["packs"] = result.Packs.Select(pack => new Dictionary + { + ["id"] = pack.Id, + ["name"] = pack.Name, + ["file_size"] = pack.FileSize, + ["payload_bytes"] = pack.PayloadBytes, + ["io_page_size"] = pack.IOPageSize, + ["io_pages"] = pack.IOPages, + ["flags"] = pack.Flags, + }).ToList(), + }); + })); + return command; + } + + private static Command RuntimeConfig(CliContext ctx) + { + Option index = Required("--index", "pack manifest"); + Option config = Required("--config", "TOML file with [runtime]"); + Command command = new("runtime-config", "atomically update per-pack runtime settings without repacking") { index, config }; + command.SetAction(parse => Run(ctx, () => + { + string typed = parse.GetValue(index)!; + AMPRRuntimeConfigResult result = AMPRPackMaintenance.WriteRuntimeConfig(Resolve(ctx, typed), Resolve(ctx, parse.GetValue(config)!)); + // Python prints Path(str(args.index) + ".runtime"): the path as typed. + PrintJson(ctx, new Dictionary + { + ["path"] = PythonPathText(typed + ".runtime"), + ["build_id"] = result.BuildId, + ["runtime"] = RuntimeJson(result.Runtime), + }); + })); + return command; + } + + // Python main(): PackError, ValueError, OSError and RuntimeError print "error: " and exit 2. + internal static int Run(CliContext ctx, Action action) + { + try + { + action(); + return 0; + } + catch (Exception exc) when (exc is AMPRPackException or ArgumentException or InvalidDataException or IOException + or UnauthorizedAccessException or InvalidOperationException) + { + ctx.Err.WriteLine($"error: {exc.Message}"); + return 2; + } + } + + private static void PrintJson(CliContext ctx, object value) => ctx.Out.WriteLine(PythonSortedJson.Indented(value)); + + private static Dictionary? RuntimeJson(AMPRRuntimeSettings? settings) => settings is null ? null : new() + { + ["decoded_cache_bytes"] = settings.DecodedCacheBytes, + ["physical_cache_bytes"] = settings.PhysicalCacheBytes, + ["workers"] = settings.Workers, + ["latency_reserve_workers"] = settings.LatencyReserveWorkers, + }; + + internal static string Resolve(CliContext ctx, string path) => Path.GetFullPath(PathRules.ExpandUser(path), ctx.WorkingDirectory); + + private static List PatternFile(CliContext ctx, string? path) => + path is null ? [] : AMPRPackConfig.LoadPatternFile(Resolve(ctx, path)); + + // str(pathlib.Path(text)): platform separators, no empty or "." components. + private static string PythonPathText(string text) + { + char separator = Path.DirectorySeparatorChar; + string normalized = OperatingSystem.IsWindows() ? text.Replace('/', '\\') : text; + string root = Path.GetPathRoot(normalized) ?? string.Empty; + string[] parts = normalized[root.Length..].Split(separator, StringSplitOptions.RemoveEmptyEntries).Where(p => p != ".").ToArray(); + string joined = root + string.Join(separator, parts); + return joined.Length == 0 ? "." : joined; + } + + // Python ConsoleBuildProgress: throttled "[pack NN%] phase: details, elapsed HH:MM:SS" lines on stderr, which the + // GUI shows in its log. The GUI bar gets one overall "pack" phase with the same percentage on every event (the + // sink also cancels the job), so it does not log a misleading "done" line per build phase. + private static Action ProgressReporter(CliContext ctx) + { + IProgressSink? sink = ctx.ExternalProgressSink; + Stopwatch clock = Stopwatch.StartNew(); + TimeSpan lastOutput = TimeSpan.Zero; + int lastPercent = -1; + string lastPhase = string.Empty; + TimeSpan? packingStarted = null; + return progress => + { + TimeSpan now = clock.Elapsed; + int percent = Percent(progress); + sink?.Step("pack", percent, 100, progress.LogicalBytesDone); + bool phaseChanged = progress.Phase != lastPhase; + if (progress.Phase == "packing" && packingStarted is null) + { + packingStarted = now; + } + + if (!phaseChanged && progress.Phase != "complete") + { + TimeSpan since = now - lastOutput; + if (since.TotalSeconds < 0.5 || (percent <= lastPercent && since.TotalSeconds < 5.0)) + { + return; + } + } + + string details = $"files {progress.FilesDone}/{progress.FilesTotal}"; + if (progress.Phase == "packing") + { + details += $", {Size(progress.LogicalBytesDone)}/{Size(progress.LogicalBytesTotal)}"; + double seconds = packingStarted is { } started ? (now - started).TotalSeconds : 0.0; + if (seconds > 0 && progress.LogicalBytesDone > 0) + { + double rate = progress.LogicalBytesDone / seconds; + long eta = rate > 0 ? (long)(Math.Max(0, progress.LogicalBytesTotal - progress.LogicalBytesDone) / rate) : 0; + details += $", {Size((long)rate)}/s, ETA {Clock(eta)}"; + } + } + + if (progress.CurrentPath.Length > 0) + { + details += $", current={progress.CurrentPath}"; + } + + ctx.Err.WriteLine($"[pack {percent,3}%] {progress.Phase}: {details}, elapsed {Clock((long)now.TotalSeconds)}"); + ctx.Err.Flush(); + lastOutput = now; + lastPercent = percent; + lastPhase = progress.Phase; + }; + } + + private static int Percent(AMPRBuildProgress progress) + { + switch (progress.Phase) + { + case "planning": + double planned = progress.FilesTotal != 0 ? (double)progress.FilesDone / progress.FilesTotal : 1.0; + return Math.Min(10, (int)(planned * 10)); + case "packing": + double packed = progress.LogicalBytesTotal != 0 + ? (double)progress.LogicalBytesDone / progress.LogicalBytesTotal + : progress.FilesTotal != 0 ? (double)progress.FilesDone / progress.FilesTotal : 1.0; + return Math.Min(95, 10 + (int)(packed * 85)); + default: + return progress.Phase switch { "finalizing" => 96, "publishing" => 99, "complete" => 100, _ => 0 }; + } + } + + private static string Clock(long seconds) => string.Create(CultureInfo.InvariantCulture, $"{seconds / 3600:00}:{seconds / 60 % 60:00}:{seconds % 60:00}"); + + // Python _progress_size. + private static string Size(long value) + { + double amount = value; + foreach (string suffix in (string[])["B", "KiB", "MiB", "GiB", "TiB"]) + { + if (amount < 1024.0 || suffix == "TiB") + { + return suffix == "B" ? $"{PythonText.FormatFixed(amount, 0)} B" : $"{PythonText.FormatFixed(amount, 1)} {suffix}"; + } + + amount /= 1024.0; + } + + throw new UnreachableException(); + } +} diff --git a/src/MkPFS.Cli/Commands/AmprProfileCommand.cs b/src/MkPFS.Cli/Commands/AmprProfileCommand.cs new file mode 100644 index 0000000..94599d8 --- /dev/null +++ b/src/MkPFS.Cli/Commands/AmprProfileCommand.cs @@ -0,0 +1,396 @@ +using System.CommandLine; +using System.Globalization; +using System.Runtime.ExceptionServices; +using MkPFS.Build.AMPRPack; +using MkPFS.Cli.Output; +using MkPFS.Core.AMPR; +using MkPFS.Core.Util; + +namespace MkPFS.Cli.Commands; + +/// +/// ampr profile: port of ampr_emu tools/ampr_pack_profile.py. generate reads APR traces +/// (ampr_commands.bin plus the run's ampr_emu.index) recorded by the debug emulator build and writes a +/// TOML pack profile, optionally the Markdown report, the metrics JSON and the runtime header; batch writes +/// one profile per trace run found in a folder or ZIP support bundle. +/// +internal static class AmprProfileCommand +{ + /// Python DEFAULT_BATCH_JOBS. + private static readonly int DefaultBatchJobs = Math.Min(4, Math.Max(1, Environment.ProcessorCount / 2)); + + public static Command Create(CliContext ctx) + { + Command command = new("profile", "generate pack rules from APR traces of the debug emulator build (ampr_pack_profile.py)"); + command.Subcommands.Add(Generate(ctx)); + command.Subcommands.Add(Batch(ctx)); + return command; + } + + // Python _add_common_options; batch defaults to hybrid patterns without the cache simulation. + private sealed class CommonOptions + { + private readonly Option _ioPageSize = new("--io-page-size") { DefaultValueFactory = _ => "64KiB" }; + private readonly Option _indexBudget = new("--pack-index-budget") { DefaultValueFactory = _ => "96MiB" }; + private readonly Option _cacheCandidates = new("--cache-candidates") { Description = "comma-separated capacities, e.g. 64MiB,128MiB,256MiB,512MiB", HelpName = "LIST" }; + private readonly Option _lanes = new("--lanes") { Description = "0 selects 1-4 lanes automatically", DefaultValueFactory = _ => 0 }; + private readonly Option _workers = new("--workers") { DefaultValueFactory = _ => 8 }; + private readonly Option _runtimeWorkers = new("--runtime-workers") { Description = "0 selects 2-8 runtime decode/I/O workers from the trace", DefaultValueFactory = _ => 0 }; + private readonly Option _strategy = new("--strategy") { Description = "trade read amplification against resident index size", DefaultValueFactory = _ => "balanced" }; + private readonly Option _patternMode; + private readonly Option _minReads = new("--min-reads") { DefaultValueFactory = _ => 1 }; + private readonly Option _minRequestedBytes = new("--min-requested-bytes") { DefaultValueFactory = _ => "1B" }; + private readonly Option _maxRuleFiles = new("--max-rule-files") { DefaultValueFactory = _ => 256 }; + private readonly Option _externalizePaths = new("--externalize-paths") + { + Description = "write larger exact include lists to portable include_from sidecars (0 keeps paths inline)", + DefaultValueFactory = _ => 128, + }; + private readonly Option _generalizeCoverage = new("--generalize-coverage") { DefaultValueFactory = _ => 0.80 }; + private readonly Option _generalizeMinFiles = new("--generalize-min-files") { DefaultValueFactory = _ => 4 }; + private readonly Option _cacheSim = new("--cache-sim") { Description = "simulate decoded-cache reuse (default for generate)" }; + private readonly Option _noCacheSim = new("--no-cache-sim") { Description = "skip the expensive decoded-cache simulation" }; + private readonly Option _cacheMaxTouches = new("--cache-max-touches") + { + Description = "simulate all decoded-cache block touches up to this limit; larger traces use deterministic contiguous windows (0 = exact)", + DefaultValueFactory = _ => AMPRProfiler.DefaultCacheMaxTouches, + }; + private readonly Option _contentRoot = new("--content-root") { Description = "optional extracted /app0 root for LZ4 sampling" }; + private readonly Option _sampleBudget = new("--sample-budget") { DefaultValueFactory = _ => "0B" }; + private readonly Option _sampleBlocksPerFile = new("--sample-blocks-per-file") { DefaultValueFactory = _ => 4 }; + private readonly Option _sampleMode = new("--sample-mode") { DefaultValueFactory = _ => "hc" }; + private readonly Option _sampleLevel = new("--sample-level") { DefaultValueFactory = _ => 12 }; + private readonly Option _sampleAcceleration = new("--sample-acceleration") { DefaultValueFactory = _ => 1 }; + private readonly bool _cacheSimulationDefault; + + public CommonOptions(string patternModeDefault, bool cacheSimulationDefault) + { + _patternMode = new("--pattern-mode") { Description = "exact is safest; hybrid generalises only high-coverage directories", DefaultValueFactory = _ => patternModeDefault }; + _patternMode.AcceptOnlyFromAmong("exact", "hybrid", "directory"); + _strategy.AcceptOnlyFromAmong("conservative", "balanced", "aggressive"); + _sampleMode.AcceptOnlyFromAmong("fast", "hc"); + _cacheSimulationDefault = cacheSimulationDefault; + } + + public Option FullMetrics { get; } = new("--full-metrics") { Description = "include all seven block-size candidates for every file in JSON" }; + + public Option Overwrite { get; } = new("--overwrite"); + + public void AddTo(Command command) + { + foreach (Option option in (Option[])[ + _ioPageSize, _indexBudget, _cacheCandidates, _lanes, _workers, _runtimeWorkers, _strategy, _patternMode, _minReads, _minRequestedBytes, + _maxRuleFiles, _externalizePaths, _generalizeCoverage, _generalizeMinFiles, _cacheSim, _noCacheSim, _cacheMaxTouches, _contentRoot, + _sampleBudget, _sampleBlocksPerFile, _sampleMode, _sampleLevel, _sampleAcceleration, FullMetrics, Overwrite]) + { + command.Options.Add(option); + } + } + + // Python _profile_options_from_args. + public AMPRProfileOptions Read(CliContext ctx, ParseResult parse, string name) + { + if (parse.GetValue(_cacheSim) && parse.GetValue(_noCacheSim)) + { + throw new AMPRPackException("--cache-sim and --no-cache-sim cannot be used together"); + } + + return new AMPRProfileOptions + { + Name = name, + IOPageSize = AMPRSize.Parse(parse.GetValue(_ioPageSize)!), + IndexBudget = AMPRSize.Parse(parse.GetValue(_indexBudget)!), + CacheCandidates = parse.GetValue(_cacheCandidates) is { } list ? CacheCandidates(list) : AMPRProfiler.DefaultCacheCandidates, + Lanes = parse.GetValue(_lanes), + Workers = parse.GetValue(_workers), + RuntimeWorkers = parse.GetValue(_runtimeWorkers), + Strategy = parse.GetValue(_strategy)!, + PatternMode = parse.GetValue(_patternMode)!, + MinReads = parse.GetValue(_minReads), + MinRequestedBytes = AMPRSize.Parse(parse.GetValue(_minRequestedBytes)!), + MaxRuleFiles = parse.GetValue(_maxRuleFiles), + ExternalizePaths = parse.GetValue(_externalizePaths), + GeneralizeCoverage = parse.GetValue(_generalizeCoverage), + GeneralizeMinFiles = parse.GetValue(_generalizeMinFiles), + CacheSimulation = parse.GetValue(_cacheSim) || (!parse.GetValue(_noCacheSim) && _cacheSimulationDefault), + CacheMaxTouches = parse.GetValue(_cacheMaxTouches), + ContentRoot = parse.GetValue(_contentRoot) is { } content ? AmprCommand.Resolve(ctx, content) : null, + SampleBudget = AMPRSize.Parse(parse.GetValue(_sampleBudget)!), + SampleBlocksPerFile = parse.GetValue(_sampleBlocksPerFile), + SampleMode = parse.GetValue(_sampleMode)!, + SampleLevel = parse.GetValue(_sampleLevel), + SampleAcceleration = parse.GetValue(_sampleAcceleration), + }; + } + } + + private static Command Generate(CliContext ctx) + { + Argument input = new("input") { Description = "directory containing ampr_commands.bin and ampr_emu.index", Arity = ArgumentArity.ZeroOrOne }; + Option trace = new("--trace") + { + Description = "COMMANDS INDEX; repeat to merge several gameplay traces", + AllowMultipleArgumentsPerToken = true, + Arity = ArgumentArity.OneOrMore, + DefaultValueFactory = _ => [], + }; + Option name = new("--name"); + Option output = new("--output") { Required = true }; + Option report = new("--report"); + Option metrics = new("--metrics"); + Option runtimeHeader = new("--runtime-header") { Description = "optional force-include header with runtime cache/pool/coalescing defines" }; + Option untracedTypes = new("--pack-untraced-types") { Description = AmprCommand.UntracedTypesDescription + " (MkPFS extension)" }; + CommonOptions common = new("exact", cacheSimulationDefault: true); + Command command = new("generate", "generate one profile") { input, trace, name, output, report, metrics, runtimeHeader, untracedTypes }; + common.AddTo(command); + command.SetAction(parse => AmprCommand.Run(ctx, () => + { + List traces = []; + if (parse.GetValue(input) is { } inputPath) + { + List discovered = AMPRProfiler.DiscoverTracePairs(AmprCommand.Resolve(ctx, inputPath)); + if (discovered.Count != 1) + { + throw new AMPRPackException($"generate input must contain exactly one trace pair; found {discovered.Count}"); + } + + traces.AddRange(discovered); + } + + string[] pairs = parse.GetValue(trace)!; + if (pairs.Length % 2 != 0) + { + throw new AMPRPackException("--trace takes two paths: COMMANDS INDEX"); + } + + for (int i = 0; i < pairs.Length; i += 2) + { + traces.Add(new AMPRTraceSpec($"trace-{(i / 2) + 1}", AmprCommand.Resolve(ctx, pairs[i]), AmprCommand.Resolve(ctx, pairs[i + 1]))); + } + + if (traces.Count == 0) + { + throw new AMPRPackException("provide an input directory or at least one --trace pair"); + } + + foreach (AMPRTraceSpec spec in traces) + { + if (!File.Exists(spec.Commands) || !File.Exists(spec.Index)) + { + throw new AMPRPackException($"missing trace input: {spec.Commands} / {spec.Index}"); + } + } + + AMPRProfileOptions options = common.Read(ctx, parse, parse.GetValue(name) ?? traces[0].Name); + options.Validate(); + AMPRProfileResult result = AMPRProfiler.Build(traces, options); + string outputText = parse.GetValue(output)!; + string outputPath = AmprCommand.Resolve(ctx, outputText); + Dictionary sidecars = []; + AMPRProfiler.UntracedAddition? added = null; + string toml; + if (parse.GetValue(untracedTypes)) + { + toml = AMPRProfiler.RenderToml(result, options, traces, sidecars, Path.GetFileNameWithoutExtension(outputPath), out AMPRProfiler.UntracedAddition addition); + added = addition; + } + else + { + toml = AMPRProfiler.RenderToml(result, options, sidecars, Path.GetFileNameWithoutExtension(outputPath)); + } + + bool replace = parse.GetValue(common.Overwrite); + WriteBundle(outputPath, toml, sidecars, replace); + if (parse.GetValue(report) is { } reportPath) + { + WriteText(AmprCommand.Resolve(ctx, reportPath), AMPRProfiler.RenderReport(result), replace); + } + + if (parse.GetValue(metrics) is { } metricsPath) + { + WriteText(AmprCommand.Resolve(ctx, metricsPath), AMPRProfiler.RenderMetricsJson(result, parse.GetValue(common.FullMetrics)), replace); + } + + if (parse.GetValue(runtimeHeader) is { } headerPath) + { + WriteText(AmprCommand.Resolve(ctx, headerPath), AMPRProfiler.RenderRuntimeHeader(result), replace); + } + + ctx.Out.WriteLine( + $"generated {outputText}: files={result.ObservedFiles} index={AMPRProfiler.HumanSize(result.ProjectedIndexBytes)} " + + $"cache={AMPRProfiler.HumanSize(result.RecommendedCacheBytes)} pool={AMPRProfiler.HumanSize(result.RecommendedPoolBytes)}"); + if (added is { } extra) + { + // Not in ampr_pack_profile.py. + ctx.Out.WriteLine( + $"added {extra.Files} untraced files of traced types: {AMPRProfiler.HumanSize(extra.Bytes)}, " + + $"index +{AMPRProfiler.HumanSize(extra.IndexBytes)}"); + } + })); + return command; + } + + private static Command Batch(CliContext ctx) + { + Argument input = new("input"); + Option outputDir = new("--output-dir") { Required = true }; + Option summary = new("--summary"); + Option batchJobs = new("--batch-jobs") + { + Description = $"parallel title profiles (0 = auto, currently {DefaultBatchJobs}); reduce this for memory-constrained hosts", + DefaultValueFactory = _ => 0, + }; + CommonOptions common = new("hybrid", cacheSimulationDefault: false); + Command command = new("batch", "scan a directory or ZIP and generate one profile per subdirectory") { input, outputDir, summary, batchJobs }; + common.AddTo(command); + command.SetAction(parse => AmprCommand.Run(ctx, () => + { + string inputPath = AmprCommand.Resolve(ctx, parse.GetValue(input)!); + string output = AmprCommand.Resolve(ctx, parse.GetValue(outputDir)!); + Directory.CreateDirectory(output); + string temp = Directory.CreateTempSubdirectory("ampr-profile-").FullName; + try + { + string scanRoot = File.Exists(inputPath) ? AMPRProfiler.ExtractTraceArchive(inputPath, temp) : inputPath; + List pairs = AMPRProfiler.DiscoverTracePairs(scanRoot); + if (pairs.Count == 0) + { + throw new AMPRPackException($"no trace pairs found under {inputPath}"); + } + + Dictionary usedNames = []; + List<(AMPRTraceSpec Pair, AMPRProfileOptions Options)> tasks = []; + foreach (AMPRTraceSpec pair in pairs) + { + string baseName = pair.Name.Length > 0 ? pair.Name : Path.GetFileName(Path.GetDirectoryName(pair.Commands)!) is { Length: > 0 } dir ? dir : "profile"; + int count = usedNames[baseName] = usedNames.GetValueOrDefault(baseName) + 1; + AMPRProfileOptions options = common.Read(ctx, parse, count == 1 ? baseName : $"{baseName}-{count}"); + // Batch content roots are ambiguous; sampling needs generate with one trace. + options.ContentRoot = null; + options.SampleBudget = 0; + options.Validate(); + tasks.Add((pair, options)); + } + + bool fullMetrics = parse.GetValue(common.FullMetrics); + bool replace = parse.GetValue(common.Overwrite); + int jobs = parse.GetValue(batchJobs) is > 0 and int requested ? requested : DefaultBatchJobs; + jobs = Math.Min(jobs, tasks.Count); + PythonJsonObject[] rows = new PythonJsonObject[tasks.Count]; + if (jobs <= 1) + { + for (int i = 0; i < tasks.Count; i++) + { + rows[i] = BuildBatchProfile(tasks[i].Pair, tasks[i].Options, output, fullMetrics, replace); + ctx.Out.WriteLine($"generated {tasks[i].Options.Name}.toml"); + } + } + else + { + // Titles are independent; results and the first error come in input order, like + // ProcessPoolExecutor.map. + Exception?[] errors = new Exception?[tasks.Count]; + Parallel.For(0, tasks.Count, new ParallelOptions { MaxDegreeOfParallelism = jobs }, i => + { + try + { + rows[i] = BuildBatchProfile(tasks[i].Pair, tasks[i].Options, output, fullMetrics, replace); + } + catch (Exception exc) + { + errors[i] = exc; + } + }); + for (int i = 0; i < tasks.Count; i++) + { + if (errors[i] is { } error) + { + ExceptionDispatchInfo.Throw(error); + } + + ctx.Out.WriteLine($"generated {tasks[i].Options.Name}.toml"); + } + } + + string summaryPath = parse.GetValue(summary) is { } path ? AmprCommand.Resolve(ctx, path) : Path.Combine(output, "summary.json"); + WriteText(summaryPath, PythonSortedJson.Indented(rows) + "\n", replace); + } + finally + { + Directory.Delete(temp, recursive: true); + } + })); + return command; + } + + // Python _build_batch_profile: TOML (+ sidecars), report, metrics JSON and runtime header per title. + private static PythonJsonObject BuildBatchProfile(AMPRTraceSpec pair, AMPRProfileOptions options, string output, bool fullMetrics, bool overwrite) + { + AMPRProfileResult result = AMPRProfiler.Build([pair], options); + string name = options.Name; + string tomlPath = Path.Combine(output, $"{name}.toml"); + Dictionary sidecars = []; + string toml = AMPRProfiler.RenderToml(result, options, sidecars, Path.GetFileNameWithoutExtension(tomlPath)); + WriteBundle(tomlPath, toml, sidecars, overwrite); + WriteText(Path.Combine(output, $"{name}.md"), AMPRProfiler.RenderReport(result), overwrite); + WriteText(Path.Combine(output, $"{name}.json"), AMPRProfiler.RenderMetricsJson(result, fullMetrics), overwrite); + WriteText(Path.Combine(output, $"{name}.runtime.h"), AMPRProfiler.RenderRuntimeHeader(result), overwrite); + return AMPRProfiler.BatchSummaryRow(result); + } + + // Python _parse_cache_candidates: sizes, sorted, without duplicates. + private static List CacheCandidates(string value) + { + SortedSet result = []; + foreach (string item in value.Split(',')) + { + string text = item.Trim(); + if (text.Length == 0) + { + continue; + } + + long parsed = AMPRSize.Parse(text); + if (parsed <= 0) + { + throw new AMPRPackException("cache capacities must be positive"); + } + + result.Add(parsed); + } + + return result.Count > 0 ? [.. result] : throw new AMPRPackException("at least one cache capacity is required"); + } + + // Python _write_profile_bundle: refuse to replace anything unless --overwrite, sidecars first, the TOML last. + private static void WriteBundle(string tomlPath, string toml, Dictionary sidecars, bool overwrite) + { + string directory = Path.GetDirectoryName(tomlPath)!; + List outputs = [tomlPath, .. sidecars.Keys.Select(name => Path.Combine(directory, name))]; + if (!overwrite && outputs.FirstOrDefault(Path.Exists) is { } existing) + { + throw new AMPRPackException($"refusing to overwrite existing file: {existing}"); + } + + foreach ((string name, string text) in sidecars) + { + WriteText(Path.Combine(directory, name), text, overwrite: true); + } + + WriteText(tomlPath, toml, overwrite: true); + } + + private static void WriteText(string path, string text, bool overwrite) + { + if (Path.Exists(path) && !overwrite) + { + throw new AMPRPackException($"refusing to overwrite existing file: {path}"); + } + + string directory = Path.GetDirectoryName(path)!; + Directory.CreateDirectory(directory); + string temp = Path.Combine(directory, $".{Path.GetFileName(path)}.tmp-{Environment.ProcessId.ToString(CultureInfo.InvariantCulture)}"); + File.WriteAllText(temp, text, new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + File.Move(temp, path, overwrite: true); + } +} diff --git a/src/MkPFS.Cli/Commands/PackCommands.cs b/src/MkPFS.Cli/Commands/PackCommands.cs index f01efdc..8655a92 100644 --- a/src/MkPFS.Cli/Commands/PackCommands.cs +++ b/src/MkPFS.Cli/Commands/PackCommands.cs @@ -494,12 +494,18 @@ private static Command Exfat(CliContext ctx) Description = "exFAT cluster size in bytes or 'auto' (default: 65536 - SMP/LVD-optimal 64 KiB)", DefaultValueFactory = _ => "auto", }; + Option freeSpace = new("--free-space") + { + Description = "free space to leave inside the image, e.g. 2GiB (default: 0, a tight image); needed by anything " + + "that writes to the mounted image, like AMPR Emu's debug log and traces on a read-write mount", + DefaultValueFactory = _ => "0", + }; Option overwrite = new("--overwrite") { Description = "Overwrite an existing output file" }; Option verbose = new("--verbose") { Description = "Verbose output" }; Option noProgress = new("--no-progress") { Description = "Disable the exFAT packing progress bar on stderr" }; Command command = new("exfat", "Build a raw exFAT image from a source directory") { - sourceDir, output, clusterSize, overwrite, verbose, noProgress, + sourceDir, output, clusterSize, freeSpace, overwrite, verbose, noProgress, }; AmprCliOptions ampr = new(command); command.SetAction(parse => @@ -517,6 +523,17 @@ private static Command Exfat(CliContext ctx) return 1; } + long freeBytes; + try + { + freeBytes = Core.AMPR.AMPRSize.Parse(parse.GetValue(freeSpace)!); + } + catch (ArgumentException ex) + { + ctx.Log.Error($"--free-space: {ex.Message}"); + return 1; + } + // Resolve the final output path first so overwrite can be checked and the path reported. string basename = GameParams.DefaultImageBasename(source) + ".exfat"; string target; @@ -553,7 +570,7 @@ private static Command Exfat(CliContext ctx) } IProgressSink? progress = ctx.CreateProgress(!parse.GetValue(noProgress)); - string written = ExfatImageWriter.Write(source, target, cluster, progress); + string written = ExfatImageWriter.Write(source, target, cluster, progress, freeBytes); ctx.Info($"Successfully wrote {Sizes.HumanReadable(new FileInfo(written).Length)} exFAT image: {written}"); return 0; }); diff --git a/src/MkPFS.Cli/MkPFSCli.cs b/src/MkPFS.Cli/MkPFSCli.cs index dab5fe3..035b24b 100644 --- a/src/MkPFS.Cli/MkPFSCli.cs +++ b/src/MkPFS.Cli/MkPFSCli.cs @@ -52,6 +52,7 @@ public static RootCommand BuildRootCommand(CliContext ctx) root.Subcommands.Add(Commands.ReadCommands.Unpack(ctx)); root.Subcommands.Add(Commands.BatchCommand.Create(ctx)); root.Subcommands.Add(Commands.RepairCommand.Create(ctx)); + root.Subcommands.Add(Commands.AmprCommand.Create(ctx)); root.Subcommands.Add(BuildSelfTestCommand()); root.Subcommands.Add(BuildPFSCBenchCommand()); return root; diff --git a/src/MkPFS.Cli/Output/CliContext.cs b/src/MkPFS.Cli/Output/CliContext.cs index b108e20..4949eb4 100644 --- a/src/MkPFS.Cli/Output/CliContext.cs +++ b/src/MkPFS.Cli/Output/CliContext.cs @@ -41,6 +41,15 @@ public CliContext(TextWriter stdout, TextWriter stderr, bool useColor, bool utf8 /// Progress bars enabled. public bool ProgressEnabled { get; } + /// Progress sink supplied by the host (the GUI), or on a terminal. + public IProgressSink? ExternalProgressSink => _progressSink; + + /// + /// Directory that relative path arguments resolve against (the process directory by default). Commands that + /// echo a path as typed, like Python does, resolve it here for file access. + /// + public string WorkingDirectory { get; init; } = Directory.GetCurrentDirectory(); + /// Context over the process console. /// Default context. public static CliContext CreateDefault() diff --git a/src/MkPFS.Core/AMPR/AMPRAssetPath.cs b/src/MkPFS.Core/AMPR/AMPRAssetPath.cs new file mode 100644 index 0000000..a42010f --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRAssetPath.cs @@ -0,0 +1,139 @@ +using System.Text; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// +/// /app0 path rules shared by the pack manifest and AMPRIDX3 (ampr_pack_format.py:170-240). +/// Like MkPFS.Build.AmprIndex.PathHash and the ampr_emu runtime, the pack manifest hashes the +/// UTF-8 bytes and folds only ASCII A..Z. +/// +public static class AMPRAssetPath +{ + /// FNV-1a 64 over bytes; 0 becomes 1 (Python fnv1a64). + /// Bytes. + /// Hash. + public static ulong Fnv1a64(ReadOnlySpan data) + { + ulong value = 0xCBF29CE484222325UL; + foreach (byte b in data) + { + value ^= b; + value = unchecked(value * 0x100000001B3UL); + } + + return value == 0 ? 1 : value; + } + + /// UTF-8 bytes with \ -> / and ASCII A..Z folded (Python ascii_fold_path_bytes). + /// Path. + /// Folded bytes. + public static byte[] AsciiFoldPathBytes(string path) + { + byte[] raw = Encoding.UTF8.GetBytes(path.Replace('\\', '/')); + for (int i = 0; i < raw.Length; i++) + { + if (raw[i] is >= 0x41 and <= 0x5A) + { + raw[i] += 0x20; + } + } + + return raw; + } + + /// Manifest path hash of the canonical form of (Python asset_path_hash). + /// Path. + /// Hash. + public static ulong Hash(string path) => Fnv1a64(AsciiFoldPathBytes(Canonical(path))); + + /// + /// Canonical absolute /app0 path: / separators, no empty or . components, .. + /// resolved. The spelling is kept; only the /app0 prefix check ignores case (Python canonical_asset_path). + /// + /// Path, absolute or relative to /. + /// Canonical path. + /// The path escapes the root, contains NUL, or is outside /app0. + public static string Canonical(string path) + { + path = path.Replace('\\', '/'); + if (!path.StartsWith('/')) + { + path = "/" + path; + } + + List parts = []; + foreach (string component in path.Split('/')) + { + if (component.Length == 0 || component == ".") + { + continue; + } + + if (component == "..") + { + if (parts.Count == 0) + { + throw new ArgumentException($"path escapes root: {path}"); + } + + parts.RemoveAt(parts.Count - 1); + continue; + } + + if (component.Contains('\0', StringComparison.Ordinal)) + { + throw new ArgumentException("NUL in path"); + } + + parts.Add(component); + } + + string normalized = "/" + string.Join('/', parts); + if (normalized.Equals("/app0", StringComparison.OrdinalIgnoreCase)) + { + return "/app0"; + } + + return normalized.StartsWith("/app0/", StringComparison.OrdinalIgnoreCase) + ? normalized + : throw new ArgumentException($"asset is outside /app0: {path}"); + } + + /// Path relative to /app0, empty for /app0 itself (Python asset_relative_path). + /// Path. + /// Relative path with / separators. + public static string Relative(string path) + { + string canonical = Canonical(path); + return canonical == "/app0" ? string.Empty : canonical[6..]; + } + + /// + /// Join a relative / path to after rejecting empty, ., .., + /// absolute and backslash paths, and paths whose parent leaves the root (Python safe_output_path). + /// + /// Output root. + /// Relative path. + /// The joined path. + /// The path is unsafe. + public static string SafeOutputPath(string root, string relative) + { + if (relative.Length == 0 + || relative.Contains('\\', StringComparison.Ordinal) + || relative.StartsWith('/') + || relative.EndsWith('/') + || relative.Split('/').Any(component => component is "" or "." or "..")) + { + throw new ArgumentException($"unsafe output path: {PythonText.Repr(relative)}"); + } + + string candidate = Path.Combine([root, .. relative.Split('/')]); + string rootFull = Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)); + string parentFull = Path.TrimEndingDirectorySeparator(Path.GetFullPath(Path.GetDirectoryName(Path.GetFullPath(candidate))!)); + StringComparison comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + string rootPrefix = Path.EndsInDirectorySeparator(rootFull) ? rootFull : rootFull + Path.DirectorySeparatorChar; + bool inside = parentFull.Equals(rootFull, comparison) || parentFull.StartsWith(rootPrefix, comparison); + return inside ? candidate : throw new ArgumentException($"output path escapes root: {PythonText.Repr(relative)}"); + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRBuildId.cs b/src/MkPFS.Core/AMPR/AMPRBuildId.cs new file mode 100644 index 0000000..9d78d08 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRBuildId.cs @@ -0,0 +1,26 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; + +namespace MkPFS.Core.AMPR; + +/// Deterministic 16-byte build id (Python deterministic_build_id). +public static class AMPRBuildId +{ + /// SHA-256 over "AMPRPACK4\0" and each part prefixed by its u64 length; first 16 bytes. + /// Parts in order. + /// 16 bytes. + public static byte[] Compute(IEnumerable parts) + { + using IncrementalHash digest = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + digest.AppendData("AMPRPACK4\0"u8); + Span length = stackalloc byte[8]; + foreach (byte[] part in parts) + { + BinaryPrimitives.WriteUInt64LittleEndian(length, (ulong)part.LongLength); + digest.AppendData(length); + digest.AppendData(part); + } + + return digest.GetHashAndReset()[..AMPRPackFormat.BuildIdSize]; + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRChunkCrcs.cs b/src/MkPFS.Core/AMPR/AMPRChunkCrcs.cs new file mode 100644 index 0000000..4f2b310 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRChunkCrcs.cs @@ -0,0 +1,109 @@ +using System.Buffers.Binary; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// +/// AMPRCRC1 sidecar (<manifest>.crc): a 48-byte header (<8sII16sQII) and one CRC-32 of the +/// decoded bytes per chunk, in chunk order. Offline only; the PRX never loads it +/// (Python build_chunk_crc_bytes, load_chunk_crcs). +/// +public static class AMPRChunkCrcs +{ + /// The sidecar payload: one little-endian CRC per chunk (Python encode_chunk_crcs; a build-id input). + /// Decoded CRC-32 per chunk. + /// Payload bytes. + public static byte[] EncodePayload(IReadOnlyList checksums) + { + byte[] payload = new byte[checked(checksums.Count * 4)]; + for (int i = 0; i < checksums.Count; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(i * 4), checksums[i]); + } + + return payload; + } + + /// Serialize a sidecar. + /// Manifest build id. + /// Decoded CRC-32 per chunk. + /// Sidecar bytes. + /// The build id is not 16 bytes. + public static byte[] Build(ReadOnlySpan buildId, IReadOnlyList checksums) + { + if (buildId.Length != AMPRPackFormat.BuildIdSize) + { + throw new ArgumentException("CRC sidecar build ID must contain 16 bytes"); + } + + byte[] data = new byte[checked(AMPRPackFormat.CrcHeaderSize + (checksums.Count * 4))]; + Span span = data; + EncodePayload(checksums).CopyTo(span[AMPRPackFormat.CrcHeaderSize..]); + + AMPRPackFormat.CrcMagic.CopyTo(span); + BinaryPrimitives.WriteUInt32LittleEndian(span[8..], AMPRPackFormat.CrcVersion); + BinaryPrimitives.WriteUInt32LittleEndian(span[12..], AMPRPackFormat.CrcHeaderSize); + buildId.CopyTo(span[16..]); + BinaryPrimitives.WriteUInt64LittleEndian(span[32..], (ulong)checksums.Count); + BinaryPrimitives.WriteUInt32LittleEndian(span[40..], Crc32.Update(0, span[AMPRPackFormat.CrcHeaderSize..])); + BinaryPrimitives.WriteUInt32LittleEndian(span[44..], Crc32.Update(0, span[..AMPRPackFormat.CrcHeaderSize])); + return data; + } + + /// Load and check a sidecar file. + /// Sidecar path. + /// Manifest build id. + /// Manifest chunk count. + /// One CRC per chunk. + /// The sidecar is malformed or belongs to another build. + public static uint[] Load(string path, ReadOnlySpan expectedBuildId, long expectedChunkCount) => + Parse(File.ReadAllBytes(path), expectedBuildId, expectedChunkCount); + + /// Parse and check sidecar bytes. + /// Sidecar bytes. + /// Manifest build id. + /// Manifest chunk count. + /// One CRC per chunk. + /// The sidecar is malformed or belongs to another build. + public static uint[] Parse(ReadOnlySpan data, ReadOnlySpan expectedBuildId, long expectedChunkCount) + { + if (data.Length < AMPRPackFormat.CrcHeaderSize) + { + throw new InvalidDataException("chunk CRC sidecar is truncated"); + } + + if (!data[..8].SequenceEqual(AMPRPackFormat.CrcMagic) + || BinaryPrimitives.ReadUInt32LittleEndian(data[8..]) != AMPRPackFormat.CrcVersion + || BinaryPrimitives.ReadUInt32LittleEndian(data[12..]) != AMPRPackFormat.CrcHeaderSize) + { + throw new InvalidDataException("unsupported chunk CRC sidecar"); + } + + ulong chunkCount = BinaryPrimitives.ReadUInt64LittleEndian(data[32..]); + if (!data.Slice(16, AMPRPackFormat.BuildIdSize).SequenceEqual(expectedBuildId) || chunkCount != (ulong)expectedChunkCount) + { + throw new InvalidDataException("chunk CRC sidecar build ID or chunk count mismatch"); + } + + byte[] header = data[..AMPRPackFormat.CrcHeaderSize].ToArray(); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(44), 0); + if (Crc32.Update(0, header) != BinaryPrimitives.ReadUInt32LittleEndian(data[44..])) + { + throw new InvalidDataException("chunk CRC sidecar header CRC mismatch"); + } + + ReadOnlySpan payload = data[AMPRPackFormat.CrcHeaderSize..]; + if ((ulong)payload.Length != chunkCount * 4 || Crc32.Update(0, payload) != BinaryPrimitives.ReadUInt32LittleEndian(data[40..])) + { + throw new InvalidDataException("chunk CRC sidecar payload CRC mismatch"); + } + + uint[] values = new uint[chunkCount]; + for (int i = 0; i < values.Length; i++) + { + values[i] = BinaryPrimitives.ReadUInt32LittleEndian(payload[(i * 4)..]); + } + + return values; + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRDataHeader.cs b/src/MkPFS.Core/AMPR/AMPRDataHeader.cs new file mode 100644 index 0000000..8bb9d79 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRDataHeader.cs @@ -0,0 +1,104 @@ +using System.Buffers.Binary; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// +/// AMPRDAT3 data volume header (64 bytes, <8sIIII16sQQII) at offset 0 of every .pak volume +/// (Python build_data_header, validate_data_header). +/// +/// Pack id. +/// 16-byte build id. +/// Payload start in the volume. +/// Payload length. +/// PackFlag* bits. +public readonly record struct AMPRDataHeader(uint PackId, byte[] BuildId, ulong PayloadOffset, ulong PayloadBytes, uint Flags = 0) +{ + /// Serialize the header with its CRC. + /// 64 bytes. + /// The build id is not 16 bytes. + public byte[] ToBytes() + { + if (BuildId.Length != AMPRPackFormat.BuildIdSize) + { + throw new ArgumentException("build id must contain 16 bytes"); + } + + byte[] data = new byte[AMPRPackFormat.DataHeaderSize]; + Span span = data; + AMPRPackFormat.DataMagic.CopyTo(span); + BinaryPrimitives.WriteUInt32LittleEndian(span[8..], AMPRPackFormat.DataVersion); + BinaryPrimitives.WriteUInt32LittleEndian(span[12..], AMPRPackFormat.DataHeaderSize); + BinaryPrimitives.WriteUInt32LittleEndian(span[16..], PackId); + BinaryPrimitives.WriteUInt32LittleEndian(span[20..], Flags); + BuildId.CopyTo(span[24..]); + BinaryPrimitives.WriteUInt64LittleEndian(span[40..], PayloadOffset); + BinaryPrimitives.WriteUInt64LittleEndian(span[48..], PayloadBytes); + BinaryPrimitives.WriteUInt32LittleEndian(span[56..], Crc32.Update(0, span)); + return data; + } + + /// Parse and check a header; optional expectations are compared after the CRC. + /// Exactly 64 bytes. + /// Pack id the manifest assigns, or . + /// Manifest build id, or . + /// Manifest pack flags, or . + /// The header. + /// The header is malformed or does not match. + public static AMPRDataHeader Parse( + ReadOnlySpan header, + uint? expectedPackId = null, + ReadOnlySpan expectedBuildId = default, + uint? expectedFlags = null) + { + if (header.Length != AMPRPackFormat.DataHeaderSize) + { + throw new InvalidDataException("pack data header is truncated"); + } + + uint version = BinaryPrimitives.ReadUInt32LittleEndian(header[8..]); + uint headerSize = BinaryPrimitives.ReadUInt32LittleEndian(header[12..]); + if (!header[..8].SequenceEqual(AMPRPackFormat.DataMagic) || version != AMPRPackFormat.DataVersion || headerSize != AMPRPackFormat.DataHeaderSize) + { + throw new InvalidDataException("unsupported pack data header"); + } + + if (BinaryPrimitives.ReadUInt32LittleEndian(header[60..]) != 0) + { + throw new InvalidDataException("invalid pack data header"); + } + + uint flags = BinaryPrimitives.ReadUInt32LittleEndian(header[20..]); + if ((flags & ~AMPRPackFormat.PackKnownFlags) != 0) + { + throw new InvalidDataException("unknown pack data flags"); + } + + byte[] copy = header.ToArray(); + BinaryPrimitives.WriteUInt32LittleEndian(copy.AsSpan(56), 0); + if (Crc32.Update(0, copy) != BinaryPrimitives.ReadUInt32LittleEndian(header[56..])) + { + throw new InvalidDataException("pack data header CRC mismatch"); + } + + AMPRDataHeader result = new( + BinaryPrimitives.ReadUInt32LittleEndian(header[16..]), + header.Slice(24, AMPRPackFormat.BuildIdSize).ToArray(), + BinaryPrimitives.ReadUInt64LittleEndian(header[40..]), + BinaryPrimitives.ReadUInt64LittleEndian(header[48..]), + flags); + if (expectedPackId is { } packId && result.PackId != packId) + { + throw new InvalidDataException("pack id mismatch"); + } + + if (!expectedBuildId.IsEmpty && !expectedBuildId.SequenceEqual(result.BuildId)) + { + throw new InvalidDataException("pack build id mismatch"); + } + + return expectedFlags is { } expected && flags != expected + ? throw new InvalidDataException("pack flags mismatch") + : result; + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRGlob.cs b/src/MkPFS.Core/AMPR/AMPRGlob.cs new file mode 100644 index 0000000..ab70b40 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRGlob.cs @@ -0,0 +1,179 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.RegularExpressions; + +namespace MkPFS.Core.AMPR; + +/// +/// Python fnmatch.fnmatchcase as used by ampr_pack (glob_matches): case-sensitive, * and +/// ? also match /, ** is the same as *, [...]/[!...] sets, no escaping. +/// Patterns are translated like CPython 3.11 fnmatch._translate and compiled to .NET regexes. +/// +public static class AMPRGlob +{ + private static readonly ConcurrentDictionary Cache = new(StringComparer.Ordinal); + + /// Python glob_matches: normalize separators, strip leading /, match any pattern. + /// Relative path. + /// Patterns. + /// when any pattern matches. + public static bool Matches(string path, IEnumerable patterns) + { + string normalized = path.Replace('\\', '/').TrimStart('/'); + return patterns.Any(pattern => FnMatchCase(normalized, pattern.Replace('\\', '/').TrimStart('/'))); + } + + /// Python fnmatch.fnmatchcase(name, pattern). + /// Name. + /// Pattern. + /// when the whole name matches. + public static bool FnMatchCase(string name, string pattern) => + Cache.GetOrAdd(pattern, static p => new Regex(Translate(p), RegexOptions.Singleline | RegexOptions.CultureInvariant)).IsMatch(name); + + /// Translate a pattern to an anchored .NET regex (CPython 3.11 fnmatch._translate). + /// Pattern. + /// Regex source. + public static string Translate(string pattern) + { + StringBuilder res = new("^"); + bool lastWasStar = false; + int i = 0; + int n = pattern.Length; + while (i < n) + { + char c = pattern[i++]; + if (c == '*') + { + // Consecutive stars compress into one. + if (!lastWasStar) + { + res.Append(".*"); + lastWasStar = true; + } + + continue; + } + + lastWasStar = false; + if (c == '?') + { + res.Append('.'); + } + else if (c == '[') + { + i = TranslateSet(pattern, i, res); + } + else + { + res.Append(Regex.Escape(c.ToString())); + } + } + + return res.Append(@"\z").ToString(); + } + + // i is just past '['. Appends the set (or a literal '[') and returns the index to continue at. + private static int TranslateSet(string pat, int i, StringBuilder res) + { + int n = pat.Length; + int j = i; + if (j < n && pat[j] == '!') + { + j++; + } + + if (j < n && pat[j] == ']') + { + j++; + } + + while (j < n && pat[j] != ']') + { + j++; + } + + if (j >= n) + { + res.Append(@"\["); + return i; + } + + string stuff; + if (!pat.AsSpan(i, j - i).Contains('-')) + { + stuff = pat[i..j].Replace(@"\", @"\\", StringComparison.Ordinal); + } + else + { + List chunks = []; + int k = pat[i] == '!' ? i + 2 : i + 1; + while (true) + { + k = k < j ? pat.IndexOf('-', k, j - k) : -1; + if (k < 0) + { + break; + } + + chunks.Add(pat[i..k]); + i = k + 1; + k += 3; + } + + string chunk = pat[i..j]; + if (chunk.Length > 0) + { + chunks.Add(chunk); + } + else + { + chunks[^1] += "-"; + } + + // Remove empty ranges (invalid in a regex). + for (int m = chunks.Count - 1; m > 0; m--) + { + if (chunks[m - 1][^1] > chunks[m][0]) + { + chunks[m - 1] = chunks[m - 1][..^1] + chunks[m][1..]; + chunks.RemoveAt(m); + } + } + + // Escape backslashes and hyphens that do not form ranges. + stuff = string.Join('-', chunks.Select(s => s.Replace(@"\", @"\\", StringComparison.Ordinal).Replace("-", @"\-", StringComparison.Ordinal))); + } + + // Python escapes set operations (&&, ~~, ||); .NET treats "-[" as class subtraction, so '[' is escaped too. + stuff = Regex.Replace(stuff, @"([&~|\[])", @"\$1"); + if (stuff.Length == 0) + { + res.Append("(?!)"); + } + else if (stuff == "!") + { + res.Append('.'); + } + else + { + if (stuff[0] == '!') + { + stuff = "^" + stuff[1..]; + } + else if (stuff[0] is '^') + { + stuff = @"\" + stuff; + } + + // A leading ']' is literal in Python; escape it for .NET. + if (stuff[0] == ']' || stuff.StartsWith("^]", StringComparison.Ordinal)) + { + stuff = stuff[0] == ']' ? @"\" + stuff : @"^\" + stuff[1..]; + } + + res.Append('[').Append(stuff).Append(']'); + } + + return j + 1; + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRIndexReader.cs b/src/MkPFS.Core/AMPR/AMPRIndexReader.cs new file mode 100644 index 0000000..5c739f2 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRIndexReader.cs @@ -0,0 +1,114 @@ +using System.Buffers.Binary; +using System.Text; + +namespace MkPFS.Core.AMPR; + +/// One AMPRIDX3 row as the pack tools see it. +/// Canonical /app0 path. +/// File size. +/// Modification time, Unix seconds. +public readonly record struct AMPRIndexEntry(string Path, ulong Size, long MTime); + +/// +/// Strict AMPRIDX3 (ampr_emu.index) reader used by the pack tools (Python read_ampridx3 in +/// ampr_pack_format.py). Row n is file id n + 1. The index writer lives in +/// MkPFS.Build.AmprIndex. +/// +public static class AMPRIndexReader +{ + private const int HeaderSize = 48; + private const int EntrySize = 24; + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// Read an index file. + /// Index path. + /// Rows in record order. + /// The index is malformed. + public static List Read(string path) => Parse(File.ReadAllBytes(path)); + + /// Parse index bytes. + /// Index bytes. + /// Rows in record order. + /// The index is malformed. + public static List Parse(ReadOnlySpan data) + { + if (data.Length < HeaderSize) + { + throw new InvalidDataException("AMPRIDX3 is truncated"); + } + + if (!data[..8].SequenceEqual("AMPRIDX3"u8) || BinaryPrimitives.ReadUInt32LittleEndian(data[8..]) != 3) + { + throw new InvalidDataException("unsupported AMPR index"); + } + + uint entrySize = BinaryPrimitives.ReadUInt32LittleEndian(data[12..]); + ulong entryCount = BinaryPrimitives.ReadUInt64LittleEndian(data[16..]); + ulong pathBytes = BinaryPrimitives.ReadUInt64LittleEndian(data[24..]); + ulong hashOffset = BinaryPrimitives.ReadUInt64LittleEndian(data[32..]); + uint hashSlotSize = BinaryPrimitives.ReadUInt32LittleEndian(data[40..]); + uint hashSlotCount = BinaryPrimitives.ReadUInt32LittleEndian(data[44..]); + if (entrySize != EntrySize) + { + throw new InvalidDataException("unexpected AMPRIDX3 entry size"); + } + + UInt128 pathOffset = HeaderSize + ((UInt128)entryCount * entrySize); + if (pathOffset + pathBytes > (UInt128)data.Length) + { + throw new InvalidDataException("AMPRIDX3 records/path blob are truncated"); + } + + if (hashOffset < pathOffset + pathBytes || hashOffset > (ulong)data.Length) + { + throw new InvalidDataException("invalid AMPRIDX3 hash offset"); + } + + if (hashSlotCount != 0 && hashSlotSize == 0) + { + throw new InvalidDataException("invalid AMPRIDX3 hash table"); + } + + List result = new((int)entryCount); + for (int index = 0; index < (int)entryCount; index++) + { + ReadOnlySpan record = data.Slice(HeaderSize + (index * EntrySize), EntrySize); + ulong pathAt = BinaryPrimitives.ReadUInt32LittleEndian(record); + ulong pathLength = BinaryPrimitives.ReadUInt32LittleEndian(record[4..]); + ulong size = BinaryPrimitives.ReadUInt64LittleEndian(record[8..]); + long mtime = BinaryPrimitives.ReadInt64LittleEndian(record[16..]); + if (pathAt + pathLength >= pathBytes) + { + throw new InvalidDataException($"AMPRIDX3 path {index} is out of range"); + } + + int start = (int)pathOffset + (int)pathAt; + int end = start + (int)pathLength; + if (data[end] != 0) + { + throw new InvalidDataException($"AMPRIDX3 path {index} is not NUL terminated"); + } + + string decoded; + try + { + decoded = StrictUtf8.GetString(data[start..end]); + } + catch (DecoderFallbackException exc) + { + throw new InvalidDataException($"AMPRIDX3 path {index} is not UTF-8", exc); + } + + try + { + result.Add(new AMPRIndexEntry(AMPRAssetPath.Canonical(decoded), size, mtime)); + } + catch (ArgumentException exc) + { + throw new InvalidDataException(exc.Message, exc); + } + } + + return result; + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRPackException.cs b/src/MkPFS.Core/AMPR/AMPRPackException.cs new file mode 100644 index 0000000..9b5cd45 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRPackException.cs @@ -0,0 +1,23 @@ +namespace MkPFS.Core.AMPR; + +/// +/// Pack build or configuration error (Python PackError). The CLI prints error: <message> and exits 2, +/// as it does for and (Python ValueError). +/// +public sealed class AMPRPackException : Exception +{ + /// Create an error. + /// Message. + public AMPRPackException(string message) + : base(message) + { + } + + /// Create an error with a cause. + /// Message. + /// Cause. + public AMPRPackException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRPackFormat.cs b/src/MkPFS.Core/AMPR/AMPRPackFormat.cs new file mode 100644 index 0000000..af29771 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRPackFormat.cs @@ -0,0 +1,191 @@ +namespace MkPFS.Core.AMPR; + +/// +/// Constants of the AMPR seekable asset-pack format (ampr_emu tools/ampr_pack_format.py): +/// the AMPRPAK4 manifest, AMPRDAT3 data volumes, AMPRCRC1 chunk CRC sidecar and +/// AMPRCFG1 runtime settings. All integers are little endian. +/// +public static class AMPRPackFormat +{ + /// Manifest magic. + public static ReadOnlySpan IndexMagic => "AMPRPAK4"u8; + + /// Data volume magic. + public static ReadOnlySpan DataMagic => "AMPRDAT3"u8; + + /// Chunk CRC sidecar magic. + public static ReadOnlySpan CrcMagic => "AMPRCRC1"u8; + + /// Runtime settings magic. + public static ReadOnlySpan RuntimeMagic => "AMPRCFG1"u8; + + /// Manifest version. + public const uint IndexVersion = 4; + + /// Data volume version. + public const uint DataVersion = 3; + + /// Chunk CRC sidecar version. + public const uint CrcVersion = 1; + + /// Runtime settings version. + public const uint RuntimeVersion = 1; + + /// Manifest byte-order marker. + public const uint EndianMarker = 0x01020304; + + /// Manifest header size (<8sIIII16sQQIIIIQQQQQIIQ). + public const int IndexHeaderSize = 128; + + /// Data volume header size (<8sIIII16sQQII). + public const int DataHeaderSize = 64; + + /// Chunk CRC sidecar header size (<8sII16sQII). + public const int CrcHeaderSize = 48; + + /// Runtime settings size (<8sII16sQQIIII). + public const int RuntimeSize = 64; + + /// File record size (<QQqIIIIIBBH). + public const int FileRecordSize = 48; + + /// Chunk record size (<QI). + public const int ChunkRecordSize = 12; + + /// Pack record size (<QQIIII). + public const int PackRecordSize = 32; + + /// Build id length. + public const int BuildIdSize = 16; + + /// File is stored in pack volumes. + public const uint FileFlagPacked = 1 << 0; + + /// File chunks are never compressed. + public const uint FileFlagStoreOnly = 1 << 1; + + /// File uses the dense streaming layout. + public const uint FileFlagStreaming = 1 << 2; + + /// File is admitted to the decoded cache immediately. + public const uint FileFlagHot = 1 << 3; + + /// File uses the random-access layout. + public const uint FileFlagRandomAccess = 1 << 4; + + /// Every defined file flag. + public const uint FileKnownFlags = + FileFlagPacked | FileFlagStoreOnly | FileFlagStreaming | FileFlagHot | FileFlagRandomAccess; + + /// Chunk stored uncompressed. + public const int ChunkCodecRaw = 0; + + /// Chunk stored as a raw LZ4 block. + public const int ChunkCodecLZ4 = 1; + + /// Chunk bytes are shared with an earlier chunk (deduplication). + public const int ChunkFlagShared = 1 << 0; + + /// Chunk belongs to a streaming extent. + public const int ChunkFlagStreaming = 1 << 1; + + /// Chunk lies inside one I/O page. + public const int ChunkFlagPageContained = 1 << 2; + + /// Chunk starts on an I/O page boundary. + public const int ChunkFlagPageAligned = 1 << 3; + + /// Every defined chunk flag. + public const int ChunkKnownFlags = + ChunkFlagShared | ChunkFlagStreaming | ChunkFlagPageContained | ChunkFlagPageAligned; + + /// Pack holds stripes of large files. + public const uint PackFlagStriped = 1 << 0; + + /// Pack uses the page-aware layout (required). + public const uint PackFlagIOPageLayout = 1 << 1; + + /// Every defined pack flag. + public const uint PackKnownFlags = PackFlagStriped | PackFlagIOPageLayout; + + /// Every defined manifest flag (none). + public const uint IndexKnownFlags = 0; + + /// Smallest block shift (16 KiB). + public const int MinBlockShift = 14; + + /// Largest block shift (1 MiB). + public const int MaxBlockShift = 20; + + /// Smallest I/O page shift (4 KiB). + public const int MinIOPageShift = 12; + + /// Largest I/O page shift (1 MiB). + public const int MaxIOPageShift = 20; + + /// Alignment of every chunk offset. + public const int PhysicalChunkAlignment = 64; + + /// Chunk offsets use 48 bits. + public const ulong ChunkOffsetMask = (1UL << 48) - 1; + + /// Bits of stored_size - 1 in a chunk descriptor. + public const int ChunkStoredBits = 20; + + /// Mask of stored_size - 1. + public const uint ChunkStoredMask = (1U << ChunkStoredBits) - 1; + + /// Codec position in a chunk descriptor. + public const int ChunkCodecShift = 20; + + /// Codec mask (after shifting). + public const uint ChunkCodecMask = 0x3; + + /// Flags position in a chunk descriptor. + public const int ChunkFlagsShift = 22; + + /// Flags mask (after shifting). + public const uint ChunkFlagsMask = 0xFF; + + /// Every defined descriptor bit. + public const uint ChunkDescriptorKnownMask = + ChunkStoredMask | (ChunkCodecMask << ChunkCodecShift) | (ChunkFlagsMask << ChunkFlagsShift); + + /// Return path + ".crc" (Python chunk_crc_path). + /// Manifest path. + /// Sidecar path. + public static string ChunkCrcPath(string indexPath) => indexPath + ".crc"; + + /// Return path + ".runtime". + /// Manifest path. + /// Runtime settings path. + public static string RuntimePath(string indexPath) => indexPath + ".runtime"; + + /// Round up to a power-of-two . + /// Value. + /// Power of two. + /// Aligned value. + public static ulong AlignUp(ulong value, ulong alignment) + { + CheckAlignment(alignment); + return checked(value + alignment - 1) & ~(alignment - 1); + } + + /// Round down to a power-of-two . + /// Value. + /// Power of two. + /// Aligned value. + public static ulong AlignDown(ulong value, ulong alignment) + { + CheckAlignment(alignment); + return value & ~(alignment - 1); + } + + private static void CheckAlignment(ulong alignment) + { + if (alignment == 0 || (alignment & (alignment - 1)) != 0) + { + throw new ArgumentException($"alignment must be a power of two, got {alignment}"); + } + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRPackManifest.cs b/src/MkPFS.Core/AMPR/AMPRPackManifest.cs new file mode 100644 index 0000000..940f932 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRPackManifest.cs @@ -0,0 +1,542 @@ +using System.Buffers.Binary; +using System.Text; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// +/// AMPRPAK4 runtime storage manifest (ampr_assets.index): a 128-byte header, then file, chunk and pack +/// records and a NUL-terminated UTF-8 string table. Port of ampr_pack_format.py PackManifest, +/// build_manifest_bytes, load_manifest and validate_manifest; error messages match. +/// +public sealed class AMPRPackManifest +{ + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// Create a manifest from records. + /// 16-byte build id. + /// Header flags (must be 0). + /// File records in file-id order. + /// Chunk records. + /// Pack records. + /// String table. + /// Manifest path, when loaded from disk. + public AMPRPackManifest( + byte[] buildId, + uint flags, + IReadOnlyList files, + IReadOnlyList chunks, + IReadOnlyList packs, + byte[] strings, + string? path = null) + { + BuildId = buildId; + Flags = flags; + Files = files; + Chunks = chunks; + Packs = packs; + Strings = strings; + Path = path; + } + + /// Manifest path, when loaded from disk. + public string? Path { get; } + + /// 16-byte build id shared by the manifest, volumes, CRC sidecar and runtime settings. + public byte[] BuildId { get; } + + /// Header flags. + public uint Flags { get; } + + /// File records; file id n is Files[n - 1]. + public IReadOnlyList Files { get; } + + /// Chunk records; loaded manifests carry the derived . + public IReadOnlyList Chunks { get; } + + /// Pack volume records; pack id n is Packs[n]. + public IReadOnlyList Packs { get; } + + /// String table (paths and volume names, each followed by NUL). + public byte[] Strings { get; } + + /// Decode a string-table entry. + /// Offset. + /// Length without NUL. + /// The string. + /// The entry is out of bounds, not NUL-terminated, or not UTF-8. + public string StringAt(uint offset, uint length) + { + if ((ulong)offset + length >= (ulong)Strings.Length) + { + throw new InvalidDataException("string is out of bounds"); + } + + if (Strings[offset + length] != 0) + { + throw new InvalidDataException("string is not NUL terminated"); + } + + try + { + return StrictUtf8.GetString(Strings, (int)offset, (int)length); + } + catch (DecoderFallbackException exc) + { + throw new InvalidDataException("string is not valid UTF-8", exc); + } + } + + /// Path of a file id (1-based). + /// File id. + /// Canonical /app0 path. + public string FilePath(int fileId) + { + if (fileId <= 0 || fileId > Files.Count) + { + throw new ArgumentOutOfRangeException(nameof(fileId), "invalid file id"); + } + + AMPRFileRecord record = Files[fileId - 1]; + return StringAt(record.PathOffset, record.PathLength); + } + + /// Volume file name of a pack id. + /// Pack id. + /// Name relative to the manifest directory. + public string PackName(int packId) + { + if (packId < 0 || packId >= Packs.Count) + { + throw new ArgumentOutOfRangeException(nameof(packId), "invalid pack id"); + } + + AMPRPackRecord record = Packs[packId]; + return StringAt(record.NameOffset, record.NameLength); + } + + /// Serialize the manifest (Python build_manifest_bytes). + /// Manifest bytes. + /// The build id or a count is outside the format limits. + public byte[] Serialize() + { + if (BuildId.Length != AMPRPackFormat.BuildIdSize) + { + throw new ArgumentException("build id must contain 16 bytes"); + } + + if (Packs.Count > 0xFFFF) + { + throw new ArgumentException("pack count exceeds uint16 chunk-record limits"); + } + + long filesOffset = AMPRPackFormat.IndexHeaderSize; + long chunksOffset = checked(filesOffset + ((long)Files.Count * AMPRPackFormat.FileRecordSize)); + long packsOffset = checked(chunksOffset + ((long)Chunks.Count * AMPRPackFormat.ChunkRecordSize)); + long stringsOffset = checked(packsOffset + ((long)Packs.Count * AMPRPackFormat.PackRecordSize)); + byte[] data = new byte[checked((int)(stringsOffset + Strings.Length))]; + Span span = data; + for (int i = 0; i < Files.Count; i++) + { + Files[i].Write(span[(int)(filesOffset + ((long)i * AMPRPackFormat.FileRecordSize))..]); + } + + for (int i = 0; i < Chunks.Count; i++) + { + Chunks[i].Write(span[(int)(chunksOffset + ((long)i * AMPRPackFormat.ChunkRecordSize))..]); + } + + for (int i = 0; i < Packs.Count; i++) + { + Packs[i].Write(span[(int)(packsOffset + ((long)i * AMPRPackFormat.PackRecordSize))..]); + } + + Strings.CopyTo(span[(int)stringsOffset..]); + uint payloadCrc = Crc32.Update(0, span[AMPRPackFormat.IndexHeaderSize..]); + + Span header = span[..AMPRPackFormat.IndexHeaderSize]; + AMPRPackFormat.IndexMagic.CopyTo(header); + BinaryPrimitives.WriteUInt32LittleEndian(header[8..], AMPRPackFormat.IndexVersion); + BinaryPrimitives.WriteUInt32LittleEndian(header[12..], AMPRPackFormat.IndexHeaderSize); + BinaryPrimitives.WriteUInt32LittleEndian(header[16..], Flags); + BinaryPrimitives.WriteUInt32LittleEndian(header[20..], AMPRPackFormat.EndianMarker); + BuildId.CopyTo(header[24..]); + BinaryPrimitives.WriteUInt64LittleEndian(header[40..], (ulong)Files.Count); + BinaryPrimitives.WriteUInt64LittleEndian(header[48..], (ulong)Chunks.Count); + BinaryPrimitives.WriteUInt32LittleEndian(header[56..], (uint)Packs.Count); + BinaryPrimitives.WriteUInt32LittleEndian(header[60..], AMPRPackFormat.FileRecordSize); + BinaryPrimitives.WriteUInt32LittleEndian(header[64..], AMPRPackFormat.ChunkRecordSize); + BinaryPrimitives.WriteUInt32LittleEndian(header[68..], AMPRPackFormat.PackRecordSize); + BinaryPrimitives.WriteUInt64LittleEndian(header[72..], (ulong)filesOffset); + BinaryPrimitives.WriteUInt64LittleEndian(header[80..], (ulong)chunksOffset); + BinaryPrimitives.WriteUInt64LittleEndian(header[88..], (ulong)packsOffset); + BinaryPrimitives.WriteUInt64LittleEndian(header[96..], (ulong)stringsOffset); + BinaryPrimitives.WriteUInt64LittleEndian(header[104..], (ulong)Strings.Length); + BinaryPrimitives.WriteUInt32LittleEndian(header[112..], payloadCrc); + // header_crc (116) covers the header with itself zeroed; reserved (120) stays 0. + BinaryPrimitives.WriteUInt32LittleEndian(header[116..], Crc32.Update(0, header)); + return data; + } + + /// Load and validate a manifest file (Python load_manifest). + /// Manifest path. + /// The manifest. + /// The manifest is malformed. + public static AMPRPackManifest Load(string path) => Parse(File.ReadAllBytes(path), path); + + /// Parse and validate manifest bytes (Python load_manifest). + /// Manifest bytes. + /// Manifest path to record. + /// The manifest. + /// The manifest is malformed. + public static AMPRPackManifest Parse(ReadOnlySpan data, string? path = null) + { + if (data.Length < AMPRPackFormat.IndexHeaderSize) + { + throw new InvalidDataException("pack index is truncated"); + } + + ReadOnlySpan header = data[..AMPRPackFormat.IndexHeaderSize]; + uint version = BinaryPrimitives.ReadUInt32LittleEndian(header[8..]); + if (!header[..8].SequenceEqual(AMPRPackFormat.IndexMagic) || version != AMPRPackFormat.IndexVersion) + { + throw new InvalidDataException("unsupported AMPR pack index"); + } + + uint headerSize = BinaryPrimitives.ReadUInt32LittleEndian(header[12..]); + uint flags = BinaryPrimitives.ReadUInt32LittleEndian(header[16..]); + uint endian = BinaryPrimitives.ReadUInt32LittleEndian(header[20..]); + byte[] buildId = header.Slice(24, AMPRPackFormat.BuildIdSize).ToArray(); + ulong fileCount = BinaryPrimitives.ReadUInt64LittleEndian(header[40..]); + ulong chunkCount = BinaryPrimitives.ReadUInt64LittleEndian(header[48..]); + uint packCount = BinaryPrimitives.ReadUInt32LittleEndian(header[56..]); + uint fileRecordSize = BinaryPrimitives.ReadUInt32LittleEndian(header[60..]); + uint chunkRecordSize = BinaryPrimitives.ReadUInt32LittleEndian(header[64..]); + uint packRecordSize = BinaryPrimitives.ReadUInt32LittleEndian(header[68..]); + ulong filesOffset = BinaryPrimitives.ReadUInt64LittleEndian(header[72..]); + ulong chunksOffset = BinaryPrimitives.ReadUInt64LittleEndian(header[80..]); + ulong packsOffset = BinaryPrimitives.ReadUInt64LittleEndian(header[88..]); + ulong stringsOffset = BinaryPrimitives.ReadUInt64LittleEndian(header[96..]); + ulong stringsSize = BinaryPrimitives.ReadUInt64LittleEndian(header[104..]); + uint payloadCrc = BinaryPrimitives.ReadUInt32LittleEndian(header[112..]); + uint headerCrc = BinaryPrimitives.ReadUInt32LittleEndian(header[116..]); + ulong reserved = BinaryPrimitives.ReadUInt64LittleEndian(header[120..]); + if (headerSize != AMPRPackFormat.IndexHeaderSize + || endian != AMPRPackFormat.EndianMarker + || (flags & ~AMPRPackFormat.IndexKnownFlags) != 0 + || reserved != 0) + { + throw new InvalidDataException("invalid AMPR pack header"); + } + + if (fileRecordSize != AMPRPackFormat.FileRecordSize + || chunkRecordSize != AMPRPackFormat.ChunkRecordSize + || packRecordSize != AMPRPackFormat.PackRecordSize) + { + throw new InvalidDataException("unsupported AMPR pack record size"); + } + + byte[] headerCopy = header.ToArray(); + BinaryPrimitives.WriteUInt32LittleEndian(headerCopy.AsSpan(116), 0); + if (Crc32.Update(0, headerCopy) != headerCrc) + { + throw new InvalidDataException("pack index header CRC mismatch"); + } + + if (Crc32.Update(0, data[AMPRPackFormat.IndexHeaderSize..]) != payloadCrc) + { + throw new InvalidDataException("pack index payload CRC mismatch"); + } + + // Python ints do not overflow; UInt128 keeps the layout check exact for any header values. + UInt128 expectedFiles = AMPRPackFormat.IndexHeaderSize; + UInt128 expectedChunks = expectedFiles + ((UInt128)fileCount * AMPRPackFormat.FileRecordSize); + UInt128 expectedPacks = expectedChunks + ((UInt128)chunkCount * AMPRPackFormat.ChunkRecordSize); + UInt128 expectedStrings = expectedPacks + ((UInt128)packCount * AMPRPackFormat.PackRecordSize); + if (filesOffset != expectedFiles + || chunksOffset != expectedChunks + || packsOffset != expectedPacks + || stringsOffset != expectedStrings + || (UInt128)stringsOffset + stringsSize != (UInt128)data.Length) + { + throw new InvalidDataException("invalid pack index section layout"); + } + + AMPRFileRecord[] files = new AMPRFileRecord[fileCount]; + for (int i = 0; i < files.Length; i++) + { + files[i] = AMPRFileRecord.Read(data[(int)(filesOffset + ((ulong)i * AMPRPackFormat.FileRecordSize))..]); + } + + AMPRChunkRecord[] chunks = new AMPRChunkRecord[chunkCount]; + for (int i = 0; i < chunks.Length; i++) + { + chunks[i] = AMPRChunkRecord.Read(data[(int)(chunksOffset + ((ulong)i * AMPRPackFormat.ChunkRecordSize))..]); + } + + // AMPRPAK4 derives the raw chunk size from the owning file's block geometry. + foreach (AMPRFileRecord file in files) + { + if (!file.IsPacked) + { + continue; + } + + // An invalid shift is reported by Validate; here it only has to stay larger than any file. + ulong blockSize = file.BlockShift < 63 ? 1UL << file.BlockShift : ulong.MaxValue; + for (ulong local = 0; local < file.ChunkCount; local++) + { + ulong chunkIndex = file.FirstChunk + local; + if (chunkIndex >= (ulong)chunks.Length) + { + throw new InvalidDataException("chunk range is invalid while decoding AMPRPAK4"); + } + + UInt128 blockBegin = (UInt128)local * blockSize; + if (blockBegin >= file.LogicalSize) + { + throw new InvalidDataException("AMPRPAK4 file has too many chunks"); + } + + ulong remaining = file.LogicalSize - (ulong)blockBegin; + chunks[chunkIndex] = chunks[chunkIndex] with { RawSize = (int)Math.Min(blockSize, Math.Min(remaining, int.MaxValue)) }; + } + } + + AMPRPackRecord[] packs = new AMPRPackRecord[packCount]; + for (int i = 0; i < packs.Length; i++) + { + packs[i] = AMPRPackRecord.Read(data[(int)(packsOffset + ((ulong)i * AMPRPackFormat.PackRecordSize))..]); + } + + AMPRPackManifest manifest = new(buildId, flags, files, chunks, packs, data[(int)stringsOffset..].ToArray(), path); + manifest.Validate(); + return manifest; + } + + /// Check every structural invariant the runtime relies on (Python validate_manifest). + /// An invariant is violated. + public void Validate() + { + if ((Flags & ~AMPRPackFormat.IndexKnownFlags) != 0) + { + throw new InvalidDataException("unknown pack index flags"); + } + + for (int packId = 0; packId < Packs.Count; packId++) + { + ValidatePack(packId, Packs[packId]); + } + + for (int fileId = 1; fileId <= Files.Count; fileId++) + { + ValidateFile(fileId, Files[fileId - 1]); + } + } + + private void ValidatePack(int packId, AMPRPackRecord record) + { + string name = StringAt(record.NameOffset, record.NameLength); + try + { + AMPRAssetPath.SafeOutputPath(".", name); + } + catch (ArgumentException exc) + { + throw new InvalidDataException($"unsafe pack name {packId}: {PythonText.Repr(name)}", exc); + } + + if ((record.Flags & ~AMPRPackFormat.PackKnownFlags) != 0) + { + throw new InvalidDataException($"unknown flags for pack {packId}"); + } + + uint page = record.IOPageSize; + if (page < 1U << AMPRPackFormat.MinIOPageShift + || page > 1U << AMPRPackFormat.MaxIOPageShift + || (page & (page - 1)) != 0 + || page % AMPRPackFormat.PhysicalChunkAlignment != 0) + { + throw new InvalidDataException($"invalid I/O page size for pack {packId}"); + } + + if ((record.Flags & AMPRPackFormat.PackFlagIOPageLayout) == 0) + { + throw new InvalidDataException($"pack {packId} does not declare page-aware layout"); + } + + if (record.FileSize < AMPRPackFormat.DataHeaderSize || record.PayloadBytes > record.FileSize) + { + throw new InvalidDataException($"invalid size for pack {packId}"); + } + + ulong payloadOffset = record.FileSize - record.PayloadBytes; + if (payloadOffset < AMPRPackFormat.DataHeaderSize || payloadOffset % page != 0 || record.FileSize % page != 0) + { + throw new InvalidDataException($"invalid payload offset for pack {packId}"); + } + } + + private void ValidateFile(int fileId, AMPRFileRecord record) + { + string path = StringAt(record.PathOffset, record.PathLength); + string canonical; + try + { + canonical = AMPRAssetPath.Canonical(path); + } + catch (ArgumentException exc) + { + throw new InvalidDataException(exc.Message, exc); + } + + if (canonical != path || canonical == "/app0") + { + throw new InvalidDataException($"non-canonical path for file id {fileId}"); + } + + if (record.Reserved != 0 || (record.Flags & ~AMPRPackFormat.FileKnownFlags) != 0) + { + throw new InvalidDataException($"unknown flags for file id {fileId}"); + } + + if (AMPRAssetPath.Hash(canonical) != record.PathHash) + { + throw new InvalidDataException($"path hash mismatch for file id {fileId}"); + } + + if (!record.IsPacked) + { + if (record.FirstChunk != 0 || record.ChunkCount != 0 || record.BlockShift != 0 || record.PackingClass != 0 || record.Flags != 0) + { + throw new InvalidDataException($"loose file id {fileId} references chunks"); + } + + return; + } + + if (record.BlockShift is < AMPRPackFormat.MinBlockShift or > AMPRPackFormat.MaxBlockShift) + { + throw new InvalidDataException($"invalid block shift for file id {fileId}"); + } + + bool streaming = (record.Flags & AMPRPackFormat.FileFlagStreaming) != 0; + if (streaming && (record.Flags & AMPRPackFormat.FileFlagRandomAccess) != 0) + { + throw new InvalidDataException($"file id {fileId} cannot be both streaming and random-access"); + } + + if ((ulong)record.FirstChunk + record.ChunkCount > (ulong)Chunks.Count) + { + throw new InvalidDataException($"chunk range is invalid for file id {fileId}"); + } + + ulong total = 0; + int blockSize = 1 << record.BlockShift; + for (uint local = 0; local < record.ChunkCount; local++) + { + AMPRChunkRecord chunk = Chunks[(int)(record.FirstChunk + local)]; + ValidateChunk(fileId, record, chunk, local, blockSize, streaming); + total += (ulong)chunk.RawSize; + } + + if (total != record.LogicalSize) + { + throw new InvalidDataException($"logical size mismatch for file id {fileId}"); + } + } + + private void ValidateChunk(int fileId, AMPRFileRecord record, AMPRChunkRecord chunk, uint local, int blockSize, bool streaming) + { + if (chunk.RawSize == 0 || chunk.RawSize > blockSize) + { + throw new InvalidDataException($"invalid raw chunk size for file id {fileId}"); + } + + if (chunk.StoredSize == 0 || chunk.StoredSize > blockSize) + { + throw new InvalidDataException($"invalid stored chunk size for file id {fileId}"); + } + + if (local + 1 != record.ChunkCount && chunk.RawSize != blockSize) + { + throw new InvalidDataException($"short non-final chunk for file id {fileId}"); + } + + if (chunk.Codec is not (AMPRPackFormat.ChunkCodecRaw or AMPRPackFormat.ChunkCodecLZ4)) + { + throw new InvalidDataException($"unknown chunk codec for file id {fileId}"); + } + + if (chunk.Codec == AMPRPackFormat.ChunkCodecRaw && chunk.StoredSize != chunk.RawSize) + { + throw new InvalidDataException($"raw chunk size mismatch for file id {fileId}"); + } + + if (chunk.PackId >= Packs.Count) + { + throw new InvalidDataException($"invalid pack id for file id {fileId}"); + } + + if ((chunk.Flags & ~AMPRPackFormat.ChunkKnownFlags) != 0) + { + throw new InvalidDataException($"invalid chunk flags/reserved for file id {fileId}"); + } + + if (streaming != ((chunk.Flags & AMPRPackFormat.ChunkFlagStreaming) != 0)) + { + throw new InvalidDataException($"streaming flag mismatch for file id {fileId}"); + } + + if ((record.Flags & AMPRPackFormat.FileFlagStoreOnly) != 0 && chunk.Codec != AMPRPackFormat.ChunkCodecRaw) + { + throw new InvalidDataException($"store-only file id {fileId} contains a compressed chunk"); + } + + AMPRPackRecord pack = Packs[chunk.PackId]; + ulong payloadOffset = pack.FileSize - pack.PayloadBytes; + ulong end = chunk.Offset + (ulong)chunk.StoredSize; + if (chunk.Offset < payloadOffset || end > pack.FileSize || chunk.Offset % AMPRPackFormat.PhysicalChunkAlignment != 0) + { + throw new InvalidDataException($"chunk range/alignment is invalid for file id {fileId}"); + } + + ulong page = pack.IOPageSize; + ulong pageBegin = AMPRPackFormat.AlignDown(chunk.Offset, page); + ulong pageEnd = AMPRPackFormat.AlignUp(end, page); + bool pageContained = (chunk.Flags & AMPRPackFormat.ChunkFlagPageContained) != 0; + bool pageAligned = (chunk.Flags & AMPRPackFormat.ChunkFlagPageAligned) != 0; + if (pageContained && ((ulong)chunk.StoredSize > page || pageBegin != AMPRPackFormat.AlignDown(end - 1, page))) + { + throw new InvalidDataException($"page-contained chunk crosses an I/O page for file id {fileId}"); + } + + if (pageAligned && chunk.Offset % page != 0) + { + throw new InvalidDataException($"page-aligned chunk is not page aligned for file id {fileId}"); + } + + bool pageSafe = pageContained || pageAligned; + if ((ulong)chunk.StoredSize <= page && pageAligned && !pageContained) + { + // Small page-aligned extents are also page-contained; requiring both bits catches packers + // that omit the stronger invariant used by the random-access planner. + throw new InvalidDataException($"small page-aligned chunk lacks containment flag for file id {fileId}"); + } + + if ((record.Flags & AMPRPackFormat.FileFlagRandomAccess) != 0 && !pageSafe) + { + throw new InvalidDataException($"random-access file uses a non-page-safe chunk for file id {fileId}"); + } + + if (!streaming && !pageSafe) + { + throw new InvalidDataException($"non-streaming file uses a non-page-safe chunk for file id {fileId}"); + } + + if (pageBegin < payloadOffset || pageEnd > pack.FileSize) + { + throw new InvalidDataException($"aligned I/O range is outside the pack for file id {fileId}"); + } + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRPackReader.cs b/src/MkPFS.Core/AMPR/AMPRPackReader.cs new file mode 100644 index 0000000..621c1da --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRPackReader.cs @@ -0,0 +1,188 @@ +using MkPFS.Core.Compression; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// +/// Reads and checks decoded chunks of a pack set: opens every volume, validates its header against the manifest, +/// and verifies each decoded chunk against the CRC sidecar (Python PackReader). +/// +public sealed class AMPRPackReader : IDisposable +{ + private readonly FileStream[] _volumes; + private readonly (long Begin, long End)[] _payloads; + + /// Open the volumes and CRC sidecar of a loaded manifest. + /// Manifest loaded from disk ( set). + /// The sidecar or a volume is missing or does not match. + /// A volume header is malformed. + public AMPRPackReader(AMPRPackManifest manifest) + { + Manifest = manifest; + string indexPath = manifest.Path ?? throw new ArgumentException("manifest has no path", nameof(manifest)); + try + { + ChunkCrcs = AMPRChunkCrcs.Load(AMPRPackFormat.ChunkCrcPath(indexPath), manifest.BuildId, manifest.Chunks.Count); + } + catch (Exception exc) when (exc is IOException or UnauthorizedAccessException or InvalidDataException) + { + throw new AMPRPackException($"invalid or missing chunk CRC sidecar: {exc.Message}", exc); + } + + string directory = Path.GetDirectoryName(Path.GetFullPath(indexPath))!; + List opened = []; + _payloads = new (long, long)[manifest.Packs.Count]; + try + { + for (int packId = 0; packId < manifest.Packs.Count; packId++) + { + AMPRPackRecord record = manifest.Packs[packId]; + string name = manifest.PackName(packId); + FileStream stream = File.OpenRead(AMPRAssetPath.SafeOutputPath(directory, name)); + opened.Add(stream); + byte[] header = new byte[AMPRPackFormat.DataHeaderSize]; + int read = stream.ReadAtLeast(header, header.Length, throwOnEndOfStream: false); + AMPRDataHeader parsed = AMPRDataHeader.Parse(header.AsSpan(0, read), (uint)packId, manifest.BuildId, record.Flags); + if ((ulong)stream.Length != record.FileSize) + { + throw new AMPRPackException($"pack size mismatch: {name}"); + } + + if (parsed.PayloadBytes != record.PayloadBytes) + { + throw new AMPRPackException($"pack payload size mismatch: {name}"); + } + + if (parsed.PayloadOffset + parsed.PayloadBytes != (ulong)stream.Length) + { + throw new AMPRPackException($"pack payload range mismatch: {name}"); + } + + _payloads[packId] = ((long)parsed.PayloadOffset, (long)(parsed.PayloadOffset + parsed.PayloadBytes)); + } + } + catch + { + foreach (FileStream stream in opened) + { + stream.Dispose(); + } + + throw; + } + + _volumes = [.. opened]; + } + + /// The manifest. + public AMPRPackManifest Manifest { get; } + + /// Decoded CRC-32 per chunk. + public uint[] ChunkCrcs { get; } + + /// Read, decode and CRC-check one chunk (Python read_chunk). + /// Chunk index. + /// Decoded bytes. + /// The chunk is out of range, truncated, undecodable or fails its CRC. + public byte[] ReadChunk(int chunkIndex) + { + if (chunkIndex < 0 || chunkIndex >= ChunkCrcs.Length) + { + throw new AMPRPackException("chunk CRC index is out of range"); + } + + AMPRChunkRecord chunk = Manifest.Chunks[chunkIndex]; + if (chunk.PackId >= _volumes.Length) + { + throw new AMPRPackException("chunk references an invalid pack"); + } + + (long begin, long end) = _payloads[chunk.PackId]; + if ((long)chunk.Offset < begin || (long)chunk.Offset + chunk.StoredSize > end) + { + throw new AMPRPackException("chunk is outside pack payload"); + } + + FileStream volume = _volumes[chunk.PackId]; + volume.Seek((long)chunk.Offset, SeekOrigin.Begin); + byte[] stored = new byte[chunk.StoredSize]; + if (volume.ReadAtLeast(stored, stored.Length, throwOnEndOfStream: false) != stored.Length) + { + throw new AMPRPackException("chunk is truncated"); + } + + byte[] raw; + if (chunk.Codec == AMPRPackFormat.ChunkCodecRaw) + { + raw = stored; + } + else if (chunk.Codec == AMPRPackFormat.ChunkCodecLZ4) + { + raw = new byte[chunk.RawSize]; + try + { + if (LZ4Codec.Decompress(stored, raw) != raw.Length) + { + throw new InvalidDataException("LZ4 output size mismatch"); + } + } + catch (InvalidDataException exc) + { + throw new AMPRPackException("LZ4 chunk decompression failed", exc); + } + } + else + { + throw new AMPRPackException($"unsupported chunk codec: {chunk.Codec}"); + } + + return raw.Length == chunk.RawSize && Crc32.Update(0, raw) == ChunkCrcs[chunkIndex] + ? raw + : throw new AMPRPackException("raw chunk CRC/size mismatch"); + } + + /// Decoded chunks of a packed file in order (Python iter_file_data). + /// File id. + /// Decoded chunks. + /// The file id is invalid or loose, or the data does not add up. + public IEnumerable ReadFile(int fileId) + { + if (fileId <= 0 || fileId > Manifest.Files.Count) + { + throw new AMPRPackException($"invalid file id: {fileId}"); + } + + AMPRFileRecord record = Manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + throw new AMPRPackException($"file id {fileId} is loose and has no packed data"); + } + + return Iterate(record, fileId); + } + + /// + public void Dispose() + { + foreach (FileStream volume in _volumes) + { + volume.Dispose(); + } + } + + private IEnumerable Iterate(AMPRFileRecord record, int fileId) + { + ulong total = 0; + for (uint local = 0; local < record.ChunkCount; local++) + { + byte[] raw = ReadChunk((int)(record.FirstChunk + local)); + total += (ulong)raw.Length; + yield return raw; + } + + if (total != record.LogicalSize) + { + throw new AMPRPackException($"file id {fileId} logical size mismatch"); + } + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRPackRecords.cs b/src/MkPFS.Core/AMPR/AMPRPackRecords.cs new file mode 100644 index 0000000..35c0515 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRPackRecords.cs @@ -0,0 +1,181 @@ +using System.Buffers.Binary; + +namespace MkPFS.Core.AMPR; + +/// +/// AMPRPAK4 file record (48 bytes, <QQqIIIIIBBH). One per AMPRIDX3 file, so +/// fileId == recordIndex + 1. Loose files have no chunks and zero flags. +/// +/// FNV-1a 64 of the ASCII-folded canonical path. +/// Decoded file size. +/// Modification time, Unix seconds. +/// Index of the first chunk record. +/// Number of chunk records. +/// Path offset in the string table. +/// Path length in bytes (without NUL). +/// FileFlag* bits. +/// log2 of the block size (0 for loose files). +/// Index of the packing group in sorted group order. +/// Must be 0. +public readonly record struct AMPRFileRecord( + ulong PathHash, + ulong LogicalSize, + long MTime, + uint FirstChunk, + uint ChunkCount, + uint PathOffset, + uint PathLength, + uint Flags, + byte BlockShift, + byte PackingClass = 0, + ushort Reserved = 0) +{ + /// Whether the file is stored in pack volumes. + public bool IsPacked => (Flags & AMPRPackFormat.FileFlagPacked) != 0; + + /// Write the record. + /// At least 48 bytes. + public void Write(Span destination) + { + BinaryPrimitives.WriteUInt64LittleEndian(destination, PathHash); + BinaryPrimitives.WriteUInt64LittleEndian(destination[8..], LogicalSize); + BinaryPrimitives.WriteInt64LittleEndian(destination[16..], MTime); + BinaryPrimitives.WriteUInt32LittleEndian(destination[24..], FirstChunk); + BinaryPrimitives.WriteUInt32LittleEndian(destination[28..], ChunkCount); + BinaryPrimitives.WriteUInt32LittleEndian(destination[32..], PathOffset); + BinaryPrimitives.WriteUInt32LittleEndian(destination[36..], PathLength); + BinaryPrimitives.WriteUInt32LittleEndian(destination[40..], Flags); + destination[44] = BlockShift; + destination[45] = PackingClass; + BinaryPrimitives.WriteUInt16LittleEndian(destination[46..], Reserved); + } + + /// Read a record. + /// At least 48 bytes. + /// The record. + public static AMPRFileRecord Read(ReadOnlySpan source) => new( + BinaryPrimitives.ReadUInt64LittleEndian(source), + BinaryPrimitives.ReadUInt64LittleEndian(source[8..]), + BinaryPrimitives.ReadInt64LittleEndian(source[16..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[24..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[28..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[32..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[36..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[40..]), + source[44], + source[45], + BinaryPrimitives.ReadUInt16LittleEndian(source[46..])); +} + +/// +/// AMPRPAK4 chunk record (12 bytes, <QI): a 64-bit location (48-bit offset, 16-bit pack id) and a +/// 32-bit descriptor (20-bit stored_size - 1, 2-bit codec, 8-bit flags). The decoded size is not stored; +/// derives from the owning file's block geometry. +/// +/// Absolute offset in the pack volume. +/// Stored bytes (1 .. 1 MiB). +/// Decoded bytes (0 until derived). +/// Pack volume index. +/// ChunkCodecRaw or ChunkCodecLZ4. +/// ChunkFlag* bits. +public readonly record struct AMPRChunkRecord(ulong Offset, int StoredSize, int RawSize, int PackId, int Codec, int Flags = 0) +{ + /// Write the record (Python ChunkRecord.pack). + /// At least 12 bytes. + /// A field is outside its bit domain. + public void Write(Span destination) + { + if (Offset > AMPRPackFormat.ChunkOffsetMask) + { + throw new ArgumentException("chunk offset exceeds the 48-bit AMPRPAK4 domain"); + } + + if (PackId is < 0 or > 0xFFFF) + { + throw new ArgumentException("chunk pack id exceeds uint16"); + } + + if (StoredSize < 1 || StoredSize > 1 << AMPRPackFormat.MaxBlockShift) + { + throw new ArgumentException("chunk stored size is outside the AMPRPAK4 domain"); + } + + if (Codec < 0 || Codec > AMPRPackFormat.ChunkCodecMask) + { + throw new ArgumentException("chunk codec is outside the compact descriptor domain"); + } + + if ((Flags & ~AMPRPackFormat.ChunkKnownFlags) != 0) + { + throw new ArgumentException("chunk contains unknown flags"); + } + + ulong location = Offset | ((ulong)PackId << 48); + uint descriptor = (uint)(StoredSize - 1) + | ((uint)Codec << AMPRPackFormat.ChunkCodecShift) + | ((uint)Flags << AMPRPackFormat.ChunkFlagsShift); + BinaryPrimitives.WriteUInt64LittleEndian(destination, location); + BinaryPrimitives.WriteUInt32LittleEndian(destination[8..], descriptor); + } + + /// Read a record; is 0 (Python ChunkRecord.unpack_from). + /// At least 12 bytes. + /// The record. + /// Reserved descriptor bits are set. + public static AMPRChunkRecord Read(ReadOnlySpan source) + { + ulong location = BinaryPrimitives.ReadUInt64LittleEndian(source); + uint descriptor = BinaryPrimitives.ReadUInt32LittleEndian(source[8..]); + if ((descriptor & ~AMPRPackFormat.ChunkDescriptorKnownMask) != 0) + { + throw new InvalidDataException("chunk descriptor contains reserved bits"); + } + + return new AMPRChunkRecord( + location & AMPRPackFormat.ChunkOffsetMask, + (int)(descriptor & AMPRPackFormat.ChunkStoredMask) + 1, + 0, + (int)(location >> 48), + (int)((descriptor >> AMPRPackFormat.ChunkCodecShift) & AMPRPackFormat.ChunkCodecMask), + (int)((descriptor >> AMPRPackFormat.ChunkFlagsShift) & AMPRPackFormat.ChunkFlagsMask)); + } +} + +/// AMPRPAK4 pack (data volume) record (32 bytes, <QQIIII). +/// Payload bytes at the end of the volume. +/// Volume file size. +/// Volume file name offset in the string table. +/// Volume file name length in bytes. +/// PackFlag* bits. +/// Filesystem I/O page size the layout was planned for. +public readonly record struct AMPRPackRecord( + ulong PayloadBytes, + ulong FileSize, + uint NameOffset, + uint NameLength, + uint Flags, + uint IOPageSize) +{ + /// Write the record. + /// At least 32 bytes. + public void Write(Span destination) + { + BinaryPrimitives.WriteUInt64LittleEndian(destination, PayloadBytes); + BinaryPrimitives.WriteUInt64LittleEndian(destination[8..], FileSize); + BinaryPrimitives.WriteUInt32LittleEndian(destination[16..], NameOffset); + BinaryPrimitives.WriteUInt32LittleEndian(destination[20..], NameLength); + BinaryPrimitives.WriteUInt32LittleEndian(destination[24..], Flags); + BinaryPrimitives.WriteUInt32LittleEndian(destination[28..], IOPageSize); + } + + /// Read a record. + /// At least 32 bytes. + /// The record. + public static AMPRPackRecord Read(ReadOnlySpan source) => new( + BinaryPrimitives.ReadUInt64LittleEndian(source), + BinaryPrimitives.ReadUInt64LittleEndian(source[8..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[16..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[20..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[24..]), + BinaryPrimitives.ReadUInt32LittleEndian(source[28..])); +} diff --git a/src/MkPFS.Core/AMPR/AMPRPackTools.cs b/src/MkPFS.Core/AMPR/AMPRPackTools.cs new file mode 100644 index 0000000..a19a2df --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRPackTools.cs @@ -0,0 +1,360 @@ +namespace MkPFS.Core.AMPR; + +/// Result of . +/// Packed files checked. +/// Distinct physical chunks decoded. +/// Stored bytes of those chunks. +/// Decoded bytes of those chunks. +public sealed record AMPRVerifyResult(long Files, long PhysicalChunks, long StoredBytes, long RawBytes); + +/// Result of . +/// Files compared. +/// Chunks compared. +/// Bytes compared. +public sealed record AMPRSourceCompareResult(long Files, long Chunks, long Bytes); + +/// Result of . +/// Files written. +/// Bytes written. +public sealed record AMPRExtractResult(long Files, long Bytes); + +/// One row of . +/// File id. +/// Canonical /app0 path. +/// Stored in volumes. +/// Decoded size. +/// Sum of stored chunk sizes (shared chunks counted per reference). +/// Block size, 0 for loose files. +/// Chunk records. +/// lz4/raw, sorted. +/// Pack ids, sorted. +/// I/O page sizes of those packs, sorted. +/// loose, streaming, random or mixed. +/// Streaming flag. +/// Random-access flag. +/// Hot flag. +public sealed record AMPRListEntry( + int FileId, + string Path, + bool Packed, + ulong LogicalSize, + long StoredSize, + long BlockSize, + uint Chunks, + IReadOnlyList Codecs, + IReadOnlyList Packs, + IReadOnlyList IOPageSizes, + string Layout, + bool Streaming, + bool RandomAccess, + bool Hot); + +/// One volume in . +/// Pack id. +/// Volume name. +/// Volume size. +/// Payload bytes. +/// I/O page size. +/// Volume size in pages. +/// Pack flags. +public sealed record AMPRInspectPack(int Id, string Name, ulong FileSize, ulong PayloadBytes, uint IOPageSize, ulong IOPages, uint Flags); + +/// Result of . +/// Build id, lower-case hex. +/// Runtime settings, if present. +/// Files. +/// Packed files. +/// Loose files. +/// Chunk records. +/// Volumes. +public sealed record AMPRInspectResult( + string BuildId, AMPRRuntimeSettings? Runtime, long Files, long PackedFiles, long LooseFiles, long Chunks, IReadOnlyList Packs); + +/// Offline pack-set tools (ampr_pack verify, unpack, list, inspect). +public static class AMPRPackTools +{ + /// File ids whose relative path matches any pattern; all ids without patterns (Python _selected_file_ids). + /// Manifest. + /// Globs over paths relative to /app0. + /// File ids in order. + public static List SelectedFileIds(AMPRPackManifest manifest, IReadOnlyList patterns) + { + List selected = []; + for (int fileId = 1; fileId <= manifest.Files.Count; fileId++) + { + if (patterns.Count == 0 || AMPRGlob.Matches(AMPRAssetPath.Relative(manifest.FilePath(fileId)), patterns)) + { + selected.Add(fileId); + } + } + + return selected; + } + + /// + /// Decode every selected packed file once per physical chunk and check sizes and CRCs; also validates the + /// runtime settings (Python verify_packs). + /// + /// Manifest path. + /// File globs. + /// Counters. + public static AMPRVerifyResult Verify(string indexPath, IReadOnlyList? patterns = null) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + AMPRRuntimeSettings.Read(AMPRPackFormat.RuntimePath(indexPath), manifest.BuildId); + Dictionary<(int Pack, ulong Offset, int Stored), (int Codec, int Raw, uint Crc)> seen = []; + long files = 0; + long chunks = 0; + long stored = 0; + long raw = 0; + using AMPRPackReader reader = new(manifest); + foreach (int fileId in SelectedFileIds(manifest, patterns ?? [])) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + continue; + } + + ulong total = 0; + for (uint local = 0; local < record.ChunkCount; local++) + { + int chunkIndex = (int)(record.FirstChunk + local); + AMPRChunkRecord chunk = manifest.Chunks[chunkIndex]; + (int, ulong, int) physical = (chunk.PackId, chunk.Offset, chunk.StoredSize); + (int, int, uint) descriptor = (chunk.Codec, chunk.RawSize, reader.ChunkCrcs[chunkIndex]); + if (seen.TryGetValue(physical, out (int, int, uint) prior)) + { + if (prior != descriptor) + { + throw new AMPRPackException("one physical chunk has conflicting metadata"); + } + + total += (ulong)chunk.RawSize; + continue; + } + + total += (ulong)reader.ReadChunk(chunkIndex).Length; + seen[physical] = descriptor; + chunks++; + stored += chunk.StoredSize; + raw += chunk.RawSize; + } + + if (total != record.LogicalSize) + { + throw new AMPRPackException($"logical size mismatch for file id {fileId}"); + } + + files++; + } + + return new AMPRVerifyResult(files, chunks, stored, raw); + } + + /// Rebuild every selected packed file and compare it byte for byte with the source tree (Python verify_packs_against_root). + /// Manifest path. + /// /app0 source tree. + /// File globs. + /// Counters. + public static AMPRSourceCompareResult VerifyAgainstRoot(string indexPath, string root, IReadOnlyList? patterns = null) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + root = Path.GetFullPath(root); + long files = 0; + long chunks = 0; + long bytes = 0; + using AMPRPackReader reader = new(manifest); + foreach (int fileId in SelectedFileIds(manifest, patterns ?? [])) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + continue; + } + + string relative = AMPRAssetPath.Relative(manifest.FilePath(fileId)); + string sourcePath = AMPRAssetPath.SafeOutputPath(root, relative); + FileInfo info = new(sourcePath); + if (!info.Exists) + { + throw new AMPRPackException($"source file missing while comparing packed data: {relative}"); + } + + if ((ulong)info.Length != record.LogicalSize) + { + throw new AMPRPackException($"source size mismatch for {relative}: manifest={record.LogicalSize}, disk={info.Length}"); + } + + long logicalOffset = 0; + using (FileStream source = File.OpenRead(sourcePath)) + { + int local = 0; + foreach (byte[] packed in reader.ReadFile(fileId)) + { + byte[] sourceRaw = new byte[packed.Length]; + if (source.ReadAtLeast(sourceRaw, sourceRaw.Length, throwOnEndOfStream: false) != sourceRaw.Length) + { + throw new AMPRPackException($"source truncated for {relative} at 0x{logicalOffset:x}"); + } + + int mismatch = sourceRaw.AsSpan().CommonPrefixLength(packed); + if (mismatch < packed.Length) + { + AMPRChunkRecord chunk = manifest.Chunks[(int)record.FirstChunk + local]; + throw new AMPRPackException( + $"packed/source mismatch for {relative}: logical=0x{logicalOffset + mismatch:x}, chunk={local}, " + + $"pack={chunk.PackId}, physical=0x{chunk.Offset:x}, source=0x{sourceRaw[mismatch]:x2}, packed=0x{packed[mismatch]:x2}"); + } + + logicalOffset += packed.Length; + bytes += packed.Length; + chunks++; + local++; + } + + if (source.ReadByte() != -1) + { + throw new AMPRPackException($"source grew while comparing: {relative}"); + } + } + + if ((ulong)logicalOffset != record.LogicalSize) + { + throw new AMPRPackException($"reconstructed size mismatch for {relative}: expected={record.LogicalSize}, got={logicalOffset}"); + } + + files++; + } + + return new AMPRSourceCompareResult(files, chunks, bytes); + } + + /// Write selected packed files under (Python extract_packs). + /// Manifest path. + /// Output directory. + /// File globs. + /// Replace existing files. + /// Set each file's modification time from the manifest. + /// Counters. + public static AMPRExtractResult Extract( + string indexPath, string outputDir, IReadOnlyList? patterns = null, bool overwrite = false, bool preserveMTime = true) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + Directory.CreateDirectory(outputDir); + long files = 0; + long bytes = 0; + using AMPRPackReader reader = new(manifest); + foreach (int fileId in SelectedFileIds(manifest, patterns ?? [])) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + continue; + } + + string destination = AMPRAssetPath.SafeOutputPath(outputDir, AMPRAssetPath.Relative(manifest.FilePath(fileId))); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + if ((File.Exists(destination) || Directory.Exists(destination)) && !overwrite) + { + throw new AMPRPackException($"destination exists: {destination}"); + } + + string temp = Path.Combine(Path.GetDirectoryName(destination)!, $".{Path.GetFileName(destination)}.tmp-{Environment.ProcessId}"); + try + { + using (FileStream handle = new(temp, FileMode.Create, FileAccess.Write, FileShare.None)) + { + foreach (byte[] raw in reader.ReadFile(fileId)) + { + handle.Write(raw); + bytes += raw.Length; + } + + handle.Flush(flushToDisk: true); + } + + File.Move(temp, destination, overwrite: true); + if (preserveMTime) + { + DateTime time = DateTime.UnixEpoch.AddSeconds(record.MTime); + File.SetLastWriteTimeUtc(destination, time); + File.SetLastAccessTimeUtc(destination, time); + } + + files++; + } + finally + { + File.Delete(temp); + } + } + + return new AMPRExtractResult(files, bytes); + } + + /// Per-file placement summary (Python list_manifest). + /// Manifest path. + /// File globs. + /// Rows. + public static List List(string indexPath, IReadOnlyList? patterns = null) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + List rows = []; + foreach (int fileId in SelectedFileIds(manifest, patterns ?? [])) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + long stored = 0; + SortedSet codecs = new(StringComparer.Ordinal); + SortedSet packs = []; + for (uint local = 0; local < record.ChunkCount; local++) + { + AMPRChunkRecord chunk = manifest.Chunks[(int)(record.FirstChunk + local)]; + stored += chunk.StoredSize; + codecs.Add(chunk.Codec == AMPRPackFormat.ChunkCodecLZ4 ? "lz4" : "raw"); + packs.Add(chunk.PackId); + } + + bool streaming = (record.Flags & AMPRPackFormat.FileFlagStreaming) != 0; + bool randomAccess = (record.Flags & AMPRPackFormat.FileFlagRandomAccess) != 0; + string layout = !record.IsPacked ? "loose" : streaming ? "streaming" : randomAccess ? "random" : "mixed"; + rows.Add(new AMPRListEntry( + fileId, + manifest.FilePath(fileId), + record.IsPacked, + record.LogicalSize, + stored, + record.BlockShift != 0 ? 1L << record.BlockShift : 0, + record.ChunkCount, + [.. codecs], + [.. packs], + [.. packs.Select(id => manifest.Packs[id].IOPageSize).Distinct().Order()], + layout, + streaming, + randomAccess, + (record.Flags & AMPRPackFormat.FileFlagHot) != 0)); + } + + return rows; + } + + /// Manifest-level summary, validating the runtime settings (CLI inspect). + /// Manifest path. + /// Summary. + public static AMPRInspectResult Inspect(string indexPath) + { + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + AMPRRuntimeSettings? runtime = AMPRRuntimeSettings.Read(AMPRPackFormat.RuntimePath(indexPath), manifest.BuildId); + long packed = manifest.Files.Count(record => record.IsPacked); + List packs = []; + for (int packId = 0; packId < manifest.Packs.Count; packId++) + { + AMPRPackRecord record = manifest.Packs[packId]; + packs.Add(new AMPRInspectPack( + packId, manifest.PackName(packId), record.FileSize, record.PayloadBytes, record.IOPageSize, record.FileSize / record.IOPageSize, record.Flags)); + } + + return new AMPRInspectResult( + Convert.ToHexStringLower(manifest.BuildId), runtime, manifest.Files.Count, packed, manifest.Files.Count - packed, manifest.Chunks.Count, packs); + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRRuntimeSettings.cs b/src/MkPFS.Core/AMPR/AMPRRuntimeSettings.cs new file mode 100644 index 0000000..edc5c81 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRRuntimeSettings.cs @@ -0,0 +1,128 @@ +using System.Buffers.Binary; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// +/// Per-title runtime settings (<manifest>.runtime, AMPRCFG1, 64 bytes, <8sII16sQQIIII) bound to +/// a manifest build id (Python RuntimeSettings, read_runtime_settings). +/// +/// Decoded-block cache size. +/// Physical-page cache size. +/// Decode workers, 1..16. +/// Workers reserved for latency-class requests, below . +public sealed record AMPRRuntimeSettings(long DecodedCacheBytes, long PhysicalCacheBytes, long Workers, long LatencyReserveWorkers) +{ + /// Check the value ranges. + /// A value is out of range. + public void Validate() + { + if (Workers < 1 || Workers > 16 || LatencyReserveWorkers < 0 || LatencyReserveWorkers >= Workers) + { + throw new ArgumentException("runtime workers must be 1..16 and reserve must be smaller"); + } + + foreach (long value in (ReadOnlySpan)[DecodedCacheBytes, PhysicalCacheBytes]) + { + if (value < 0 || value % 16384 != 0) + { + throw new ArgumentException("runtime cache sizes must be nonnegative 16 KiB multiples"); + } + } + } + + /// Serialize for a manifest build id. + /// Manifest build id. + /// 64 bytes. + /// A value is out of range or the build id is not 16 bytes. + public byte[] Encode(ReadOnlySpan buildId) + { + Validate(); + if (buildId.Length != AMPRPackFormat.BuildIdSize) + { + throw new ArgumentException("runtime profile needs a 16-byte build ID"); + } + + byte[] data = new byte[AMPRPackFormat.RuntimeSize]; + Span span = data; + AMPRPackFormat.RuntimeMagic.CopyTo(span); + BinaryPrimitives.WriteUInt32LittleEndian(span[8..], AMPRPackFormat.RuntimeVersion); + BinaryPrimitives.WriteUInt32LittleEndian(span[12..], AMPRPackFormat.RuntimeSize); + buildId.CopyTo(span[16..]); + BinaryPrimitives.WriteUInt64LittleEndian(span[32..], (ulong)DecodedCacheBytes); + BinaryPrimitives.WriteUInt64LittleEndian(span[40..], (ulong)PhysicalCacheBytes); + BinaryPrimitives.WriteUInt32LittleEndian(span[48..], (uint)Workers); + BinaryPrimitives.WriteUInt32LittleEndian(span[52..], (uint)LatencyReserveWorkers); + BinaryPrimitives.WriteUInt32LittleEndian(span[56..], Crc32.Update(0, span)); + return data; + } + + /// Read settings bound to ; when the file does not exist. + /// Settings path. + /// Manifest build id. + /// Settings, or . + /// The file is malformed or belongs to another build. + public static AMPRRuntimeSettings? Read(string path, ReadOnlySpan buildId) + { + byte[] data; + try + { + data = File.ReadAllBytes(path); + } + catch (FileNotFoundException) + { + return null; + } + + return Parse(data, buildId); + } + + /// Parse settings bytes bound to . + /// Settings bytes. + /// Manifest build id. + /// Settings. + /// The bytes are malformed or belong to another build. + public static AMPRRuntimeSettings Parse(ReadOnlySpan data, ReadOnlySpan buildId) + { + if (data.Length != AMPRPackFormat.RuntimeSize) + { + throw new InvalidDataException("invalid runtime profile size"); + } + + byte[] copy = data.ToArray(); + BinaryPrimitives.WriteUInt32LittleEndian(copy.AsSpan(56), 0); + if (!data[..8].SequenceEqual(AMPRPackFormat.RuntimeMagic) + || BinaryPrimitives.ReadUInt32LittleEndian(data[8..]) != AMPRPackFormat.RuntimeVersion + || BinaryPrimitives.ReadUInt32LittleEndian(data[12..]) != AMPRPackFormat.RuntimeSize + || !data.Slice(16, AMPRPackFormat.BuildIdSize).SequenceEqual(buildId) + || BinaryPrimitives.ReadUInt32LittleEndian(data[60..]) != 0 + || Crc32.Update(0, copy) != BinaryPrimitives.ReadUInt32LittleEndian(data[56..])) + { + throw new InvalidDataException("invalid runtime profile header, build ID or CRC"); + } + + ulong decoded = BinaryPrimitives.ReadUInt64LittleEndian(data[32..]); + ulong physical = BinaryPrimitives.ReadUInt64LittleEndian(data[40..]); + if (decoded > long.MaxValue || physical > long.MaxValue) + { + // Python accepts values below 2**64; no 16 KiB-aligned cache that large is meaningful. + throw new InvalidDataException("runtime cache sizes must be nonnegative 16 KiB multiples"); + } + + AMPRRuntimeSettings result = new( + (long)decoded, + (long)physical, + BinaryPrimitives.ReadUInt32LittleEndian(data[48..]), + BinaryPrimitives.ReadUInt32LittleEndian(data[52..])); + try + { + result.Validate(); + } + catch (ArgumentException exc) + { + throw new InvalidDataException(exc.Message, exc); + } + + return result; + } +} diff --git a/src/MkPFS.Core/AMPR/AMPRSize.cs b/src/MkPFS.Core/AMPR/AMPRSize.cs new file mode 100644 index 0000000..f7f8f29 --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRSize.cs @@ -0,0 +1,102 @@ +using System.Globalization; +using MkPFS.Core.Util; + +namespace MkPFS.Core.AMPR; + +/// Size strings such as 64KiB or 1.5MB (Python parse_size, block_shift). +public static class AMPRSize +{ + private static readonly Dictionary Units = new(StringComparer.Ordinal) + { + ["b"] = 1, + ["k"] = 1000, + ["kb"] = 1000, + ["kib"] = 1024, + ["m"] = 1000L * 1000, + ["mb"] = 1000L * 1000, + ["mib"] = 1024L * 1024, + ["g"] = 1000L * 1000 * 1000, + ["gb"] = 1000L * 1000 * 1000, + ["gib"] = 1024L * 1024 * 1024, + ["t"] = 1000L * 1000 * 1000 * 1000, + ["tb"] = 1000L * 1000 * 1000 * 1000, + ["tib"] = 1024L * 1024 * 1024 * 1024, + }; + + /// Return a byte count; integers pass through. + /// Non-negative byte count. + /// . + /// The value is negative. + public static long Parse(long value) => + value >= 0 ? value : throw new ArgumentException("size must be non-negative"); + + /// + /// Parse a number with an optional unit (b, k/kb/kib ... tib, case-insensitive; + /// underscores ignored). Decimal units are powers of 1000; the product is truncated like Python int(). + /// + /// Size text. + /// Byte count. + /// The text is empty, malformed, negative or has an unknown unit. + public static long Parse(string value) + { + string text = PythonText.Strip(value).Replace("_", string.Empty, StringComparison.Ordinal); + if (text.Length == 0) + { + throw new ArgumentException("empty size"); + } + + int split = text.Length; + while (split > 0 && char.IsLetter(text[split - 1])) + { + split--; + } + + string number = text[..split]; + string suffix = split == text.Length ? "b" : text[split..].ToLowerInvariant(); + if (!Units.TryGetValue(suffix, out long unit)) + { + throw new ArgumentException($"unknown size suffix: {suffix}"); + } + + if (!double.TryParse(PythonText.Strip(number), NumberStyles.Float, CultureInfo.InvariantCulture, out double numeric) + || !double.IsFinite(numeric)) + { + throw new ArgumentException($"invalid size: {value}"); + } + + double product = Math.Truncate(numeric * unit); + if (product < 0) + { + throw new ArgumentException("size must be non-negative"); + } + + return product < 9.2233720368547758E18 ? (long)product : throw new ArgumentException($"invalid size: {value}"); + } + + /// Return log2 of a power-of-two block size between 16 KiB and 1 MiB. + /// Block size in bytes. + /// Shift 14..20. + /// The size is not a power of two or is out of range. + public static int BlockShift(long size) + { + size = Parse(size); + if (size == 0 || (size & (size - 1)) != 0) + { + throw new ArgumentException($"block size must be a power of two, got {size}"); + } + + int shift = 63 - (int)long.LeadingZeroCount(size); + if (shift < AMPRPackFormat.MinBlockShift || shift > AMPRPackFormat.MaxBlockShift) + { + throw new ArgumentException( + $"block size must be between {1 << AMPRPackFormat.MinBlockShift} and {1 << AMPRPackFormat.MaxBlockShift} bytes"); + } + + return shift; + } + + /// Parse a size string and return its block shift. + /// Block size text. + /// Shift 14..20. + public static int BlockShift(string size) => BlockShift(Parse(size)); +} diff --git a/src/MkPFS.Core/AMPR/AMPRStringTable.cs b/src/MkPFS.Core/AMPR/AMPRStringTable.cs new file mode 100644 index 0000000..d12486a --- /dev/null +++ b/src/MkPFS.Core/AMPR/AMPRStringTable.cs @@ -0,0 +1,44 @@ +using System.Text; + +namespace MkPFS.Core.AMPR; + +/// Deduplicating UTF-8 string table; each entry ends with NUL (Python StringTable). +public sealed class AMPRStringTable +{ + private readonly List _data = []; + private readonly Dictionary _offsets = new(StringComparer.Ordinal); + + /// Add a string, or return the location of an identical earlier one. + /// String without NUL. + /// Offset and byte length (without NUL). + /// The value contains NUL or the table exceeds 4 GiB. + public (uint Offset, uint Length) Add(string value) + { + if (_offsets.TryGetValue(value, out (uint Offset, uint Length) prior)) + { + return prior; + } + + byte[] encoded = Encoding.UTF8.GetBytes(value); + if (encoded.Contains((byte)0)) + { + throw new ArgumentException("NUL in string table value"); + } + + long offset = _data.Count; + if (offset + encoded.Length + 1 > 0x100000000L) + { + throw new ArgumentException("string table exceeds uint32 format limits"); + } + + _data.AddRange(encoded); + _data.Add(0); + (uint Offset, uint Length) result = ((uint)offset, (uint)encoded.Length); + _offsets[value] = result; + return result; + } + + /// Return the table bytes. + /// Table bytes. + public byte[] ToArray() => [.. _data]; +} diff --git a/src/MkPFS.Core/Compression/LZ4Codec.cs b/src/MkPFS.Core/Compression/LZ4Codec.cs new file mode 100644 index 0000000..ffa910c --- /dev/null +++ b/src/MkPFS.Core/Compression/LZ4Codec.cs @@ -0,0 +1,108 @@ +using System.Runtime.InteropServices; + +namespace MkPFS.Core.Compression; + +/// +/// Raw LZ4 block helpers (no frame, no size prefix) backed by the bundled lz4 1.9.4. Output is +/// byte-identical to ampr_emu ampr_pack_format.Lz4Codec with python-lz4 4.4.5. +/// +public static class LZ4Codec +{ + /// Highest HC level; ampr_pack's default. + public const int MaxHCLevel = 12; + + /// lz4 version string reported by the native library (expected 1.9.4). + public static unsafe string NativeVersion => + Marshal.PtrToStringAnsi((nint)LZ4Native.Version()) ?? string.Empty; + + /// Return the worst-case block size for bytes. + /// Uncompressed length. + /// Upper bound of the raw block length. + public static int CompressBound(int sourceLength) + { + ArgumentOutOfRangeException.ThrowIfNegative(sourceLength); + int bound = LZ4Native.Bound(sourceLength); + return bound > 0 ? bound : throw new ArgumentOutOfRangeException(nameof(sourceLength), "input exceeds LZ4_MAX_INPUT_SIZE"); + } + + /// Decode one raw block, like the ampr_emu runtime (LZ4_decompress_safe). + /// Raw LZ4 block. + /// Output buffer. + /// Number of bytes written to . + /// The block is malformed or does not fit. + public static unsafe int Decompress(ReadOnlySpan source, Span destination) + { + int written; + fixed (byte* src = source) + fixed (byte* dst = destination) + { + written = LZ4Native.DecompressSafe(src, source.Length, dst, destination.Length); + } + + return written >= 0 + ? written + : throw new InvalidDataException($"LZ4 block is malformed or does not fit in {destination.Length} bytes"); + } +} + +/// +/// Reusable LZ4 encoder; every call produces an independent raw block. Calls lz4 the way python-lz4 +/// does (stream reset + *_continue), which differs from LZ4_compress_fast. Not +/// thread-safe; use one instance per worker. +/// +public sealed class LZ4Encoder : IDisposable +{ + private readonly LZ4EncoderHandle _handle = new(); + + /// Create an encoder (about 280 KB of native state). + public LZ4Encoder() + { + nint raw = LZ4Native.EncoderCreate(); + if (raw == 0) + { + throw new OutOfMemoryException("LZ4 encoder allocation failed"); + } + + Marshal.InitHandle(_handle, raw); + } + + /// Compress with the fast encoder. + /// Bytes to compress. + /// Acceleration; values below 1 become 1, like ampr_pack. + /// The raw block; empty for empty input, like ampr_pack. + public byte[] CompressFast(ReadOnlySpan source, int acceleration = 1) => + Compress(source, highCompression: false, Math.Max(1, acceleration)); + + /// Compress with the HC encoder. + /// Bytes to compress. + /// HC level; clamped to 1..12, like ampr_pack. + /// The raw block; empty for empty input, like ampr_pack. + public byte[] CompressHC(ReadOnlySpan source, int level = LZ4Codec.MaxHCLevel) => + Compress(source, highCompression: true, Math.Clamp(level, 1, LZ4Codec.MaxHCLevel)); + + private unsafe byte[] Compress(ReadOnlySpan source, bool highCompression, int parameter) + { + ObjectDisposedException.ThrowIf(_handle.IsClosed, this); + if (source.IsEmpty) + { + return []; + } + + byte[] buffer = new byte[LZ4Codec.CompressBound(source.Length)]; + int written; + fixed (byte* src = source) + fixed (byte* dst = buffer) + { + written = highCompression + ? LZ4Native.CompressHC(_handle, src, source.Length, dst, buffer.Length, parameter) + : LZ4Native.CompressFast(_handle, src, source.Length, dst, buffer.Length, parameter); + } + + return written > 0 + ? buffer.AsSpan(0, written).ToArray() + : throw new InvalidOperationException("LZ4 output exceeded LZ4_compressBound"); + } + + /// + public void Dispose() => _handle.Dispose(); +} diff --git a/src/MkPFS.Core/Compression/LZ4Native.cs b/src/MkPFS.Core/Compression/LZ4Native.cs new file mode 100644 index 0000000..db6b7bc --- /dev/null +++ b/src/MkPFS.Core/Compression/LZ4Native.cs @@ -0,0 +1,59 @@ +using System.Runtime.InteropServices; + +namespace MkPFS.Core.Compression; + +/// P/Invoke surface of the native mkpfs_lz4_* shim (see native/mkpfs_lz4.c). +internal static unsafe partial class LZ4Native +{ + private const string Library = "mkpfs_zlib"; + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_version")] + internal static partial byte* Version(); + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_bound")] + internal static partial int Bound(int sourceLength); + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_encoder_create")] + internal static partial nint EncoderCreate(); + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_encoder_free")] + internal static partial void EncoderFree(nint handle); + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_compress_fast")] + internal static partial int CompressFast( + LZ4EncoderHandle handle, + byte* source, + int sourceLength, + byte* destination, + int destinationCapacity, + int acceleration); + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_compress_hc")] + internal static partial int CompressHC( + LZ4EncoderHandle handle, + byte* source, + int sourceLength, + byte* destination, + int destinationCapacity, + int level); + + [LibraryImport(Library, EntryPoint = "mkpfs_lz4_decompress_safe")] + internal static partial int DecompressSafe(byte* source, int sourceLength, byte* destination, int destinationCapacity); +} + +/// Owns a native encoder created by mkpfs_lz4_encoder_create. +internal sealed class LZ4EncoderHandle : SafeHandle +{ + public LZ4EncoderHandle() + : base(invalidHandleValue: 0, ownsHandle: true) + { + } + + public override bool IsInvalid => handle == 0; + + protected override bool ReleaseHandle() + { + LZ4Native.EncoderFree(handle); + return true; + } +} diff --git a/src/MkPFS.Core/Util/CpuTopology.cs b/src/MkPFS.Core/Util/CpuTopology.cs new file mode 100644 index 0000000..17edce2 --- /dev/null +++ b/src/MkPFS.Core/Util/CpuTopology.cs @@ -0,0 +1,125 @@ +using System.Globalization; +using System.Runtime.InteropServices; + +namespace MkPFS.Core.Util; + +/// +/// Physical CPU cores of this machine. counts logical processors, which is +/// twice the core count on a CPU with SMT (Hyper-Threading); .NET has no API for cores, so each OS is asked directly. +/// +public static partial class CpuTopology +{ + private static readonly Lazy Cores = new(Detect); + + /// Physical cores, at least 1 and at most ; the logical processor + /// count when the OS does not say. + public static int PhysicalCores => Cores.Value; + + private static int Detect() + { + int logical = Environment.ProcessorCount; + int? cores = null; + try + { + cores = OperatingSystem.IsWindows() ? WindowsCores() + : OperatingSystem.IsLinux() ? LinuxCores("/sys/devices/system/cpu") + : OperatingSystem.IsMacOS() ? MacCores() + : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DllNotFoundException + or EntryPointNotFoundException or FormatException) + { + cores = null; + } + + return cores is > 0 ? Math.Min(cores.Value, logical) : logical; + } + + /// + /// Distinct (package, core) pairs of the online CPUs under + /// (/sys/devices/system/cpu), or when the topology is not exposed. + /// + /// sysfs CPU folder. + /// Core count, or . + internal static int? LinuxCores(string cpuRoot) + { + HashSet<(string Package, string Core)> cores = []; + foreach (string cpu in Directory.EnumerateDirectories(cpuRoot, "cpu*")) + { + string name = Path.GetFileName(cpu); + if (!int.TryParse(name.AsSpan(3), NumberStyles.None, CultureInfo.InvariantCulture, out _)) + { + continue; // cpufreq, cpuidle + } + + string online = Path.Combine(cpu, "online"); + if (File.Exists(online) && File.ReadAllText(online).Trim() == "0") + { + continue; + } + + string core = Path.Combine(cpu, "topology", "core_id"); + string package = Path.Combine(cpu, "topology", "physical_package_id"); + if (!File.Exists(core)) + { + return null; + } + + cores.Add((File.Exists(package) ? File.ReadAllText(package).Trim() : "0", File.ReadAllText(core).Trim())); + } + + return cores.Count > 0 ? cores.Count : null; + } + + // One SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX record per core for RelationProcessorCore. + private static int? WindowsCores() + { + const int relationProcessorCore = 0; + uint length = 0; + GetLogicalProcessorInformationEx(relationProcessorCore, null, ref length); + if (length == 0) + { + return null; + } + + byte[] buffer = new byte[length]; + if (!GetLogicalProcessorInformationEx(relationProcessorCore, buffer, ref length)) + { + return null; + } + + int count = 0; + for (int offset = 0; offset + 8 <= length;) + { + int relationship = BitConverter.ToInt32(buffer, offset); + int size = BitConverter.ToInt32(buffer, offset + 4); + if (size <= 0) + { + break; + } + + if (relationship == relationProcessorCore) + { + count++; + } + + offset += size; + } + + return count; + } + + private static int? MacCores() + { + int value = 0; + nint size = sizeof(int); + return sysctlbyname("hw.physicalcpu", ref value, ref size, 0, 0) == 0 ? value : null; + } + + [LibraryImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static partial bool GetLogicalProcessorInformationEx(int relationshipType, byte[]? buffer, ref uint returnedLength); + + [LibraryImport("libc", StringMarshalling = StringMarshalling.Utf8)] + private static partial int sysctlbyname(string name, ref int oldValue, ref nint oldLength, nint newValue, nint newLength); +} diff --git a/src/MkPFS.Core/Util/PythonCodePointComparer.cs b/src/MkPFS.Core/Util/PythonCodePointComparer.cs new file mode 100644 index 0000000..5054996 --- /dev/null +++ b/src/MkPFS.Core/Util/PythonCodePointComparer.cs @@ -0,0 +1,37 @@ +using System.Text; + +namespace MkPFS.Core.Util; + +/// Orders strings by Unicode code point, like Python string comparison. +public sealed class PythonCodePointComparer : IComparer +{ + /// Shared instance. + public static readonly PythonCodePointComparer Instance = new(); + + /// + public int Compare(string? x, string? y) + { + if (x is null || y is null) + { + return x is null ? (y is null ? 0 : -1) : 1; + } + + StringRuneEnumerator a = x.EnumerateRunes(); + StringRuneEnumerator b = y.EnumerateRunes(); + while (true) + { + bool hasA = a.MoveNext(); + bool hasB = b.MoveNext(); + if (!hasA || !hasB) + { + return hasA ? 1 : hasB ? -1 : 0; + } + + int diff = a.Current.Value - b.Current.Value; + if (diff != 0) + { + return diff; + } + } + } +} diff --git a/src/MkPFS.Core/Util/PythonSortedJson.cs b/src/MkPFS.Core/Util/PythonSortedJson.cs new file mode 100644 index 0000000..d6d5b7d --- /dev/null +++ b/src/MkPFS.Core/Util/PythonSortedJson.cs @@ -0,0 +1,169 @@ +using System.Globalization; +using System.Text; + +namespace MkPFS.Core.Util; + +/// +/// Python json.dumps(value, sort_keys=True, ...) for byte-exact output: keys sorted by code point, floats as +/// Python repr. Values: , , integers, , +/// , with string keys, and other +/// as arrays. A keeps its insertion +/// order, like a dict dumped without sort_keys. +/// +public static class PythonSortedJson +{ + /// json.dumps(value, sort_keys=True, separators=(",", ":")) (ASCII escapes). + /// Value tree. + /// JSON text. + public static string Compact(object? value) + { + StringBuilder builder = new(); + Write(builder, value, indent: null, level: 0, ensureAscii: true); + return builder.ToString(); + } + + /// json.dumps(value, indent=2, sort_keys=True, ensure_ascii=...). + /// Value tree. + /// Escape non-ASCII characters as \uXXXX. + /// JSON text without a trailing newline. + public static string Indented(object? value, bool ensureAscii = false) + { + StringBuilder builder = new(); + Write(builder, value, indent: 2, level: 0, ensureAscii); + return builder.ToString(); + } + + private static void Write(StringBuilder builder, object? value, int? indent, int level, bool ensureAscii) + { + switch (value) + { + case null: + builder.Append("null"); + break; + case bool b: + builder.Append(b ? "true" : "false"); + break; + case int or long or uint or ulong or byte or ushort: + builder.Append(((IFormattable)value).ToString(null, CultureInfo.InvariantCulture)); + break; + case double d: + builder.Append(double.IsNaN(d) ? "NaN" : double.IsInfinity(d) ? (d > 0 ? "Infinity" : "-Infinity") : PythonText.FloatRepr(d)); + break; + case string s: + WriteString(builder, s, ensureAscii); + break; + case PythonJsonObject ordered: + WriteContainer(builder, '{', '}', ordered.Count, indent, level, i => + { + WriteString(builder, ordered[i].Key, ensureAscii); + builder.Append(indent is null ? ":" : ": "); + Write(builder, ordered[i].Value, indent, level + 1, ensureAscii); + }); + break; + case IReadOnlyDictionary map: + List keys = [.. map.Keys.OrderBy(k => k, PythonCodePointComparer.Instance)]; + WriteContainer(builder, '{', '}', keys.Count, indent, level, i => + { + WriteString(builder, keys[i], ensureAscii); + builder.Append(indent is null ? ":" : ": "); + Write(builder, map[keys[i]], indent, level + 1, ensureAscii); + }); + break; + case System.Collections.IEnumerable items: + List list = [.. items.Cast()]; + WriteContainer(builder, '[', ']', list.Count, indent, level, i => Write(builder, list[i], indent, level + 1, ensureAscii)); + break; + default: + throw new ArgumentException($"unsupported JSON value type {value.GetType().Name}"); + } + } + + // Python: empty containers stay "[]"/"{}"; with indent each item goes on its own line. + private static void WriteContainer(StringBuilder builder, char open, char close, int count, int? indent, int level, Action item) + { + builder.Append(open); + if (count == 0) + { + builder.Append(close); + return; + } + + for (int i = 0; i < count; i++) + { + if (i > 0) + { + builder.Append(','); + } + + if (indent is { } width) + { + builder.Append('\n').Append(' ', width * (level + 1)); + } + + item(i); + } + + if (indent is { } closing) + { + builder.Append('\n').Append(' ', closing * level); + } + + builder.Append(close); + } + + private static void WriteString(StringBuilder builder, string value, bool ensureAscii) + { + builder.Append('"'); + foreach (char c in value) + { + switch (c) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append(@"\\"); + break; + case '\n': + builder.Append(@"\n"); + break; + case '\r': + builder.Append(@"\r"); + break; + case '\t': + builder.Append(@"\t"); + break; + case '\b': + builder.Append(@"\b"); + break; + case '\f': + builder.Append(@"\f"); + break; + default: + // ensure_ascii escapes everything outside ' '..'~'; otherwise only control characters. + if (c < 0x20 || (ensureAscii && c > 0x7E)) + { + builder.Append(@"\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + builder.Append(c); + } + + break; + } + } + + builder.Append('"'); + } +} + + +/// A JSON object written in insertion order (a Python dict dumped without sort_keys). +public sealed class PythonJsonObject : List> +{ + /// Append a member. + /// Key. + /// Value. + public void Add(string key, object? value) => Add(new KeyValuePair(key, value)); +} \ No newline at end of file diff --git a/src/MkPFS.Core/Util/PythonText.cs b/src/MkPFS.Core/Util/PythonText.cs index c2ff081..f53fd42 100644 --- a/src/MkPFS.Core/Util/PythonText.cs +++ b/src/MkPFS.Core/Util/PythonText.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Numerics; using System.Text; namespace MkPFS.Core.Util; @@ -81,4 +82,204 @@ or UnicodeCategory.DecimalDigitNumber or UnicodeCategory.LetterNumber or UnicodeCategory.OtherNumber; } + + /// + /// Python repr(str): single quotes unless the text has ' and no "; escapes + /// backslash, the quote, \t \n \r, and non-printable code points as \xNN, \uNNNN + /// or \UNNNNNNNN. + /// + /// Text. + /// The repr, including quotes. + public static string Repr(string value) + { + char quote = value.Contains('\'', StringComparison.Ordinal) && !value.Contains('"', StringComparison.Ordinal) ? '"' : '\''; + StringBuilder builder = new(value.Length + 2); + builder.Append(quote); + foreach (Rune rune in value.EnumerateRunes()) + { + int code = rune.Value; + if (code == '\\' || code == quote) + { + builder.Append('\\').Append((char)code); + } + else if (code == '\t') + { + builder.Append("\\t"); + } + else if (code == '\n') + { + builder.Append("\\n"); + } + else if (code == '\r') + { + builder.Append("\\r"); + } + else if (IsPrintable(rune)) + { + builder.Append(rune.ToString()); + } + else + { + builder.Append(code switch + { + <= 0xFF => $"\\x{code:x2}", + <= 0xFFFF => $"\\u{code:x4}", + _ => $"\\U{code:x8}", + }); + } + } + + return builder.Append(quote).ToString(); + } + + /// Python str.splitlines(): splits on LF, CR, CRLF, VT, FF, U+001C..U+001E, U+0085, U+2028 and U+2029. + /// Text. + /// Lines without terminators; no trailing empty line. + public static List SplitLines(string value) + { + List lines = []; + int start = 0; + int i = 0; + while (i < value.Length) + { + char c = value[i]; + if (c is '\n' or '\r' or '\v' or '\f' or '\u001C' or '\u001D' or '\u001E' or '\u0085' or '\u2028' or '\u2029') + { + lines.Add(value[start..i]); + i += c == '\r' && i + 1 < value.Length && value[i + 1] == '\n' ? 2 : 1; + start = i; + } + else + { + i++; + } + } + + if (start < value.Length) + { + lines.Add(value[start..]); + } + + return lines; + } + + /// + /// Python repr(float) (also json.dumps): shortest round-trip digits, fixed notation when the + /// decimal exponent is in (-4, 16], otherwise d.ddde+/-XX; inf, -inf, nan. + /// + /// Value. + /// Text. + public static string FloatRepr(double value) + { + if (double.IsNaN(value)) + { + return "nan"; + } + + if (double.IsInfinity(value)) + { + return value > 0 ? "inf" : "-inf"; + } + + // "R" gives the shortest round-trip digits, as mantissa[E+/-exp]. + string text = value.ToString("R", CultureInfo.InvariantCulture); + bool negative = text.StartsWith('-'); + if (negative) + { + text = text[1..]; + } + + int exponent = 0; + int e = text.IndexOf('E', StringComparison.Ordinal); + if (e >= 0) + { + exponent = int.Parse(text.AsSpan(e + 1), NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + text = text[..e]; + } + + int dot = text.IndexOf('.', StringComparison.Ordinal); + string digits = dot < 0 ? text : text[..dot] + text[(dot + 1)..]; + int pointPosition = (dot < 0 ? text.Length : dot) + exponent; + int leadingZeros = digits.Length - digits.TrimStart('0').Length; + digits = digits.Trim('0'); + if (digits.Length == 0) + { + return negative ? "-0.0" : "0.0"; + } + + // Value is 0.DIGITS x 10^decpt. + int decpt = pointPosition - leadingZeros; + string body; + if (decpt > -4 && decpt <= 16) + { + body = decpt <= 0 + ? "0." + new string('0', -decpt) + digits + : decpt >= digits.Length + ? digits + new string('0', decpt - digits.Length) + ".0" + : digits[..decpt] + "." + digits[decpt..]; + } + else + { + int exp = decpt - 1; + string mantissa = digits.Length == 1 ? digits : digits[0] + "." + digits[1..]; + body = mantissa + "e" + (exp < 0 ? "-" : "+") + Math.Abs(exp).ToString("00", CultureInfo.InvariantCulture); + } + + return negative ? "-" + body : body; + } + + /// + /// Python format(value, f".{decimals}f"): the exact binary value rounded half to even, unlike .NET + /// fixed-point formatting, which rounds ties away from zero. + /// + /// Finite value. + /// Digits after the point. + /// Text. + public static string FormatFixed(double value, int decimals) + { + if (!double.IsFinite(value)) + { + return FloatRepr(value); + } + + long bits = BitConverter.DoubleToInt64Bits(value); + bool negative = bits < 0; + int exponentBits = (int)((bits >> 52) & 0x7FF); + long fraction = bits & 0xFFFFFFFFFFFFFL; + BigInteger mantissa = exponentBits == 0 ? fraction : fraction | (1L << 52); + int exponent = (exponentBits == 0 ? 1 : exponentBits) - 1075; + + // value * 10^decimals = mantissa * 2^exponent * 10^decimals, rounded half to even. + BigInteger numerator = mantissa * BigInteger.Pow(10, decimals); + BigInteger scaled; + if (exponent >= 0) + { + scaled = numerator << exponent; + } + else + { + BigInteger denominator = BigInteger.One << -exponent; + scaled = BigInteger.DivRem(numerator, denominator, out BigInteger remainder); + int compare = (remainder * 2).CompareTo(denominator); + if (compare > 0 || (compare == 0 && !scaled.IsEven)) + { + scaled += 1; + } + } + + string digits = scaled.ToString(CultureInfo.InvariantCulture).PadLeft(decimals + 1, '0'); + string text = decimals == 0 ? digits : digits[..^decimals] + "." + digits[^decimals..]; + return negative ? "-" + text : text; + } + + // Python str.isprintable(): everything except categories Cc, Cf, Cs, Co, Cn, Zl, Zp and Zs other than space. + private static bool IsPrintable(Rune rune) => + rune.Value == ' ' || Rune.GetUnicodeCategory(rune) is not (UnicodeCategory.Control + or UnicodeCategory.Format + or UnicodeCategory.Surrogate + or UnicodeCategory.PrivateUse + or UnicodeCategory.OtherNotAssigned + or UnicodeCategory.LineSeparator + or UnicodeCategory.ParagraphSeparator + or UnicodeCategory.SpaceSeparator); } diff --git a/src/MkPFS.Gui/Jobs/JobRunner.cs b/src/MkPFS.Gui/Jobs/JobRunner.cs index 620374b..a28322d 100644 --- a/src/MkPFS.Gui/Jobs/JobRunner.cs +++ b/src/MkPFS.Gui/Jobs/JobRunner.cs @@ -101,6 +101,8 @@ public sealed partial class JobRunner : ObservableObject private string _label = string.Empty; private long _done; private long _total; + private IReadOnlyList? _plan; + private double _overall; /// Create a runner that applies updates on the UI thread. public JobRunner() @@ -122,7 +124,7 @@ public JobRunner(Action post) [ObservableProperty] public partial bool IsRunning { get; private set; } - /// Progress of the current phase, 0 to 1. + /// Progress of the whole run, 0 to 1 (of the current phase for a run with one phase). [ObservableProperty] public partial double Progress { get; private set; } @@ -140,11 +142,13 @@ public JobRunner(Action post) /// Run mkpfs in-process with , echoing the command line first. /// CLI arguments. /// Outcome. - public Task RunCliAsync(IReadOnlyList args) => RunAsync(job => - { - Echo(args); - return MkPFSCli.Run([.. args], job.CreateCliContext()); - }); + public Task RunCliAsync(IReadOnlyList args) => RunAsync( + job => + { + Echo(args); + return MkPFSCli.Run([.. args], job.CreateCliContext()); + }, + ProgressPlan.For(args)); /// Log the equivalent mkpfs command line (callable from the job thread). /// CLI arguments. @@ -152,9 +156,11 @@ public Task RunCliAsync(IReadOnlyList args) => RunAsync(job /// Run on a background thread; its return value is the exit code. /// Job body. + /// The phases the job reports, so the bar covers the whole run (); + /// shows the current phase. /// Outcome, once the final lines are in . /// A job is already running. - public async Task RunAsync(Func work) + public async Task RunAsync(Func work, IReadOnlyList? plan = null) { if (IsRunning) { @@ -165,8 +171,14 @@ public async Task RunAsync(Func work) _cancel = cancel; TaskCompletionSource idle = new(TaskCreationOptions.RunContinuationsAsynchronously); _idle = idle; - _phase = string.Empty; - _label = string.Empty; + lock (_gate) + { + _phase = string.Empty; + _label = string.Empty; + _plan = plan; + _overall = 0; + } + IsRunning = true; IsIndeterminate = true; Progress = 0; @@ -307,7 +319,7 @@ private void Flush() progressDirty = _progressDirty; _progressDirty = false; _flushQueued = false; - ratio = _total > 0 ? Math.Clamp((double)_done / _total, 0, 1) : 0; + ratio = _plan is null ? (_total > 0 ? Math.Clamp((double)_done / _total, 0, 1) : 0) : _overall; label = _label; } @@ -342,6 +354,13 @@ private void OnStep(string phase, long done, long total) _label = phase; _done = done; _total = total; + if (_plan is { } plan && ProgressPlan.Index(plan, phase) is int index and >= 0) + { + // One bar for the whole run: it never moves back when the next phase starts at 0%. + _overall = Math.Max(_overall, ProgressPlan.Overall(plan, phase, total > 0 ? (double)done / total : 0) ?? _overall); + _label = $"{phase} ({index + 1}/{plan.Count})"; + } + _progressDirty = true; QueueFlush(); } diff --git a/src/MkPFS.Gui/Jobs/ProgressPlan.cs b/src/MkPFS.Gui/Jobs/ProgressPlan.cs new file mode 100644 index 0000000..63b4dc0 --- /dev/null +++ b/src/MkPFS.Gui/Jobs/ProgressPlan.cs @@ -0,0 +1,69 @@ +namespace MkPFS.Gui.Jobs; + +/// One phase of a run and its share of the progress bar. +/// Phase name as the command reports it (compress, verify, ...). +/// Relative share of the whole run. +public readonly record struct ProgressPhase(string Name, double Weight); + +/// +/// The phases a command reports, in order, so one bar can show the progress of the whole run instead of starting +/// over at every phase. Weights follow how much data each phase moves. +/// +public static class ProgressPlan +{ + /// The phases mkpfs reports for . + /// CLI arguments. + /// Phases in order, or when the command reports one phase or none. + public static IReadOnlyList? For(IReadOnlyList args) + { + bool Has(string flag) => args.Contains(flag); + string command = args.Count > 0 ? args[0] : string.Empty; + string sub = args.Count > 1 ? args[1] : string.Empty; + List phases = (command, sub) switch + { + // Compressing reads and deflates the image; verifying decodes it, comparing reads both again. + ("pack", "file") when Has("--verify") => [new("compress", 3), new("verify", 1), new("compare", 1)], + ("verify", _) when Has("--source-file") || Has("--source-dir") => [new("verify", 1), new("compare", 1)], + ("repair", _) when !Has("--scan") => [new("scan", 1), new("repair", 1), new("verify", 1)], + // Packing compresses every packed byte with LZ4 HC; the image only copies the output. + ("ampr", "game") when Has("--exfat") => [new("pack", 4), new("exfat", 1)], + _ => [], + }; + return phases.Count > 1 ? phases : null; + } + + /// Overall progress, 0 to 1, when is done. + /// Phases of the run. + /// Current phase. + /// Progress of the current phase, 0 to 1. + /// Overall progress, or for a phase outside the plan. + public static double? Overall(IReadOnlyList plan, string phase, double ratio) + { + int index = Index(plan, phase); + if (index < 0) + { + return null; + } + + double total = plan.Sum(p => p.Weight); + double before = plan.Take(index).Sum(p => p.Weight); + return total > 0 ? Math.Clamp((before + (plan[index].Weight * Math.Clamp(ratio, 0, 1))) / total, 0, 1) : 0; + } + + /// Position of in , or -1. + /// Phases. + /// Phase name. + /// Index. + public static int Index(IReadOnlyList plan, string phase) + { + for (int i = 0; i < plan.Count; i++) + { + if (plan[i].Name == phase) + { + return i; + } + } + + return -1; + } +} diff --git a/src/MkPFS.Gui/Localization/Strings.resx b/src/MkPFS.Gui/Localization/Strings.resx index 581316e..0f620db 100644 --- a/src/MkPFS.Gui/Localization/Strings.resx +++ b/src/MkPFS.Gui/Localization/Strings.resx @@ -16,8 +16,6 @@ Keep Output Extension Keep Inner File Name Alternate Key Derivation (newCrypt) - Raw PFS Image (.ffpfs) - Require Game Files Skip Post-Pack Check Use Spool Builder Advanced @@ -32,28 +30,63 @@ Folder with libSceAmpr.sprx… APR Emu Treat as APR title + Build playable game (packs + loose files) + Choose the TOML profile for this game, or a folder of APR traces to build the rules from. Profiles are TOML files shared for each game. + Your TOML profile decides what is packed. + Trace folder (optional) + Folder with ampr_commands.bin and ampr_emu.index from the debug AMPR Emu, one run per subfolder + ✓ Rules from traces: {0} recorded run(s) found. Only the files the game read through AMPR are packed; every other file stays loose. + No traces in this folder: each run needs ampr_commands.bin and its ampr_emu.index side by side. + ✗ Choose a TOML profile or a trace folder: without rules nothing is packed. + Use either a TOML file or a trace folder, not both. + Also pack files no session read, when a traced file has the same type. They were not seen being read, so test the game with them. + Free space inside the image (optional, e.g. 2GiB): needed for a test run with the debug AMPR Emu, which writes its log into the game folder + ⚠ Experimental: whether a packed game runs depends on how it reads its files, and only a test on the PS5 shows that. Keep the original game until the packed one has been played. + Builds a new folder you can copy to the PS5 and play: your libraries in fakelib/, ampr_emu.index, the LZ4 packs, and every file that stays loose. The game folder is only read. Needs AMPR Emu 0.4.2.1 or newer and a game that uses AMPR. + Library Folder (fakelib) + Folder with libSceAmpr.sprx and other fakelib libraries (optional if the game has them)… + Verify the packs and loose files after building + Also build an exFAT image (64 KiB clusters, mount read-only) + exFAT Image (optional) + Default: <titleId>.exfat next to the output folder + Inspect manifest + List files + Pack files only (manifest and .pak volumes) + Remove packed sources + Update runtime settings + Unpack packed files + Verify packs + Action + Leave missing source files loose + TOML Configuration + The game's TOML profile… + Delete the packed source files (cannot be undone; unchecked shows the plan only) + ✗ Pack manifest is required. + ✗ Game folder and output folder are both required. + ✗ Pack manifest and game folder are both required. + ✗ Pack manifest and TOML configuration are both required. + ✗ Pack manifest and output folder are both required. + ✗ Workers must be a whole number from 1 to 256. + AMPR Index (optional) + Default: ampr_emu.index in the game folder + JSON output + Pack Manifest + Select ampr_assets.index… + Writes only ampr_assets.index, the .pak volumes and the .crc file into the output folder, like ampr_pack.py. For a folder you can copy to the PS5 and play, use Build playable game. + Packs serve files the game reads through AMPR and its ordinary file reads (AMPR Emu 0.4.2.1). Memory-mapped files must stay loose, so keep the source files until the packed game is tested. + Output Folder + Folder for the manifest and .pak volumes… + Overwrite existing files + Game Folder (/app0) + Folder with eboot.bin and sce_sys… + Never leave selected files loose (store incompressible blocks uncompressed) + Build and check AMPR Emu LZ4 asset packs for a game folder + AMPR Packs + Workers (empty = from the configuration) PlayStation File System Browse File Folder - PFSC Compression - Dry Run - ✗ Output folder is required. - ✗ Source folder is required. - Output Folder - Select output folder… - Overwrite Existing - {0} item(s): {1} folder(s), {2} file(s) - Select a source folder to preview the batch queue. - Unable to scan batch folder: {0} - Select an existing folder to preview the batch queue. - No packable items found. - Scanning batch items... - Batch Queue - Source Folder - Select source folder… - Convert multiple games at once into PFS images - Batch Convert Version Cancel ✗ Cancelled. @@ -65,7 +98,8 @@ Partly written output is removed; an in-place repair finishes its rewrite first Block Size (PS5 games use 64 KiB) Auto (64 KiB) Auto-fit (least padding) - CPU Cores (0 = auto: {0}) + CPU Cores + Auto ({0} cores) ✗ Store Files Smaller Than must be a whole number of bytes. Compression Level (0–9) Max Compressed Size (% of original) @@ -105,11 +139,10 @@ Partly written output is removed; an in-place repair finishes its rewrite first Inspect Language Import Preview - Batch + AMPR Packs Inspect Pack exFAT - Pack File - Pack Folder + Pack FFPFSC Repair Tree Unpack @@ -119,19 +152,8 @@ Partly written output is removed; an in-place repair finishes its rewrite first Options Output Log Paths - PFSC Compression Dry Run - ✗ Output image path is required. - ✗ Source folder is required. - Output Image - Select output path… Signed Image - Source Folder - Select source folder… - Build a PFS image from a source directory - Temp Folder (optional) - Custom temp folder… - Pack Folder Verify After Pack Version PFSC Compression @@ -153,7 +175,7 @@ Partly written output is removed; an in-place repair finishes its rewrite first Wrap a single file into a PFSC compressed image Temp Folder (optional) Custom temp folder… - Pack File + Pack FFPFSC Version Bad Blocks List (optional — Game Compressor TSV) bad_blocks.tsv… @@ -232,4 +254,29 @@ Partly written output is removed; an in-place repair finishes its rewrite first Validate image structure, inodes, and payload checksums Verify v{0} + Quick settings for the compression level and CPU cores. Changing either value selects the matching preset, or Custom. + zlib level for each block: 1 is the fastest, 9 gives the smallest image, 0 stores the blocks uncompressed. The default is 7. + How many CPU cores compress blocks at the same time. Auto uses every core of this computer; choose fewer to keep the computer responsive or to use less memory. + Size of the blocks the image is split into and compressed one by one. PS5 games use 64 KiB; change it only if you know the target needs another size. + A compressed block is kept only if it saves at least this percentage; otherwise it is stored uncompressed. 0 keeps every block that gets smaller. + A file stays uncompressed when its compressed size would be above this percentage of the original. 100 compresses every file that gets smaller. + Files smaller than this many bytes are stored uncompressed without trying. Empty uses the block size. + Store executables and modules (eboot.bin, .prx, .sprx) uncompressed. + Build a signed PFS image, using a zero EKPFS key and seed. + Check the source and show what would be written, without writing the image. + After packing, decode the whole image and compare it with the source. Slower, but proves the image is correct. + Skip the quick structure check that runs after packing. + Use the output file name exactly as typed. Otherwise the extension is changed to match the format (.ffpfsc or .ffpfs). + Keep the source file's name inside the image. Otherwise it is renamed to a safe, normalized name. + Build through a temporary staging file instead of writing straight into the image. Slower; use it only if the default way fails. + PFS format profile: PS5 for PS5 games, PS4 for PS4 images. + Width of the inode entries: 32-bit, the default, or 64-bit. + Make file names inside the image case-sensitive. By default they are not. + Write more detail to the log. + Encrypt the image blocks with AES-XTS. + Derive the encryption keys with the alternate newCrypt method. + 64 hexadecimal characters: the EKPFS key for encrypting or signing. Empty uses all zeros. + exFAT cluster size. Auto (64 KiB) works best with ShadowMountPlus; smaller clusters waste less space on small files but read slower. + How to read the image: Auto detects PFS or exFAT; choose one to force it. + Warn when sce_sys/param.json, eboot.bin or pfs-version.dat is missing from the game. diff --git a/src/MkPFS.Gui/Localization/Strings_de.resx b/src/MkPFS.Gui/Localization/Strings_de.resx index 780ca0c..687b010 100644 --- a/src/MkPFS.Gui/Localization/Strings_de.resx +++ b/src/MkPFS.Gui/Localization/Strings_de.resx @@ -16,8 +16,6 @@ Ausgabe-Endung beibehalten Inneren Dateinamen beibehalten Alternative Schlüsselableitung (newCrypt) - Rohes PFS-Image (.ffpfs) - Spieldateien erforderlich Prüfung nach dem Packen überspringen Spool-Builder verwenden Erweitert @@ -32,28 +30,63 @@ Ordner mit libSceAmpr.sprx… APR Emu Als APR-Titel behandeln + Spielfertigen Ordner erstellen (Packs + lose Dateien) + Wähle das TOML-Profil für dieses Spiel oder einen Ordner mit APR-Traces, aus dem die Regeln erstellt werden. Profile sind TOML-Dateien, die pro Spiel geteilt werden. + Dein TOML-Profil bestimmt, was gepackt wird. + Trace-Ordner (optional) + Ordner mit ampr_commands.bin und ampr_emu.index vom Debug-AMPR-Emu, ein Lauf pro Unterordner + ✓ Regeln aus Traces: {0} aufgezeichnete(r) Lauf/Läufe gefunden. Nur die Dateien, die das Spiel über AMPR gelesen hat, werden gepackt; alle anderen bleiben einzeln. + Keine Traces in diesem Ordner: jeder Lauf braucht ampr_commands.bin und seine ampr_emu.index nebeneinander. + ✗ Wähle ein TOML-Profil oder einen Trace-Ordner: ohne Regeln wird nichts gepackt. + Verwende entweder eine TOML-Datei oder einen Trace-Ordner, nicht beides. + Auch Dateien packen, die keine Sitzung gelesen hat, wenn eine aufgezeichnete Datei denselben Typ hat. Ihr Lesen wurde nicht beobachtet, also das Spiel damit testen. + Freier Platz im Abbild (optional, z. B. 2GiB): nötig für einen Test mit dem Debug-AMPR-Emu, der sein Protokoll in den Spielordner schreibt + ⚠ Experimentell: Ob ein gepacktes Spiel läuft, hängt davon ab, wie es seine Dateien liest, und das zeigt nur ein Test auf der PS5. Behalte das Originalspiel, bis das gepackte gespielt wurde. + Erstellt einen neuen Ordner, den du auf die PS5 kopierst und spielst: deine Bibliotheken in fakelib/, ampr_emu.index, die LZ4-Packs und alle Dateien, die lose bleiben. Der Spielordner wird nur gelesen. Benötigt AMPR Emu 0.4.2.1 oder neuer und ein Spiel, das AMPR nutzt. + Bibliotheksordner (fakelib) + Ordner mit libSceAmpr.sprx und weiteren fakelib-Bibliotheken (optional, wenn das Spiel sie hat)… + Packs und lose Dateien nach dem Erstellen prüfen + Zusätzlich ein exFAT-Abbild erstellen (64-KiB-Cluster, schreibgeschützt einbinden) + exFAT-Abbild (optional) + Standard: <titleId>.exfat neben dem Ausgabeordner + Manifest untersuchen + Dateien auflisten + Nur packen (Manifest und .pak-Volumes) + Gepackte Quelldateien entfernen + Laufzeiteinstellungen aktualisieren + Gepackte Dateien entpacken + Pakete prüfen + Aktion + Fehlende Quelldateien lose lassen + TOML-Konfiguration + TOML-Profil des Spiels… + Gepackte Quelldateien löschen (nicht rückgängig zu machen; sonst nur den Plan anzeigen) + ✗ Paket-Manifest ist erforderlich. + ✗ Spielordner und Ausgabeordner sind beide erforderlich. + ✗ Paket-Manifest und Spielordner sind beide erforderlich. + ✗ Paket-Manifest und TOML-Konfiguration sind beide erforderlich. + ✗ Paket-Manifest und Ausgabeordner sind beide erforderlich. + ✗ Worker muss eine ganze Zahl von 1 bis 256 sein. + AMPR-Index (optional) + Standard: ampr_emu.index im Spielordner + JSON-Ausgabe + Paket-Manifest + ampr_assets.index auswählen… + Schreibt nur ampr_assets.index, die .pak-Volumes und die .crc-Datei in den Ausgabeordner, wie ampr_pack.py. Für einen Ordner, den du auf die PS5 kopierst und spielst, nutze Spielfertigen Ordner erstellen. + Pakete liefern Dateien, die das Spiel über AMPR und mit normalen Dateizugriffen liest (AMPR Emu 0.4.2.1). Speicherabgebildete Dateien müssen lose bleiben; behalte die Quelldateien, bis das gepackte Spiel getestet ist. + Ausgabeordner + Ordner für Manifest und .pak-Volumes… + Vorhandene Dateien überschreiben + Spielordner (/app0) + Ordner mit eboot.bin und sce_sys… + Ausgewählte Dateien nie lose lassen (nicht komprimierbare Blöcke unkomprimiert speichern) + AMPR-Emu-LZ4-Asset-Pakete für einen Spielordner erstellen und prüfen + AMPR-Pakete + Worker (leer = aus der Konfiguration) PlayStation File System Durchsuchen Datei Ordner - PFSC-Komprimierung - Probelauf - ✗ Ausgabeordner ist erforderlich. - ✗ Quellordner ist erforderlich. - Ausgabeordner - Ausgabeordner wählen… - Vorhandene überschreiben - {0} Element(e): {1} Ordner, {2} Datei(en) - Wähle einen Quellordner für die Vorschau der Warteschlange. - Stapelordner kann nicht gelesen werden: {0} - Wähle einen vorhandenen Ordner für die Vorschau der Warteschlange. - Keine packbaren Elemente gefunden. - Elemente werden gesucht... - Stapel-Warteschlange - Quellordner - Quellordner wählen… - Mehrere Spiele auf einmal in PFS-Images umwandeln - Stapelkonvertierung Version Abbrechen ✗ Abgebrochen. @@ -65,7 +98,8 @@ Teilweise geschriebene Ausgaben werden entfernt; eine Reparatur vor Ort beendet Blockgröße (PS5-Spiele nutzen 64 KiB) Auto (64 KiB) Auto-Anpassung (wenigste Auffüllung) - CPU-Kerne (0 = auto: {0}) + CPU-Kerne + Auto ({0} Kerne) ✗ „Dateien kleiner als speichern“ muss eine ganze Zahl von Bytes sein. Komprimierungsstufe (0–9) Max. komprimierte Größe (% des Originals) @@ -105,11 +139,10 @@ Teilweise geschriebene Ausgaben werden entfernt; eine Reparatur vor Ort beendet Untersuchen Sprache Import-Vorschau - Stapel + AMPR-Pakete Untersuchen exFAT packen - Datei packen - Ordner packen + FFPFSC packen Reparieren Baum Entpacken @@ -119,19 +152,8 @@ Teilweise geschriebene Ausgaben werden entfernt; eine Reparatur vor Ort beendet Optionen Ausgabeprotokoll Pfade - PFSC-Komprimierung Probelauf - ✗ Ausgabe-Image-Pfad ist erforderlich. - ✗ Quellordner ist erforderlich. - Ausgabe-Image - Ausgabepfad wählen… Signiertes Image - Quellordner - Quellordner wählen… - Ein PFS-Image aus einem Quellverzeichnis erstellen - Temp-Ordner (optional) - Eigener Temp-Ordner… - Ordner packen Nach dem Packen prüfen Version PFSC-Komprimierung @@ -153,7 +175,7 @@ Teilweise geschriebene Ausgaben werden entfernt; eine Reparatur vor Ort beendet Eine einzelne Datei in ein PFSC-komprimiertes Image packen Temp-Ordner (optional) Eigener Temp-Ordner… - Datei packen + FFPFSC packen Version Liste defekter Blöcke (optional — Game-Compressor-TSV) bad_blocks.tsv… @@ -232,4 +254,29 @@ Teilweise geschriebene Ausgaben werden entfernt; eine Reparatur vor Ort beendet Image-Struktur, Inodes und Prüfsummen der Inhalte validieren Prüfen v{0} + Schnelleinstellungen für Kompressionsstufe und CPU-Kerne. Wenn du einen der Werte änderst, wird die passende Vorgabe gewählt oder Benutzerdefiniert. + zlib-Stufe für jeden Block: 1 ist am schnellsten, 9 ergibt das kleinste Image, 0 speichert die Blöcke unkomprimiert. Standard ist 7. + Wie viele CPU-Kerne gleichzeitig Blöcke komprimieren. Auto nutzt alle Kerne dieses Computers; wähle weniger, damit der Computer reaktionsfähig bleibt oder weniger Speicher braucht. + Größe der Blöcke, in die das Image geteilt und einzeln komprimiert wird. PS5-Spiele nutzen 64 KiB; ändere sie nur, wenn das Ziel eine andere Größe braucht. + Ein komprimierter Block wird nur behalten, wenn er mindestens diesen Prozentsatz spart; sonst wird er unkomprimiert gespeichert. 0 behält jeden Block, der kleiner wird. + Eine Datei bleibt unkomprimiert, wenn ihre komprimierte Größe über diesem Prozentsatz des Originals läge. 100 komprimiert jede Datei, die kleiner wird. + Dateien unter dieser Bytezahl werden ohne Versuch unkomprimiert gespeichert. Leer bedeutet die Blockgröße. + Ausführbare Dateien und Module (eboot.bin, .prx, .sprx) unkomprimiert speichern. + Ein signiertes PFS-Image erstellen, mit EKPFS-Schlüssel und Seed aus Nullen. + Die Quelle prüfen und zeigen, was geschrieben würde, ohne das Image zu schreiben. + Nach dem Packen das ganze Image dekodieren und mit der Quelle vergleichen. Langsamer, beweist aber, dass das Image korrekt ist. + Die schnelle Strukturprüfung nach dem Packen überspringen. + Den Ausgabenamen genau wie eingegeben verwenden. Sonst wird die Endung an das Format angepasst (.ffpfsc oder .ffpfs). + Den Namen der Quelldatei im Image behalten. Sonst wird sie in einen sicheren, normalisierten Namen umbenannt. + Über eine temporäre Zwischendatei bauen statt direkt ins Image zu schreiben. Langsamer; nur verwenden, wenn der Standardweg scheitert. + PFS-Formatprofil: PS5 für PS5-Spiele, PS4 für PS4-Images. + Breite der Inode-Einträge: 32 Bit (Standard) oder 64 Bit. + Dateinamen im Image zwischen Groß- und Kleinschreibung unterscheiden lassen. Standardmäßig nicht. + Mehr Details ins Protokoll schreiben. + Die Image-Blöcke mit AES-XTS verschlüsseln. + Die Verschlüsselungsschlüssel mit der alternativen newCrypt-Methode ableiten. + 64 Hexadezimalzeichen: der EKPFS-Schlüssel zum Verschlüsseln oder Signieren. Leer bedeutet nur Nullen. + exFAT-Clustergröße. Auto (64 KiB) funktioniert am besten mit ShadowMountPlus; kleinere Cluster verschwenden weniger Platz bei kleinen Dateien, lesen aber langsamer. + Wie das Image gelesen wird: Auto erkennt PFS oder exFAT; wähle eines, um es zu erzwingen. + Warnen, wenn sce_sys/param.json, eboot.bin oder pfs-version.dat im Spiel fehlt. diff --git a/src/MkPFS.Gui/Localization/Strings_es.resx b/src/MkPFS.Gui/Localization/Strings_es.resx index 5661024..8beadcb 100644 --- a/src/MkPFS.Gui/Localization/Strings_es.resx +++ b/src/MkPFS.Gui/Localization/Strings_es.resx @@ -16,8 +16,6 @@ Conservar Extensión de Salida Conservar Nombre del Archivo Interno Derivación de Clave Alternativa (newCrypt) - Imagen PFS Sin Envoltura (.ffpfs) - Exigir Archivos del Juego Omitir Comprobación Final Usar Constructor con Spool Avanzado @@ -32,28 +30,63 @@ Carpeta con libSceAmpr.sprx… APR Emu Tratar como título APR + Crear juego listo (packs + archivos sueltos) + Elige el perfil TOML de este juego, o una carpeta de trazas APR para crear las reglas. Los perfiles son archivos TOML que se comparten para cada juego. + Tu perfil TOML decide qué se empaqueta. + Carpeta de trazas (opcional) + Carpeta con ampr_commands.bin y ampr_emu.index del AMPR Emu de depuración, una ejecución por subcarpeta + ✓ Reglas a partir de trazas: {0} ejecución(es) grabada(s). Solo se empaquetan los archivos que el juego leyó mediante AMPR; el resto queda suelto. + No hay trazas en esta carpeta: cada ejecución necesita ampr_commands.bin y su ampr_emu.index juntos. + ✗ Elige un perfil TOML o una carpeta de trazas: sin reglas no se empaqueta nada. + Usa un archivo TOML o una carpeta de trazas, no ambos. + Empaquetar también los archivos que ninguna sesión leyó, si un archivo trazado tiene el mismo tipo. No se vio cómo se leían, así que prueba el juego con ellos. + Espacio libre dentro de la imagen (opcional, p. ej. 2GiB): necesario para una prueba con el AMPR Emu de depuración, que escribe su registro en la carpeta del juego + ⚠ Experimental: que un juego empaquetado funcione depende de cómo lee sus archivos, y solo una prueba en la PS5 lo muestra. Conserva el juego original hasta haber jugado el empaquetado. + Crea una carpeta nueva que puedes copiar a la PS5 y jugar: tus bibliotecas en fakelib/, ampr_emu.index, los packs LZ4 y todos los archivos que quedan sueltos. La carpeta del juego solo se lee. Requiere AMPR Emu 0.4.2.1 o posterior y un juego que use AMPR. + Carpeta de Bibliotecas (fakelib) + Carpeta con libSceAmpr.sprx y otras bibliotecas fakelib (opcional si el juego ya las tiene)… + Verificar los packs y los archivos sueltos al terminar + Crear también una imagen exFAT (clústeres de 64 KiB, montar en solo lectura) + Imagen exFAT (opcional) + Por defecto: <titleId>.exfat junto a la carpeta de salida + Inspeccionar manifiesto + Listar archivos + Solo empaquetar (manifiesto y volúmenes .pak) + Eliminar archivos fuente empaquetados + Actualizar ajustes de ejecución + Desempaquetar archivos + Verificar paquetes + Acción + Dejar sueltos los archivos fuente que falten + Configuración TOML + Perfil TOML del juego… + Eliminar los archivos fuente empaquetados (no se puede deshacer; sin marcar solo muestra el plan) + ✗ Se requiere el manifiesto del paquete. + ✗ Se requieren la carpeta del juego y la carpeta de salida. + ✗ Se requieren el manifiesto del paquete y la carpeta del juego. + ✗ Se requieren el manifiesto del paquete y la configuración TOML. + ✗ Se requieren el manifiesto del paquete y la carpeta de salida. + ✗ Los hilos deben ser un número entero de 1 a 256. + Índice AMPR (opcional) + Por defecto: ampr_emu.index en la carpeta del juego + Salida JSON + Manifiesto del paquete + Selecciona ampr_assets.index… + Escribe solo ampr_assets.index, los volúmenes .pak y el archivo .crc en la carpeta de salida, como ampr_pack.py. Para una carpeta que copias a la PS5 y juegas, usa Crear juego listo. + Los paquetes sirven los archivos que el juego lee mediante AMPR y con lecturas de archivo normales (AMPR Emu 0.4.2.1). Los archivos mapeados en memoria deben quedar sueltos; conserva los archivos fuente hasta probar el juego empaquetado. + Carpeta de salida + Carpeta para el manifiesto y los volúmenes .pak… + Sobrescribir archivos existentes + Carpeta del juego (/app0) + Carpeta con eboot.bin y sce_sys… + No dejar nunca sueltos los archivos seleccionados (guardar sin comprimir los bloques incompresibles) + Crea y comprueba paquetes de recursos LZ4 de AMPR Emu para una carpeta de juego + Paquetes AMPR + Hilos (vacío = de la configuración) Sistema de Archivos PlayStation Buscar Archivo Carpeta - Compresión PFSC - Simulación - ✗ La carpeta de salida es obligatoria. - ✗ La carpeta fuente es obligatoria. - Carpeta de Salida - Seleccionar carpeta de salida… - Sobrescribir Existente - {0} elemento(s): {1} carpeta(s), {2} archivo(s) - Selecciona una carpeta fuente para previsualizar la cola del lote. - No fue posible escanear la carpeta del lote: {0} - Selecciona una carpeta existente para previsualizar la cola del lote. - No se encontraron elementos empaquetables. - Escaneando elementos del lote... - Cola del Lote - Carpeta Fuente - Seleccionar carpeta fuente… - Convertir varios juegos a la vez en imágenes PFS - Convertir en Lote Versión Cancelar ✗ Cancelado. @@ -65,7 +98,8 @@ La salida escrita a medias se elimina; una reparación in situ termina su reescr Tamaño de Bloque (los juegos de PS5 usan 64 KiB) Automático (64 KiB) Ajuste automático (menos relleno) - Núcleos de CPU (0 = auto: {0}) + Núcleos de CPU + Automático ({0} núcleos) ✗ Almacenar Archivos Menores Que debe ser un número entero de bytes. Nivel de Compresión (0–9) Tamaño Comprimido Máx. (% del original) @@ -105,11 +139,10 @@ La salida escrita a medias se elimina; una reparación in situ termina su reescr Inspeccionar Idioma Vista Previa de Importación - Lote + Paquetes AMPR Inspeccionar Empaquetar EXFAT - Empaquetar Archivo - Empaquetar Carpeta + Empaquetar FFPFSC Reparar Árbol Desempaquetar @@ -119,19 +152,8 @@ La salida escrita a medias se elimina; una reparación in situ termina su reescr Opciones Registro de Salida Rutas - Compresión PFSC Simulación (Dry Run) - ✗ La ruta de salida es obligatoria. - ✗ La carpeta fuente es obligatoria. - Imagen de Salida - Seleccionar ruta de salida… Imagen Firmada - Carpeta Fuente - Seleccionar carpeta fuente… - Construir una imagen PFS desde un directorio fuente - Carpeta Temporal (opcional) - Carpeta temp personalizada… - Empaquetar Carpeta Verificar Tras Empaquetar Versión Compresión PFSC @@ -153,7 +175,7 @@ La salida escrita a medias se elimina; una reparación in situ termina su reescr Envolver un solo archivo en una imagen PFSC comprimida Carpeta Temporal (opcional) Carpeta temp personalizada… - Empaquetar Archivo + Empaquetar FFPFSC Versión Lista de Bloques Dañados (opcional — TSV de Game Compressor) bad_blocks.tsv… @@ -232,4 +254,29 @@ La salida escrita a medias se elimina; una reparación in situ termina su reescr Validar estructura de imagen, inodes y checksums de payload Verificar v{0} + Ajustes rápidos del nivel de compresión y de los núcleos de CPU. Al cambiar cualquiera de los dos se elige el ajuste que coincide, o Personalizado. + Nivel zlib de cada bloque: 1 es el más rápido, 9 da la imagen más pequeña y 0 guarda los bloques sin comprimir. El valor por defecto es 7. + Cuántos núcleos de CPU comprimen bloques a la vez. Automático usa todos los núcleos de este equipo; elige menos para que el equipo siga respondiendo o para usar menos memoria. + Tamaño de los bloques en que se divide la imagen y que se comprimen uno a uno. Los juegos de PS5 usan 64 KiB; cámbialo solo si sabes que el destino necesita otro tamaño. + Un bloque comprimido solo se conserva si ahorra al menos este porcentaje; si no, se guarda sin comprimir. 0 conserva todo bloque que se reduzca. + Un archivo queda sin comprimir si su tamaño comprimido superaría este porcentaje del original. 100 comprime todo archivo que se reduzca. + Los archivos con menos bytes que este valor se guardan sin comprimir, sin intentarlo. Vacío usa el tamaño de bloque. + Guardar sin comprimir los ejecutables y módulos (eboot.bin, .prx, .sprx). + Crear una imagen PFS firmada, con clave EKPFS y semilla a cero. + Revisar el origen y mostrar lo que se escribiría, sin escribir la imagen. + Tras empaquetar, decodificar toda la imagen y compararla con el origen. Más lento, pero demuestra que la imagen es correcta. + Omitir la comprobación rápida de estructura que se hace tras empaquetar. + Usar el nombre de salida tal como se escribió. Si no, la extensión se ajusta al formato (.ffpfsc o .ffpfs). + Conservar dentro de la imagen el nombre del archivo de origen. Si no, se cambia por un nombre seguro y normalizado. + Crear a través de un archivo temporal intermedio en lugar de escribir directamente en la imagen. Más lento; úsalo solo si el modo normal falla. + Perfil del formato PFS: PS5 para juegos de PS5, PS4 para imágenes de PS4. + Ancho de las entradas de inodo: 32 bits (por defecto) o 64 bits. + Hacer que los nombres de archivo de la imagen distingan mayúsculas y minúsculas. Por defecto no lo hacen. + Escribir más detalles en el registro. + Cifrar los bloques de la imagen con AES-XTS. + Derivar las claves de cifrado con el método alternativo newCrypt. + 64 caracteres hexadecimales: la clave EKPFS para cifrar o firmar. Vacío usa todo ceros. + Tamaño de clúster exFAT. Automático (64 KiB) funciona mejor con ShadowMountPlus; los clústeres más pequeños desperdician menos espacio en archivos pequeños, pero leen más lento. + Cómo leer la imagen: Automático detecta PFS o exFAT; elige uno para forzarlo. + Avisar si al juego le falta sce_sys/param.json, eboot.bin o pfs-version.dat. diff --git a/src/MkPFS.Gui/Localization/Strings_fr.resx b/src/MkPFS.Gui/Localization/Strings_fr.resx index 2d98c1d..c94070e 100644 --- a/src/MkPFS.Gui/Localization/Strings_fr.resx +++ b/src/MkPFS.Gui/Localization/Strings_fr.resx @@ -16,8 +16,6 @@ Conserver l'extension de sortie Conserver le nom du fichier interne Dérivation de clé alternative (newCrypt) - Image PFS brute (.ffpfs) - Exiger les fichiers du jeu Ignorer la vérification après empaquetage Utiliser le générateur avec spool Avancé @@ -32,28 +30,63 @@ Dossier contenant libSceAmpr.sprx… APR Emu Traiter comme un titre APR + Créer le jeu prêt à lancer (packs + fichiers libres) + Choisissez le profil TOML de ce jeu, ou un dossier de traces APR pour en tirer les règles. Les profils sont des fichiers TOML partagés pour chaque jeu. + Votre profil TOML décide de ce qui est empaqueté. + Dossier de traces (facultatif) + Dossier avec ampr_commands.bin et ampr_emu.index de l’AMPR Emu de débogage, une session par sous-dossier + ✓ Règles issues des traces : {0} session(s) enregistrée(s). Seuls les fichiers que le jeu a lus via AMPR sont empaquetés ; les autres restent à part. + Aucune trace dans ce dossier : chaque session a besoin de ampr_commands.bin et de son ampr_emu.index côte à côte. + ✗ Choisissez un profil TOML ou un dossier de traces : sans règles, rien n’est empaqueté. + Utilisez un fichier TOML ou un dossier de traces, pas les deux. + Empaqueter aussi les fichiers qu’aucune session n’a lus, quand un fichier tracé a le même type. Leur lecture n’a pas été observée : testez le jeu avec eux. + Espace libre dans l’image (facultatif, ex. 2GiB) : nécessaire pour un test avec l’AMPR Emu de débogage, qui écrit son journal dans le dossier du jeu + ⚠ Expérimental : qu’un jeu empaqueté fonctionne dépend de la façon dont il lit ses fichiers, et seul un test sur la PS5 le montre. Conservez le jeu d’origine jusqu’à avoir joué à la version empaquetée. + Crée un nouveau dossier à copier sur la PS5 pour jouer : vos bibliothèques dans fakelib/, ampr_emu.index, les packs LZ4 et tous les fichiers qui restent libres. Le dossier du jeu est seulement lu. Nécessite AMPR Emu 0.4.2.1 ou plus récent et un jeu qui utilise AMPR. + Dossier des bibliothèques (fakelib) + Dossier avec libSceAmpr.sprx et d'autres bibliothèques fakelib (facultatif si le jeu les a déjà)… + Vérifier les packs et les fichiers libres après la création + Créer aussi une image exFAT (clusters de 64 Kio, montage en lecture seule) + Image exFAT (facultatif) + Par défaut : <titleId>.exfat à côté du dossier de sortie + Inspecter le manifeste + Lister les fichiers + Empaqueter seulement (manifeste et volumes .pak) + Supprimer les sources empaquetées + Mettre à jour les réglages d’exécution + Extraire les fichiers empaquetés + Vérifier les paquets + Action + Laisser libres les fichiers sources manquants + Configuration TOML + Profil TOML du jeu… + Supprimer les fichiers sources empaquetés (irréversible ; décoché, affiche seulement le plan) + ✗ Le manifeste du paquet est requis. + ✗ Le dossier du jeu et le dossier de sortie sont tous deux requis. + ✗ Le manifeste du paquet et le dossier du jeu sont tous deux requis. + ✗ Le manifeste du paquet et la configuration TOML sont tous deux requis. + ✗ Le manifeste du paquet et le dossier de sortie sont tous deux requis. + ✗ Les threads doivent être un entier de 1 à 256. + Index AMPR (facultatif) + Par défaut : ampr_emu.index dans le dossier du jeu + Sortie JSON + Manifeste du paquet + Sélectionner ampr_assets.index… + N’écrit que ampr_assets.index, les volumes .pak et le fichier .crc dans le dossier de sortie, comme ampr_pack.py. Pour un dossier à copier sur la PS5 pour jouer, utilisez Créer le jeu prêt à lancer. + Les paquets servent les fichiers que le jeu lit via AMPR et par des lectures de fichiers ordinaires (AMPR Emu 0.4.2.1). Les fichiers mappés en mémoire doivent rester libres ; conservez les fichiers sources jusqu’à ce que le jeu empaqueté soit testé. + Dossier de sortie + Dossier du manifeste et des volumes .pak… + Écraser les fichiers existants + Dossier du jeu (/app0) + Dossier contenant eboot.bin et sce_sys… + Ne jamais laisser libres les fichiers sélectionnés (stocker non compressés les blocs incompressibles) + Créer et vérifier les paquets de ressources LZ4 d’AMPR Emu pour un dossier de jeu + Paquets AMPR + Threads (vide = selon la configuration) PlayStation File System Parcourir Fichier Dossier - Compression PFSC - Simulation - ✗ Le dossier de sortie est obligatoire. - ✗ Le dossier source est obligatoire. - Dossier de sortie - Choisir le dossier de sortie… - Écraser les existants - {0} élément(s) : {1} dossier(s), {2} fichier(s) - Choisissez un dossier source pour prévisualiser la file d'attente. - Impossible d'analyser le dossier du lot : {0} - Choisissez un dossier existant pour prévisualiser la file d'attente. - Aucun élément à empaqueter trouvé. - Analyse des éléments... - File d'attente du lot - Dossier source - Choisir le dossier source… - Convertir plusieurs jeux à la fois en images PFS - Conversion par lot Version Annuler ✗ Annulé. @@ -65,7 +98,8 @@ La sortie partiellement écrite est supprimée ; une réparation sur place termi Taille de bloc (les jeux PS5 utilisent 64 Kio) Auto (64 Kio) Ajustement auto (moins de remplissage) - Cœurs CPU (0 = auto : {0}) + Cœurs CPU + Auto ({0} cœurs) ✗ « Stocker les fichiers plus petits que » doit être un nombre entier d’octets. Niveau de compression (0–9) Taille compressée max. (% de l'original) @@ -105,11 +139,10 @@ La sortie partiellement écrite est supprimée ; une réparation sur place termi Inspecter Langue Aperçu de l'import - Lot + Paquets AMPR Inspecter Empaqueter exFAT - Empaqueter fichier - Empaqueter dossier + Empaqueter FFPFSC Réparer Arborescence Extraire @@ -119,19 +152,8 @@ La sortie partiellement écrite est supprimée ; une réparation sur place termi Options Journal de sortie Chemins - Compression PFSC Simulation - ✗ Le chemin de l'image de sortie est obligatoire. - ✗ Le dossier source est obligatoire. - Image de sortie - Choisir le chemin de sortie… Image signée - Dossier source - Choisir le dossier source… - Créer une image PFS à partir d’un répertoire source - Dossier temporaire (facultatif) - Dossier temporaire personnalisé… - Empaqueter dossier Vérifier après l'empaquetage Version Compression PFSC @@ -153,7 +175,7 @@ La sortie partiellement écrite est supprimée ; une réparation sur place termi Empaqueter un seul fichier dans une image compressée PFSC Dossier temporaire (facultatif) Dossier temporaire personnalisé… - Empaqueter fichier + Empaqueter FFPFSC Version Liste des blocs défectueux (facultatif — TSV Game Compressor) bad_blocks.tsv… @@ -232,4 +254,29 @@ La sortie partiellement écrite est supprimée ; une réparation sur place termi Valider la structure de l'image, les inodes et les sommes de contrôle Vérifier v{0} + Réglages rapides du niveau de compression et des cœurs CPU. Modifier l’une des valeurs sélectionne le préréglage correspondant, ou Personnalisé. + Niveau zlib de chaque bloc : 1 est le plus rapide, 9 donne l’image la plus petite, 0 stocke les blocs sans compression. La valeur par défaut est 7. + Nombre de cœurs CPU qui compressent des blocs en même temps. Auto utilise tous les cœurs de cet ordinateur ; choisissez-en moins pour garder l’ordinateur réactif ou utiliser moins de mémoire. + Taille des blocs en lesquels l’image est découpée et compressée un à un. Les jeux PS5 utilisent 64 Kio ; ne la changez que si la cible exige une autre taille. + Un bloc compressé n’est gardé que s’il économise au moins ce pourcentage ; sinon il est stocké sans compression. 0 garde tout bloc qui devient plus petit. + Un fichier reste non compressé si sa taille compressée dépasserait ce pourcentage de l’original. 100 compresse tout fichier qui devient plus petit. + Les fichiers de moins de ce nombre d’octets sont stockés sans compression, sans essai. Vide utilise la taille de bloc. + Stocker sans compression les exécutables et modules (eboot.bin, .prx, .sprx). + Créer une image PFS signée, avec une clé EKPFS et une graine à zéro. + Vérifier la source et montrer ce qui serait écrit, sans écrire l’image. + Après l’empaquetage, décoder toute l’image et la comparer à la source. Plus lent, mais prouve que l’image est correcte. + Ignorer la vérification rapide de structure effectuée après l’empaquetage. + Utiliser le nom de sortie tel que saisi. Sinon, l’extension est adaptée au format (.ffpfsc ou .ffpfs). + Garder le nom du fichier source dans l’image. Sinon, il est renommé avec un nom sûr et normalisé. + Construire via un fichier temporaire intermédiaire au lieu d’écrire directement dans l’image. Plus lent ; à n’utiliser que si la méthode par défaut échoue. + Profil du format PFS : PS5 pour les jeux PS5, PS4 pour les images PS4. + Largeur des entrées d’inode : 32 bits (par défaut) ou 64 bits. + Rendre les noms de fichiers de l’image sensibles à la casse. Par défaut, ils ne le sont pas. + Écrire plus de détails dans le journal. + Chiffrer les blocs de l’image avec AES-XTS. + Dériver les clés de chiffrement avec la méthode alternative newCrypt. + 64 caractères hexadécimaux : la clé EKPFS pour chiffrer ou signer. Vide utilise uniquement des zéros. + Taille de cluster exFAT. Auto (64 Kio) fonctionne le mieux avec ShadowMountPlus ; des clusters plus petits gaspillent moins de place pour les petits fichiers mais lisent plus lentement. + Comment lire l’image : Auto détecte PFS ou exFAT ; choisissez-en un pour le forcer. + Avertir s’il manque sce_sys/param.json, eboot.bin ou pfs-version.dat dans le jeu. diff --git a/src/MkPFS.Gui/Localization/Strings_pt_BR.resx b/src/MkPFS.Gui/Localization/Strings_pt_BR.resx index 9f150cb..aac7a65 100644 --- a/src/MkPFS.Gui/Localization/Strings_pt_BR.resx +++ b/src/MkPFS.Gui/Localization/Strings_pt_BR.resx @@ -16,8 +16,6 @@ Manter Extensão de Saída Manter Nome do Arquivo Interno Derivação de Chave Alternativa (newCrypt) - Imagem PFS Bruta (.ffpfs) - Exigir Arquivos do Jogo Pular Verificação Final Usar Construtor com Spool Avançado @@ -32,28 +30,63 @@ Pasta com libSceAmpr.sprx… APR Emu Tratar como título APR + Criar jogo pronto (packs + arquivos soltos) + Escolha o perfil TOML deste jogo, ou uma pasta de traces APR para criar as regras. Perfis são arquivos TOML compartilhados para cada jogo. + O seu perfil TOML decide o que é empacotado. + Pasta de traces (opcional) + Pasta com ampr_commands.bin e ampr_emu.index do AMPR Emu de debug, uma execução por subpasta + ✓ Regras a partir dos traces: {0} execução(ões) gravada(s). Só os arquivos que o jogo leu pelo AMPR são empacotados; os demais ficam soltos. + Nenhum trace nesta pasta: cada execução precisa de ampr_commands.bin e do seu ampr_emu.index lado a lado. + ✗ Escolha um perfil TOML ou uma pasta de traces: sem regras nada é empacotado. + Use um arquivo TOML ou uma pasta de traces, não os dois. + Empacotar também os arquivos que nenhuma sessão leu, quando um arquivo rastreado tem o mesmo tipo. Eles não foram vistos sendo lidos, então teste o jogo com eles. + Espaço livre dentro da imagem (opcional, ex. 2GiB): necessário para um teste com o AMPR Emu de debug, que grava o log na pasta do jogo + ⚠ Experimental: se um jogo empacotado funciona depende de como ele lê seus arquivos, e só um teste no PS5 mostra isso. Mantenha o jogo original até jogar o empacotado. + Cria uma nova pasta que você pode copiar para o PS5 e jogar: suas bibliotecas em fakelib/, ampr_emu.index, os packs LZ4 e todos os arquivos que ficam soltos. A pasta do jogo é apenas lida. Requer AMPR Emu 0.4.2.1 ou mais recente e um jogo que use AMPR. + Pasta de Bibliotecas (fakelib) + Pasta com libSceAmpr.sprx e outras bibliotecas fakelib (opcional se o jogo já as tiver)… + Verificar os packs e os arquivos soltos após criar + Criar também uma imagem exFAT (clusters de 64 KiB, montar somente leitura) + Imagem exFAT (opcional) + Padrão: <titleId>.exfat ao lado da pasta de saída + Inspecionar manifesto + Listar arquivos + Somente empacotar (manifesto e volumes .pak) + Remover arquivos-fonte empacotados + Atualizar configurações de execução + Desempacotar arquivos + Verificar pacotes + Ação + Deixar soltos os arquivos-fonte ausentes + Configuração TOML + Perfil TOML do jogo… + Excluir os arquivos-fonte empacotados (não pode ser desfeito; desmarcado mostra apenas o plano) + ✗ O manifesto do pacote é obrigatório. + ✗ A pasta do jogo e a pasta de saída são obrigatórias. + ✗ O manifesto do pacote e a pasta do jogo são obrigatórios. + ✗ O manifesto do pacote e a configuração TOML são obrigatórios. + ✗ O manifesto do pacote e a pasta de saída são obrigatórios. + ✗ Threads devem ser um número inteiro de 1 a 256. + Índice AMPR (opcional) + Padrão: ampr_emu.index na pasta do jogo + Saída JSON + Manifesto do pacote + Selecione ampr_assets.index… + Grava apenas ampr_assets.index, os volumes .pak e o arquivo .crc na pasta de saída, como o ampr_pack.py. Para uma pasta que você copia para o PS5 e joga, use Criar jogo pronto. + Os pacotes atendem os arquivos que o jogo lê pelo AMPR e por leituras comuns de arquivo (AMPR Emu 0.4.2.1). Arquivos mapeados em memória devem ficar soltos; mantenha os arquivos-fonte até testar o jogo empacotado. + Pasta de saída + Pasta para o manifesto e os volumes .pak… + Sobrescrever arquivos existentes + Pasta do jogo (/app0) + Pasta com eboot.bin e sce_sys… + Nunca deixar soltos os arquivos selecionados (guardar sem compressão os blocos incompressíveis) + Crie e verifique pacotes de recursos LZ4 do AMPR Emu para uma pasta de jogo + Pacotes AMPR + Threads (vazio = da configuração) Sistema de Arquivos PlayStation Procurar Arquivo Pasta - Compressão PFSC - Simulação - ✗ A pasta de saída é obrigatória. - ✗ A pasta fonte é obrigatória. - Pasta de Saída - Selecione a pasta de saída… - Sobrescrever Existente - {0} item(ns): {1} pasta(s), {2} arquivo(s) - Selecione uma pasta fonte para pré-visualizar a fila do lote. - Não foi possível escanear a pasta do lote: {0} - Selecione uma pasta existente para pré-visualizar a fila do lote. - Nenhum item empacotável encontrado. - Escaneando itens do lote... - Fila do Lote - Pasta Fonte - Selecione a pasta fonte… - Converter vários jogos de uma vez em imagens PFS - Converter em Lote Versão Cancelar ✗ Cancelado. @@ -65,7 +98,8 @@ A saída escrita pela metade é removida; um reparo no local termina sua regrava Tamanho do Bloco (jogos de PS5 usam 64 KiB) Automático (64 KiB) Ajuste automático (menos preenchimento) - Núcleos de CPU (0 = auto: {0}) + Núcleos de CPU + Automático ({0} núcleos) ✗ Armazenar Arquivos Menores Que deve ser um número inteiro de bytes. Nível de Compressão (0–9) Tamanho Comprimido Máx. (% do original) @@ -105,11 +139,10 @@ A saída escrita pela metade é removida; um reparo no local termina sua regrava Inspecionar Idioma Prévia da Importação - Lote + Pacotes AMPR Inspecionar Empacotar EXFAT - Empacotar Arquivo - Empacotar Pasta + Empacotar FFPFSC Reparar Árvore Desempacotar @@ -119,19 +152,8 @@ A saída escrita pela metade é removida; um reparo no local termina sua regrava Opções Log de Saída Caminhos - Compressão PFSC Simulação (Dry Run) - ✗ O caminho de saída é obrigatório. - ✗ A pasta fonte é obrigatória. - Imagem de Saída - Selecione o caminho de saída… Imagem Assinada - Pasta Fonte - Selecione a pasta fonte… - Construir uma imagem PFS a partir de um diretório fonte - Pasta Temporária (opcional) - Pasta temp personalizada… - Empacotar Pasta Verificar Após Empacotar Versão Compressão PFSC @@ -153,7 +175,7 @@ A saída escrita pela metade é removida; um reparo no local termina sua regrava Encapsular um único arquivo em uma imagem PFSC comprimida Pasta Temporária (opcional) Pasta temp personalizada… - Empacotar Arquivo + Empacotar FFPFSC Versão Lista de Blocos Ruins (opcional — TSV do Game Compressor) bad_blocks.tsv… @@ -232,4 +254,29 @@ A saída escrita pela metade é removida; um reparo no local termina sua regrava Validar estrutura da imagem, inodes e checksums Verificar v{0} + Ajustes rápidos do nível de compressão e dos núcleos de CPU. Alterar qualquer um dos valores seleciona a predefinição correspondente, ou Personalizado. + Nível zlib de cada bloco: 1 é o mais rápido, 9 gera a menor imagem e 0 guarda os blocos sem compressão. O padrão é 7. + Quantos núcleos de CPU comprimem blocos ao mesmo tempo. Automático usa todos os núcleos deste computador; escolha menos para manter o computador responsivo ou usar menos memória. + Tamanho dos blocos em que a imagem é dividida e comprimida um a um. Jogos de PS5 usam 64 KiB; altere apenas se souber que o destino precisa de outro tamanho. + Um bloco comprimido só é mantido se economizar pelo menos esta porcentagem; caso contrário, é guardado sem compressão. 0 mantém todo bloco que ficar menor. + Um arquivo fica sem compressão quando o tamanho comprimido ultrapassaria esta porcentagem do original. 100 comprime todo arquivo que ficar menor. + Arquivos menores que este número de bytes são guardados sem compressão, sem tentar. Vazio usa o tamanho do bloco. + Guardar sem compressão os executáveis e módulos (eboot.bin, .prx, .sprx). + Criar uma imagem PFS assinada, com chave EKPFS e semente zeradas. + Verificar a origem e mostrar o que seria gravado, sem gravar a imagem. + Depois de empacotar, decodificar a imagem inteira e compará-la com a origem. Mais lento, mas comprova que a imagem está correta. + Pular a verificação rápida de estrutura feita após empacotar. + Usar o nome de saída exatamente como digitado. Caso contrário, a extensão é ajustada ao formato (.ffpfsc ou .ffpfs). + Manter dentro da imagem o nome do arquivo de origem. Caso contrário, ele é renomeado para um nome seguro e normalizado. + Criar por meio de um arquivo temporário intermediário em vez de gravar direto na imagem. Mais lento; use apenas se o modo padrão falhar. + Perfil do formato PFS: PS5 para jogos de PS5, PS4 para imagens de PS4. + Largura das entradas de inode: 32 bits (padrão) ou 64 bits. + Fazer os nomes de arquivo da imagem diferenciarem maiúsculas e minúsculas. Por padrão, não diferenciam. + Gravar mais detalhes no registro. + Criptografar os blocos da imagem com AES-XTS. + Derivar as chaves de criptografia com o método alternativo newCrypt. + 64 caracteres hexadecimais: a chave EKPFS para criptografar ou assinar. Vazio usa somente zeros. + Tamanho do cluster exFAT. Automático (64 KiB) funciona melhor com o ShadowMountPlus; clusters menores desperdiçam menos espaço em arquivos pequenos, mas leem mais devagar. + Como ler a imagem: Automático detecta PFS ou exFAT; escolha um para forçá-lo. + Avisar quando faltar sce_sys/param.json, eboot.bin ou pfs-version.dat no jogo. diff --git a/src/MkPFS.Gui/Localization/Strings_ro.resx b/src/MkPFS.Gui/Localization/Strings_ro.resx index 73a5cc7..d8cbc1d 100644 --- a/src/MkPFS.Gui/Localization/Strings_ro.resx +++ b/src/MkPFS.Gui/Localization/Strings_ro.resx @@ -16,8 +16,6 @@ Păstrează extensia de ieșire Păstrează numele fișierului intern Derivare alternativă a cheii (newCrypt) - Imagine PFS brută (.ffpfs) - Cere fișierele jocului Omite verificarea după împachetare Folosește constructorul cu spool Avansat @@ -32,28 +30,63 @@ Folder cu libSceAmpr.sprx… APR Emu Tratează ca titlu APR + Creează jocul gata de rulat (pack-uri + fișiere libere) + Alege profilul TOML pentru acest joc sau un dosar cu trace-uri APR din care să se creeze regulile. Profilurile sunt fișiere TOML partajate pentru fiecare joc. + Profilul tău TOML decide ce se împachetează. + Dosar cu trace-uri (opțional) + Dosar cu ampr_commands.bin și ampr_emu.index de la AMPR Emu de depanare, o rulare pe subdosar + ✓ Reguli din trace-uri: {0} rulare(i) înregistrată(e). Se împachetează doar fișierele citite de joc prin AMPR; restul rămân separate. + Niciun trace în acest dosar: fiecare rulare are nevoie de ampr_commands.bin și de ampr_emu.index alături. + ✗ Alege un profil TOML sau un dosar cu trace-uri: fără reguli nu se împachetează nimic. + Folosește fie un fișier TOML, fie un dosar cu trace-uri, nu amândouă. + Împachetează și fișierele pe care nicio sesiune nu le-a citit, dacă un fișier urmărit are același tip. Nu s-a văzut cum sunt citite, așa că testează jocul cu ele. + Spațiu liber în imagine (opțional, ex. 2GiB): necesar pentru un test cu AMPR Emu de depanare, care își scrie jurnalul în dosarul jocului + ⚠ Experimental: dacă un joc împachetat rulează depinde de cum își citește fișierele, iar doar un test pe PS5 arată asta. Păstrează jocul original până când ai jucat varianta împachetată. + Creează un folder nou pe care îl copiezi pe PS5 și joci: bibliotecile tale în fakelib/, ampr_emu.index, pack-urile LZ4 și toate fișierele care rămân libere. Folderul jocului este doar citit. Necesită AMPR Emu 0.4.2.1 sau mai nou și un joc care folosește AMPR. + Folder de Biblioteci (fakelib) + Folder cu libSceAmpr.sprx și alte biblioteci fakelib (opțional dacă jocul le are deja)… + Verifică pack-urile și fișierele libere după creare + Creează și o imagine exFAT (clustere de 64 KiB, montare doar citire) + Imagine exFAT (opțional) + Implicit: <titleId>.exfat lângă folderul de ieșire + Inspectează manifestul + Listează fișierele + Doar împachetare (manifest și volume .pak) + Șterge sursele împachetate + Actualizează setările de rulare + Despachetează fișierele împachetate + Verifică pachetele + Acțiune + Lasă libere fișierele sursă lipsă + Configurație TOML + Profilul TOML al jocului… + Șterge fișierele sursă împachetate (ireversibil; nebifat arată doar planul) + ✗ Manifestul pachetului este necesar. + ✗ Folderul jocului și folderul de ieșire sunt ambele necesare. + ✗ Manifestul pachetului și folderul jocului sunt ambele necesare. + ✗ Manifestul pachetului și configurația TOML sunt ambele necesare. + ✗ Manifestul pachetului și folderul de ieșire sunt ambele necesare. + ✗ Firele trebuie să fie un număr întreg de la 1 la 256. + Index AMPR (opțional) + Implicit: ampr_emu.index din folderul jocului + Ieșire JSON + Manifestul pachetului + Selectează ampr_assets.index… + Scrie doar ampr_assets.index, volumele .pak și fișierul .crc în folderul de ieșire, ca ampr_pack.py. Pentru un folder pe care îl copiezi pe PS5 și joci, folosește Creează jocul gata de rulat. + Pachetele servesc fișierele pe care jocul le citește prin AMPR și prin citiri obișnuite de fișiere (AMPR Emu 0.4.2.1). Fișierele mapate în memorie trebuie să rămână libere; păstrează fișierele sursă până testezi jocul împachetat. + Folder de ieșire + Folder pentru manifest și volumele .pak… + Suprascrie fișierele existente + Folderul jocului (/app0) + Folder cu eboot.bin și sce_sys… + Nu lăsa niciodată libere fișierele selectate (stochează necomprimate blocurile necompresibile) + Creează și verifică pachete de resurse LZ4 AMPR Emu pentru un folder de joc + Pachete AMPR + Fire (gol = din configurație) PlayStation File System Răsfoiește Fișier Folder - Compresie PFSC - Simulare - ✗ Folderul de ieșire este obligatoriu. - ✗ Folderul sursă este obligatoriu. - Folder de ieșire - Alege folderul de ieșire… - Suprascrie existente - {0} element(e): {1} folder(e), {2} fișier(e) - Alege un folder sursă pentru a previzualiza coada. - Nu se poate scana folderul: {0} - Alege un folder existent pentru a previzualiza coada. - Nu s-au găsit elemente de împachetat. - Se scanează elementele... - Coadă lot - Folder sursă - Alege folderul sursă… - Convertește mai multe jocuri deodată în imagini PFS - Conversie în lot Versiune Anulează ✗ Anulat. @@ -65,7 +98,8 @@ Ieșirea scrisă parțial este ștearsă; o reparare pe loc își termină mai Dimensiune bloc (jocurile PS5 folosesc 64 KiB) Auto (64 KiB) Auto-potrivire (umplere minimă) - Nuclee CPU (0 = auto: {0}) + Nuclee CPU + Automat ({0} nuclee) ✗ „Stochează fișierele mai mici de” trebuie să fie un număr întreg de octeți. Nivel de compresie (0–9) Dimensiune comprimată max. (% din original) @@ -105,11 +139,10 @@ Ieșirea scrisă parțial este ștearsă; o reparare pe loc își termină mai Inspectare Limbă Previzualizare import - Lot + Pachete AMPR Inspectare Împachetare exFAT - Împachetare fișier - Împachetare folder + Împachetare FFPFSC Reparare Arbore Despachetare @@ -119,19 +152,8 @@ Ieșirea scrisă parțial este ștearsă; o reparare pe loc își termină mai Opțiuni Jurnal de ieșire Căi - Compresie PFSC Simulare - ✗ Calea imaginii de ieșire este obligatorie. - ✗ Folderul sursă este obligatoriu. - Imagine de ieșire - Alege calea de ieșire… Imagine semnată - Folder sursă - Alege folderul sursă… - Creează o imagine PFS dintr-un director sursă - Folder temporar (opțional) - Folder temporar personalizat… - Împachetare folder Verifică după împachetare Versiune Compresie PFSC @@ -153,7 +175,7 @@ Ieșirea scrisă parțial este ștearsă; o reparare pe loc își termină mai Împachetează un singur fișier într-o imagine comprimată PFSC Folder temporar (opțional) Folder temporar personalizat… - Împachetare fișier + Împachetare FFPFSC Versiune Listă blocuri defecte (opțional — TSV Game Compressor) bad_blocks.tsv… @@ -232,4 +254,29 @@ Ieșirea scrisă parțial este ștearsă; o reparare pe loc își termină mai Validează structura imaginii, inode-urile și sumele de control Verificare v{0} + Setări rapide pentru nivelul de compresie și nucleele CPU. Dacă schimbi oricare valoare, se alege presetarea potrivită sau Personalizat. + Nivelul zlib pentru fiecare bloc: 1 este cel mai rapid, 9 dă cea mai mică imagine, 0 păstrează blocurile necomprimate. Implicit este 7. + Câte nuclee CPU comprimă blocuri în același timp. Automat folosește toate nucleele acestui calculator; alege mai puține ca să rămână calculatorul responsiv sau să folosească mai puțină memorie. + Dimensiunea blocurilor în care este împărțită imaginea și comprimată bloc cu bloc. Jocurile PS5 folosesc 64 KiB; schimb-o doar dacă știi că ținta are nevoie de altă dimensiune. + Un bloc comprimat este păstrat doar dacă economisește cel puțin acest procent; altfel este stocat necomprimat. 0 păstrează orice bloc care devine mai mic. + Un fișier rămâne necomprimat dacă dimensiunea comprimată ar depăși acest procent din original. 100 comprimă orice fișier care devine mai mic. + Fișierele mai mici decât acest număr de octeți sunt stocate necomprimate, fără încercare. Gol înseamnă dimensiunea blocului. + Stochează necomprimate executabilele și modulele (eboot.bin, .prx, .sprx). + Creează o imagine PFS semnată, cu cheie EKPFS și seed zero. + Verifică sursa și arată ce s-ar scrie, fără să scrie imaginea. + După împachetare, decodează toată imaginea și o compară cu sursa. Mai lent, dar dovedește că imaginea este corectă. + Sari peste verificarea rapidă a structurii de după împachetare. + Folosește numele de ieșire exact cum a fost scris. Altfel, extensia este potrivită formatului (.ffpfsc sau .ffpfs). + Păstrează în imagine numele fișierului sursă. Altfel, este redenumit cu un nume sigur, normalizat. + Construiește printr-un fișier temporar intermediar în loc să scrie direct în imagine. Mai lent; folosește-l doar dacă modul implicit eșuează. + Profilul formatului PFS: PS5 pentru jocuri PS5, PS4 pentru imagini PS4. + Lățimea intrărilor inode: 32 de biți (implicit) sau 64 de biți. + Numele fișierelor din imagine țin cont de majuscule. Implicit nu țin. + Scrie mai multe detalii în jurnal. + Criptează blocurile imaginii cu AES-XTS. + Derivă cheile de criptare cu metoda alternativă newCrypt. + 64 de caractere hexazecimale: cheia EKPFS pentru criptare sau semnare. Gol înseamnă doar zerouri. + Dimensiunea clusterului exFAT. Automat (64 KiB) merge cel mai bine cu ShadowMountPlus; clusterele mai mici irosesc mai puțin spațiu la fișiere mici, dar citesc mai încet. + Cum se citește imaginea: Automat detectează PFS sau exFAT; alege unul ca să-l forțezi. + Avertizează dacă jocului îi lipsește sce_sys/param.json, eboot.bin sau pfs-version.dat. diff --git a/src/MkPFS.Gui/ViewModels/AmprPackPanelViewModel.cs b/src/MkPFS.Gui/ViewModels/AmprPackPanelViewModel.cs new file mode 100644 index 0000000..e9fe91e --- /dev/null +++ b/src/MkPFS.Gui/ViewModels/AmprPackPanelViewModel.cs @@ -0,0 +1,354 @@ +using System.Globalization; +using Avalonia.Media; +using CommunityToolkit.Mvvm.ComponentModel; +using MkPFS.Gui.Jobs; +using MkPFS.Gui.Localization; + +namespace MkPFS.Gui.ViewModels; + +/// +/// AMPR Packs page: runs mkpfs ampr (game, pack, verify, unpack, list, inspect, runtime-config, remove-sources). +/// Only the fields the selected action uses are shown. +/// +/// Accent color. +/// Job runner, or for the UI-thread runner. +public sealed partial class AmprPackPanelViewModel(Color accent, JobRunner? job = null) : PanelViewModel("ap_title", "ap_subtitle", accent, job) +{ + private static readonly Choice[] ActionChoices = + [ + new("game", "ap_act_game"), + new("pack", "ap_act_pack"), + new("verify", "ap_act_verify"), + new("unpack", "ap_act_unpack"), + new("list", "ap_act_list"), + new("inspect", "ap_act_inspect"), + new("runtime-config", "ap_act_runtime"), + new("remove-sources", "ap_act_remove"), + ]; + + /// Actions, in menu order; values are the CLI subcommands. + public IReadOnlyList Actions => ActionChoices; + + /// Selected action. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ShowRoot), nameof(ShowPackPaths), nameof(ShowOutput), nameof(ShowManifest), nameof(ShowConfig))] + [NotifyPropertyChangedFor(nameof(ShowPackOptions), nameof(ShowOverwrite), nameof(ShowJson), nameof(ShowConfirm))] + [NotifyPropertyChangedFor(nameof(ShowGameOptions), nameof(ShowAllowMissing), nameof(ShowExfatPath), nameof(ShowPackNote))] + [NotifyPropertyChangedFor(nameof(ShowRulesMissing), nameof(ShowRulesConfig), nameof(ShowRulesTraces), nameof(ShowNoTraces))] + public partial Choice Action { get; set; } = ActionChoices[0]; + + /// --fakelib (game): folder with AMPR Emu and other libraries to add. + [ObservableProperty] + public partial string Libs { get; set; } = string.Empty; + + /// Verify the packs and loose files after a game build (--skip-verify when off). + [ObservableProperty] + public partial bool VerifyGame { get; set; } = true; + + /// Also build an exFAT image of the game folder (--exfat). + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ShowExfatPath))] + public partial bool Exfat { get; set; } + + /// exFAT image path; empty for <titleId>.exfat next to the output folder. + [ObservableProperty] + public partial string ExfatPath { get; set; } = string.Empty; + + /// --exfat-free-space: free space inside the image, e.g. 2GiB; empty for none. + [ObservableProperty] + public partial string ExfatFree { get; set; } = string.Empty; + + /// --root: the game's /app0 folder. + [ObservableProperty] + public partial string Root { get; set; } = string.Empty; + + /// --ampr-index; defaults to <root>/ampr_emu.index. + [ObservableProperty] + public partial string AmprIndex { get; set; } = string.Empty; + + /// --output. + [ObservableProperty] + public partial string Output { get; set; } = string.Empty; + + /// --index: the pack manifest. + [ObservableProperty] + public partial string Manifest { get; set; } = string.Empty; + + /// --config: the game's TOML profile; game and pack need it or a trace folder, runtime-config needs it. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ShowRulesMissing), nameof(ShowRulesConfig), nameof(ShowRulesTraces), nameof(ShowNoTraces))] + public partial string Config { get; set; } = string.Empty; + + /// --traces (game, pack): folder with APR traces of the debug emulator build. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ShowRulesMissing), nameof(ShowRulesConfig), nameof(ShowRulesTraces), nameof(ShowNoTraces))] + [NotifyPropertyChangedFor(nameof(RulesTracesText))] + public partial string Traces { get; set; } = string.Empty; + + /// --pack-untraced-types: also pack untraced files of the traced file types. + [ObservableProperty] + public partial bool UntracedTypes { get; set; } + + /// --workers; empty for the configuration's value. + [ObservableProperty] + public partial string Workers { get; set; } = string.Empty; + + /// --self-contained. + [ObservableProperty] + public partial bool SelfContained { get; set; } + + /// --allow-missing. + [ObservableProperty] + public partial bool AllowMissing { get; set; } + + /// --overwrite (unpack). + [ObservableProperty] + public partial bool Overwrite { get; set; } + + /// --json (list). + [ObservableProperty] + public partial bool Json { get; set; } + + /// --confirm (remove-sources): delete instead of printing the plan. + [ObservableProperty] + public partial bool Confirm { get; set; } + + /// Game folder field (pack, verify, remove-sources). + public bool ShowRoot => Action.Value is "game" or "pack" or "verify" or "remove-sources"; + + /// AMPR index field (pack). + public bool ShowPackPaths => Action.Value == "pack"; + + /// Output folder field (game, pack, unpack). + public bool ShowOutput => Action.Value is "game" or "pack" or "unpack"; + + /// Pack manifest field (every action but game and pack). + public bool ShowManifest => Action.Value is not ("game" or "pack"); + + /// Configuration field (game, pack, runtime-config). + public bool ShowConfig => Action.Value is "game" or "pack" or "runtime-config"; + + /// Workers and self-contained (game, pack). + public bool ShowPackOptions => Action.Value is "game" or "pack"; + + /// Allow-missing (pack). + public bool ShowAllowMissing => Action.Value == "pack"; + + /// Note that pack writes only the pack set (pack). + public bool ShowPackNote => Action.Value == "pack"; + + /// Library folder, verification and exFAT image (game). + public bool ShowGameOptions => Action.Value == "game"; + + /// exFAT image path (game with an image). + public bool ShowExfatPath => ShowGameOptions && Exfat; + + // Trace runs found in the trace folder, or null when the field is empty. + private int? _traceRuns; + + /// The trace folder has runs: the rules come from the traces. + public bool ShowRulesTraces => ShowPackOptions && Config.Trim().Length == 0 && _traceRuns > 0; + + /// The trace folder has no runs. + public bool ShowNoTraces => ShowPackOptions && Config.Trim().Length == 0 && _traceRuns == 0; + + /// How many trace runs were found. + public string RulesTracesText => Localizer.Instance.Format("ap_rules_traces", _traceRuns ?? 0); + + /// Neither a TOML profile nor a trace folder is chosen yet (game, pack). + public bool ShowRulesMissing => ShowPackOptions && Config.Trim().Length == 0 && Traces.Trim().Length == 0; + + /// The user's TOML file decides what is packed. + public bool ShowRulesConfig => ShowPackOptions && Config.Trim().Length > 0 && Traces.Trim().Length == 0; + + /// Overwrite (unpack). + public bool ShowOverwrite => Action.Value == "unpack"; + + /// JSON output (list). + public bool ShowJson => Action.Value == "list"; + + /// Delete sources (remove-sources). + public bool ShowConfirm => Action.Value == "remove-sources"; + + /// + public override object Form => this; + + /// + protected internal override IReadOnlyList? BuildArguments(out string? error) + { + // Files may have been copied into the folders since they were picked. + RefreshDetection(); + string root = Root.Trim(); + string manifest = Manifest.Trim(); + string output = Output.Trim(); + string config = Config.Trim(); + string traces = Traces.Trim(); + error = Action.Value switch + { + "game" or "pack" when config.Length > 0 && traces.Length > 0 => Localizer.Instance["ap_err_traces_config"], + "game" when root.Length == 0 || output.Length == 0 => Localizer.Instance["ap_err_pack"], + "pack" when root.Length == 0 || output.Length == 0 => Localizer.Instance["ap_err_pack"], + "game" or "pack" when config.Length == 0 && traces.Length == 0 => Localizer.Instance["ap_err_rules"], + "game" or "pack" when Workers.Trim().Length > 0 && !(int.TryParse(Workers.Trim(), NumberStyles.None, CultureInfo.InvariantCulture, out int n) && n is >= 1 and <= 256) + => Localizer.Instance["ap_err_workers"], + "verify" or "list" or "inspect" when manifest.Length == 0 => Localizer.Instance["ap_err_manifest"], + "unpack" when manifest.Length == 0 || output.Length == 0 => Localizer.Instance["ap_err_unpack"], + "runtime-config" when manifest.Length == 0 || config.Length == 0 => Localizer.Instance["ap_err_runtime"], + "remove-sources" when manifest.Length == 0 || root.Length == 0 => Localizer.Instance["ap_err_remove"], + _ => null, + }; + if (error is not null) + { + return null; + } + + List args = ["ampr", Action.Value]; + switch (Action.Value) + { + case "game": + args.AddRange(["--root", root, "--output", output]); + AddOption(args, "--fakelib", Libs); + AddOption(args, "--config", config); + AddOption(args, "--traces", traces); + AddFlag(args, traces.Length > 0 && UntracedTypes, "--pack-untraced-types"); + AddOption(args, "--workers", Workers); + AddFlag(args, SelfContained, "--self-contained"); + AddFlag(args, !VerifyGame, "--skip-verify"); + if (Exfat) + { + // An existing folder receives .exfat: by default the folder that holds the output. + args.AddRange(["--exfat", ExfatPath.Trim() is { Length: > 0 } image ? image : Path.GetDirectoryName(Path.GetFullPath(output)) ?? output]); + AddOption(args, "--exfat-free-space", ExfatFree); + } + + break; + case "pack": + args.AddRange(["--root", root, "--ampr-index", AmprIndex.Trim() is { Length: > 0 } index ? index : Path.Combine(root, "ampr_emu.index"), "--output", output]); + AddOption(args, "--config", config); + AddOption(args, "--traces", traces); + AddFlag(args, traces.Length > 0 && UntracedTypes, "--pack-untraced-types"); + AddOption(args, "--workers", Workers); + AddFlag(args, SelfContained, "--self-contained"); + AddFlag(args, AllowMissing, "--allow-missing"); + break; + case "verify": + args.AddRange(["--index", manifest]); + AddOption(args, "--root", root); + break; + case "unpack": + args.AddRange(["--index", manifest, "--output", output]); + AddFlag(args, Overwrite, "--overwrite"); + break; + case "list": + args.AddRange(["--index", manifest]); + AddFlag(args, Json, "--json"); + break; + case "inspect": + args.AddRange(["--index", manifest]); + break; + case "runtime-config": + args.AddRange(["--index", manifest, "--config", config]); + break; + default: + args.AddRange(["--index", manifest, "--root", root]); + AddFlag(args, Confirm, "--confirm"); + break; + } + + return args; + } + + /// Count the trace runs in the chosen folder (each needs ampr_commands.bin and ampr_emu.index). + /// + /// The folder is searched recursively, so the count runs off the UI thread after typing pauses: a + /// half-typed path such as a drive root would otherwise freeze the window. + /// + partial void OnTracesChanged(string value) + { + _traceCount?.Cancel(); + _traceRuns = null; + TraceCount = CountTraceRunsLaterAsync(value); + } + + /// Wait after the last trace folder change before counting its runs. + public TimeSpan TraceCountDebounce { get; set; } = TimeSpan.FromMilliseconds(250); + + /// The pending background count of the trace folder (tests await it). + internal Task TraceCount { get; private set; } = Task.CompletedTask; + + private CancellationTokenSource? _traceCount; + + private async Task CountTraceRunsLaterAsync(string value) + { + using CancellationTokenSource pending = new(); + _traceCount = pending; + try + { + if (TraceCountDebounce > TimeSpan.Zero) + { + await Task.Delay(TraceCountDebounce, pending.Token).ConfigureAwait(true); + } + + int? runs = await Task.Run(() => TraceRuns(value), pending.Token).ConfigureAwait(true); + if (!pending.IsCancellationRequested) + { + _traceRuns = runs; + NotifyTraceLines(); + } + } + catch (OperationCanceledException) + { + // Superseded by a newer folder or by RefreshDetection. + } + finally + { + if (ReferenceEquals(_traceCount, pending)) + { + _traceCount = null; + } + } + } + + /// Look at the trace folder again and update the rules lines. + public void RefreshDetection() + { + _traceCount?.Cancel(); + _traceRuns = TraceRuns(Traces); + NotifyTraceLines(); + } + + private void NotifyTraceLines() + { + OnPropertyChanged(nameof(ShowRulesTraces)); + OnPropertyChanged(nameof(ShowNoTraces)); + OnPropertyChanged(nameof(RulesTracesText)); + } + + // Trace runs in the folder, or null when the field is empty. + private static int? TraceRuns(string value) + { + string folder = value.Trim(); + if (folder.Length == 0) + { + return null; + } + + try + { + return Directory.Exists(folder) ? Build.AMPRPack.AMPRProfiler.DiscoverTracePairs(Path.GetFullPath(folder)).Count : 0; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or Core.AMPR.AMPRPackException) + { + return 0; + } + } + + /// The manifest a pack run writes, so the other actions can use it right away. + partial void OnOutputChanged(string value) + { + if (Manifest.Length == 0 && value.Trim().Length > 0 && Action.Value is "game" or "pack") + { + Manifest = Path.Combine(value.Trim(), Build.AMPRPack.AMPRPackConfig.DefaultIndexName); + } + } +} diff --git a/src/MkPFS.Gui/ViewModels/BatchQueueViewModel.cs b/src/MkPFS.Gui/ViewModels/BatchQueueViewModel.cs deleted file mode 100644 index 5588ae5..0000000 --- a/src/MkPFS.Gui/ViewModels/BatchQueueViewModel.cs +++ /dev/null @@ -1,134 +0,0 @@ -using System.Collections.ObjectModel; -using Avalonia.Media; -using Avalonia.Media.Imaging; -using CommunityToolkit.Mvvm.ComponentModel; -using MkPFS.Build; -using MkPFS.Build.PFS; -using MkPFS.Core.Diagnostics; -using MkPFS.Core.Metadata; -using MkPFS.Gui.Localization; - -namespace MkPFS.Gui.ViewModels; - -/// One discovered batch item (Python BatchItemRow). -public sealed class BatchQueueRow -{ - internal BatchQueueRow(BatchItem item, GameMetadata metadata, IBrush accent) - { - Accent = accent; - Localizer tr = Localizer.Instance; - string Or(string value) => value.Length > 0 ? value : "-"; - string kind = item.Kind == BatchItemKind.Folder ? "FOLDER" : "FILE"; - Title = metadata.GameTitle.Length > 0 ? metadata.GameTitle : metadata.FileName.Length > 0 ? metadata.FileName : item.Name; - Details = $"{tr["pkf_meta_title_id"]}: {Or(metadata.TitleId)} | {tr["pkf_meta_version"]}: {Or(metadata.Version)}" - + $" | {(metadata.PackageType.Length > 0 ? metadata.PackageType : kind)} | {tr["pkf_meta_size"]}: {metadata.SizeDisplay}" - + $" | {tr["pkf_meta_region"]}: {Or(metadata.Region)}"; - ContentId = $"{tr["pkf_meta_content_id"]}: {Or(metadata.ContentId)}"; - Kind = kind; - AprEmu = metadata.HasAprEmu ? $"{tr["pkf_meta_apr_emu"]}: {tr["pkf_meta_apr_yes"]}" : null; - Cover = MetadataPreviewViewModel.DecodeCover(metadata.IconBytes); - } - - /// Game title, file name, or item name. - public string Title { get; } - - /// Title ID, version, type, size, and region. - public string Details { get; } - - /// Content ID line. - public string ContentId { get; } - - /// FOLDER or FILE. - public string Kind { get; } - - /// "APR-EMU: Yes" when the item carries the APR Emu marker. - public string? AprEmu { get; } - - /// Cover image. - public Bitmap? Cover { get; } - - /// A cover image is shown. - public bool HasCover => Cover is not null; - - /// Panel accent for the details line. - public IBrush Accent { get; } -} - -/// Batch queue preview: every packable item of the source folder (Python BatchQueuePreview). -/// Panel accent for the details lines. -public sealed partial class BatchQueueViewModel(IBrush accent) : ObservableObject -{ - private int _token; - - /// Discovered items. - public ObservableCollection Rows { get; } = []; - - /// Summary next to the section title. - [ObservableProperty] - public partial string Summary { get; private set; } = Localizer.Instance["bt_preview_empty"]; - - /// The list is empty and shows instead. - [ObservableProperty] - public partial bool IsEmpty { get; private set; } = true; - - /// Scan and read every item's metadata; a newer call supersedes it. - /// Batch source folder. - /// Completes when the list is updated or superseded. - public async Task LoadAsync(string? path) - { - int token = ++_token; - Localizer tr = Localizer.Instance; - Rows.Clear(); - IsEmpty = true; - string trimmed = path?.Trim() ?? string.Empty; - if (trimmed.Length == 0) - { - Summary = tr["bt_preview_empty"]; - return; - } - - if (!Directory.Exists(trimmed)) - { - Summary = tr["bt_preview_invalid"]; - return; - } - - Summary = tr["bt_preview_scanning"]; - List rows; - try - { - rows = await Task.Run(() => Batch.Discover(trimmed, NullLog.Instance) - .Select(item => new BatchQueueRow(item, GameMetadataReader.Read(item.Source), accent)) - .ToList()).ConfigureAwait(true); - } - catch (Exception ex) when (ex is BuildException or IOException or UnauthorizedAccessException) - { - if (token == _token) - { - Summary = tr.Format("bt_preview_error", ex.Message); - } - - return; - } - - if (token != _token) - { - return; - } - - if (rows.Count == 0) - { - Summary = tr["bt_preview_none"]; - return; - } - - int folders = rows.Count(r => r.Kind == "FOLDER"); - Summary = tr.Format("bt_preview_count", rows.Count, folders, rows.Count - folders); - foreach (BatchQueueRow row in rows) - { - Rows.Add(row); - } - - IsEmpty = false; - } -} diff --git a/src/MkPFS.Gui/ViewModels/BuildPanels.cs b/src/MkPFS.Gui/ViewModels/BuildPanels.cs index 5c66f21..213551c 100644 --- a/src/MkPFS.Gui/ViewModels/BuildPanels.cs +++ b/src/MkPFS.Gui/ViewModels/BuildPanels.cs @@ -6,107 +6,6 @@ namespace MkPFS.Gui.ViewModels; -/// Pack Folder page (Python PackFolderPanel): mkpfs pack folder. -/// Accent color. -/// Job runner, or for the UI-thread runner. -public sealed partial class PackFolderPanelViewModel(Color accent, JobRunner? job = null) : PanelViewModel("pf_title", "pf_subtitle", accent, job) -{ - /// Cover and details of the source. - public MetadataPreviewViewModel Metadata { get; } = new(); - - /// Compression tuning. - public CompressionSettingsViewModel Compression { get; } = new(allowAutoFit: true, offerSkipExecutables: true); - - /// Source folder. - [ObservableProperty] - public partial string Source { get; set; } = string.Empty; - - /// Output image. - [ObservableProperty] - public partial string Output { get; set; } = string.Empty; - - /// PFSC compression (off adds --no-compress). - [ObservableProperty] - public partial bool Compress { get; set; } = true; - - /// --signed. - [ObservableProperty] - public partial bool Signed { get; set; } - - /// --verify. - [ObservableProperty] - public partial bool VerifyAfter { get; set; } - - /// --dry-run. - [ObservableProperty] - public partial bool DryRun { get; set; } - - /// --temp-folder, optional. - [ObservableProperty] - public partial string TempFolder { get; set; } = string.Empty; - - /// APR Emu libraries and index. - public AmprSettingsViewModel Ampr { get; } = new(); - - /// Advanced PFS options; the inode width applies to images only. - public PFSOptionsViewModel PFS { get; } = new(offerInodeBits: true, offerNewCrypt: false) { InodeBitsEnabled = false }; - - /// --raw. - [ObservableProperty] - public partial bool Raw { get; set; } - - /// --require-game-files. - [ObservableProperty] - public partial bool RequireGameFiles { get; set; } - - /// --skip-verification (not with ). - [ObservableProperty] - public partial bool SkipVerification { get; set; } - - /// --no-adjust-output-file-extension. - [ObservableProperty] - public partial bool KeepExtension { get; set; } - - /// - public override object Form => this; - - /// - protected internal override IReadOnlyList? BuildArguments(out string? error) - { - string source = Source.Trim(); - string output = Output.Trim(); - error = source.Length == 0 ? Localizer.Instance["pf_err_src"] : output.Length == 0 ? Localizer.Instance["pf_err_out"] : null; - if (error is not null) - { - return null; - } - - List args = ["pack", "folder", source, output]; - AddFlag(args, !Compress, "--no-compress"); - AddFlag(args, Signed, "--signed"); - AddFlag(args, VerifyAfter, "--verify"); - AddFlag(args, DryRun, "--dry-run"); - AddOption(args, "--temp-folder", TempFolder); - AddFlag(args, Raw, "--raw"); - AddFlag(args, RequireGameFiles, "--require-game-files"); - AddFlag(args, SkipVerification && !VerifyAfter, "--skip-verification"); - AddFlag(args, KeepExtension, "--no-adjust-output-file-extension"); - PFS.AppendTo(args); - Ampr.AppendTo(args); - return Compression.AppendTo(args, Compress, out error) ? args : null; - } - - partial void OnRawChanged(bool value) => PFS.InodeBitsEnabled = value; - - // Python _on_src_changed: preview the source and suggest /.ffpfsc. - partial void OnSourceChanged(string value) - { - _ = Metadata.LoadAsync(value); - Output = SuggestOutput(value, Output, Directory.Exists, Path.GetFileName, ".ffpfsc"); - Ampr.NoteSource(AmprSettingsViewModel.HasIndex(value)); - } -} - /// Pack exFAT page (Python ExfatPanel): mkpfs pack exfat. /// Accent color. /// Job runner, or for the UI-thread runner. @@ -281,95 +180,3 @@ partial void OnSourceChanged(string value) Output = SuggestOutput(value, Output, File.Exists, Path.GetFileNameWithoutExtension, ".ffpfsc"); } } - -/// Batch page (Python BatchPanel): mkpfs batch. -/// Accent color. -/// Job runner, or for the UI-thread runner. -public sealed partial class BatchPanelViewModel(Color accent, JobRunner? job = null) : PanelViewModel("bt_title", "bt_subtitle", accent, job) -{ - /// Compression tuning (batch takes no "auto-fit" block size). - public CompressionSettingsViewModel Compression { get; } = new(allowAutoFit: false, offerSkipExecutables: true); - - /// Items the batch would convert. - public BatchQueueViewModel Queue { get; } = new(new Avalonia.Media.Immutable.ImmutableSolidColorBrush(accent)); - - /// Source folder. - [ObservableProperty] - public partial string Source { get; set; } = string.Empty; - - /// Output folder. - [ObservableProperty] - public partial string Output { get; set; } = string.Empty; - - /// PFSC compression (off adds --no-compress). - [ObservableProperty] - public partial bool Compress { get; set; } = true; - - /// --overwrite. - [ObservableProperty] - public partial bool Overwrite { get; set; } - - /// --dry-run. - [ObservableProperty] - public partial bool DryRun { get; set; } - - /// --verify. - [ObservableProperty] - public partial bool VerifyAfter { get; set; } - - /// APR Emu libraries and index. - public AmprSettingsViewModel Ampr { get; } = new(); - - /// Advanced PFS options (batch images always use 32-bit inodes). - public PFSOptionsViewModel PFS { get; } = new(offerInodeBits: false, offerNewCrypt: true); - - /// - public override object Form => this; - - /// - protected internal override IReadOnlyList? BuildArguments(out string? error) - { - string source = Source.Trim(); - string output = Output.Trim(); - error = source.Length == 0 ? Localizer.Instance["bt_err_src"] : output.Length == 0 ? Localizer.Instance["bt_err_out"] : null; - if (error is not null) - { - return null; - } - - List args = ["batch", source, output]; - AddFlag(args, !Compress, "--no-compress"); - AddFlag(args, Overwrite, "--overwrite"); - AddFlag(args, DryRun, "--dry-run"); - AddFlag(args, VerifyAfter, "--verify"); - PFS.AppendTo(args); - Ampr.AppendTo(args); - return Compression.AppendTo(args, Compress, out error) ? args : null; - } - - // Python: preview the queue; the output folder defaults to the source folder. - partial void OnSourceChanged(string value) - { - _ = Queue.LoadAsync(value); - if (Output.Trim().Length == 0 && value.Trim() is { Length: > 0 } source && Directory.Exists(source)) - { - Output = source; - } - - Ampr.NoteSource(AnyFolderHasIndex(value)); - } - - // Batch packs the folders directly inside the source. - private static bool AnyFolderHasIndex(string source) - { - try - { - return source.Trim() is { Length: > 0 } path && Directory.Exists(path) && - Directory.EnumerateDirectories(path).Any(AmprSettingsViewModel.HasIndex); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - return false; - } - } -} diff --git a/src/MkPFS.Gui/ViewModels/CompressionSettingsViewModel.cs b/src/MkPFS.Gui/ViewModels/CompressionSettingsViewModel.cs index d2ee724..ea7bbc7 100644 --- a/src/MkPFS.Gui/ViewModels/CompressionSettingsViewModel.cs +++ b/src/MkPFS.Gui/ViewModels/CompressionSettingsViewModel.cs @@ -1,6 +1,5 @@ using System.Globalization; using CommunityToolkit.Mvvm.ComponentModel; -using MkPFS.Core.PFSC; using MkPFS.Gui.Localization; namespace MkPFS.Gui.ViewModels; @@ -66,11 +65,19 @@ public CompressionSettingsViewModel(bool allowAutoFit, bool offerSkipExecutables } BlockSizes = sizes; + CpuChoices = [AutoCpuChoice(), .. Enumerable.Range(1, MaxCpu).Select(n => new Choice(n.ToString(CultureInfo.InvariantCulture), null, n.ToString(CultureInfo.InvariantCulture)))]; Preset = Presets[1]; BlockSize = BlockSizes[0]; - Localizer.Instance.PropertyChanged += (_, _) => OnPropertyChanged(nameof(CpuLabel)); + Localizer.Instance.PropertyChanged += (_, _) => + { + // The Auto label carries the core count, so it is rebuilt for the new language. + CpuChoices[0] = AutoCpuChoice(); + OnPropertyChanged(nameof(CpuChoice)); + }; } + private Choice AutoCpuChoice() => new("0", null, Localizer.Instance.Format("ct_cpu_auto", MaxCpu)); + /// Presets in picker order. public IReadOnlyList Presets { get; } @@ -80,11 +87,24 @@ public CompressionSettingsViewModel(bool allowAutoFit, bool offerSkipExecutables /// Show the executables option. public bool OfferSkipExecutables { get; } - /// Most CPU cores the picker allows. - public decimal MaxCpu { get; } = Environment.ProcessorCount; + /// Physical CPU cores of this machine (not logical processors): what Auto uses, and the most the picker offers. + public int MaxCpu { get; } = MkPFS.Core.Util.CpuTopology.PhysicalCores; - /// "CPU Cores (0 = auto: N)". - public string CpuLabel => Localizer.Instance.Format("ct_cpu", PFSCEncoder.ResolveWorkerCount(0)); + /// The CPU core picker: Auto (every core of this machine), then 1 to . + public System.Collections.ObjectModel.ObservableCollection CpuChoices { get; } + + /// The selected entry of , kept in step with . + public Choice CpuChoice + { + get => CpuChoices[Math.Clamp(Whole(CpuCount, 0), 0, MaxCpu)]; + set + { + if (value is not null) + { + CpuCount = int.Parse(value.Value, CultureInfo.InvariantCulture); + } + } + } /// Selected preset; "custom" once a value no longer matches one. [ObservableProperty] @@ -94,7 +114,7 @@ public CompressionSettingsViewModel(bool allowAutoFit, bool offerSkipExecutables [ObservableProperty] public partial decimal? Level { get; set; } = DefaultLevel; - /// --cpu-count (0 = auto). + /// --cpu-count; 0 is Auto, which uses every core of this machine. [ObservableProperty] public partial decimal? CpuCount { get; set; } = 0; @@ -136,7 +156,10 @@ public bool AppendTo(List args, bool compress, out string? error) if (compress) { Add(args, "--compression-level", Whole(Level, DefaultLevel), DefaultLevel); - Add(args, "--cpu-count", Whole(CpuCount, 0), 0); + // The CLI's own default leaves a core free and stops at 16; Auto here means every core. + int cpu = Whole(CpuCount, 0); + args.Add("--cpu-count"); + args.Add((cpu > 0 ? cpu : MaxCpu).ToString(CultureInfo.InvariantCulture)); Add(args, "--threshold-gain", Whole(ThresholdGain, 0), 0); Add(args, "--max-compressed-ratio", Whole(MaxRatio, 100), 100); if (MinCompressSize.Trim().Length > 0) @@ -183,7 +206,11 @@ partial void OnPresetChanged(Choice value) partial void OnLevelChanged(decimal? value) => MatchPreset(); - partial void OnCpuCountChanged(decimal? value) => MatchPreset(); + partial void OnCpuCountChanged(decimal? value) + { + OnPropertyChanged(nameof(CpuChoice)); + MatchPreset(); + } // A hand-edited level or core count selects the preset it equals, or Custom. private void MatchPreset() diff --git a/src/MkPFS.Gui/ViewModels/Icons.cs b/src/MkPFS.Gui/ViewModels/Icons.cs index ae5fba0..88fbfc4 100644 --- a/src/MkPFS.Gui/ViewModels/Icons.cs +++ b/src/MkPFS.Gui/ViewModels/Icons.cs @@ -3,8 +3,6 @@ namespace MkPFS.Gui.ViewModels; /// Navigation icon paths (Material Design Icons 7.4.47, Apache-2.0, 24×24 viewbox). internal static class Icons { - public const string Batch = "M12 16.54L19.37 10.8L21 12.07L12 19.07L3 12.07L4.62 10.81L12 16.54M12 14L3 7L12 0L21 7L12 14M12 2.53L6.26 7L12 11.47L17.74 7L12 2.53M12 21.47L19.37 15.73L21 17L12 24L3 17L4.62 15.74L12 21.47"; - public const string PackFolder = "M20 6H12L10 4H4C2.9 4 2 4.9 2 6V18C2 19.1 2.9 20 4 20H20C21.1 20 22 19.1 22 18V8C22 6.9 21.1 6 20 6M20 18H16V16H14V18H4V8H14V10H16V8H20V18M16 12V10H18V12H16M14 12H16V14H14V12M18 16H16V14H18V16Z"; public const string PackExfat = "M6,2H18A2,2 0 0,1 20,4V20A2,2 0 0,1 18,22H6A2,2 0 0,1 4,20V4A2,2 0 0,1 6,2M12,4A6,6 0 0,0 6,10C6,13.31 8.69,16 12.1,16L11.22,13.77C10.95,13.29 11.11,12.68 11.59,12.4L12.45,11.9C12.93,11.63 13.54,11.79 13.82,12.27L15.74,14.69C17.12,13.59 18,11.9 18,10A6,6 0 0,0 12,4M12,9A1,1 0 0,1 13,10A1,1 0 0,1 12,11A1,1 0 0,1 11,10A1,1 0 0,1 12,9M7,18A1,1 0 0,0 6,19A1,1 0 0,0 7,20A1,1 0 0,0 8,19A1,1 0 0,0 7,18M12.09,13.27L14.58,19.58L17.17,18.08L12.95,12.77L12.09,13.27Z"; public const string PackFile = "M6,2A2,2 0 0,0 4,4V20A2,2 0 0,0 6,22H18A2,2 0 0,0 20,20V8L14,2H6M6,4H13V9H18V20H6V4M8,12V14H16V12H8M8,16V18H13V16H8Z"; public const string Verify = "M21,11C21,16.55 17.16,21.74 12,23C6.84,21.74 3,16.55 3,11V5L12,1L21,5V11M12,21C15.75,20 19,15.54 19,11.22V6.3L12,3.18L5,6.3V11.22C5,15.54 8.25,20 12,21M10,17L6,13L7.41,11.59L10,14.17L16.59,7.58L18,9"; @@ -12,4 +10,5 @@ internal static class Icons public const string Inspect = "M11,9H13V7H11M12,20C7.59,20 4,16.41 4,12C4,7.59 7.59,4 12,4C16.41,4 20,7.59 20,12C20,16.41 16.41,20 12,20M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12A10,10 0 0,0 12,2M11,17H13V11H11V17Z"; public const string Tree = "M12 13H7V18H12V20H5V10H7V11H12V13M8 4V6H4V4H8M10 2H2V8H10V2M20 11V13H16V11H20M22 9H14V15H22V9M20 18V20H16V18H20M22 16H14V22H22V16Z"; public const string Unpack = "M2,10.96C1.5,10.68 1.35,10.07 1.63,9.59L3.13,7C3.24,6.8 3.41,6.66 3.6,6.58L11.43,2.18C11.59,2.06 11.79,2 12,2C12.21,2 12.41,2.06 12.57,2.18L20.47,6.62C20.66,6.72 20.82,6.88 20.91,7.08L22.36,9.6C22.64,10.08 22.47,10.69 22,10.96L21,11.54V16.5C21,16.88 20.79,17.21 20.47,17.38L12.57,21.82C12.41,21.94 12.21,22 12,22C11.79,22 11.59,21.94 11.43,21.82L3.53,17.38C3.21,17.21 3,16.88 3,16.5V10.96C2.7,11.13 2.32,11.14 2,10.96M12,4.15V4.15L12,10.85V10.85L17.96,7.5L12,4.15M5,15.91L11,19.29V12.58L5,9.21V15.91M19,15.91V12.69L14,15.59C13.67,15.77 13.3,15.76 13,15.6V19.29L19,15.91M13.85,13.36L20.13,9.73L19.55,8.72L13.27,12.35L13.85,13.36Z"; + public const string AmprPack = "M21,16.5C21,16.88 20.79,17.21 20.47,17.38L12.57,21.82C12.41,21.94 12.21,22 12,22C11.79,22 11.59,21.94 11.43,21.82L3.53,17.38C3.21,17.21 3,16.88 3,16.5V7.5C3,7.12 3.21,6.79 3.53,6.62L11.43,2.18C11.59,2.06 11.79,2 12,2C12.21,2 12.41,2.06 12.57,2.18L20.47,6.62C20.79,6.79 21,7.12 21,7.5V16.5M12,4.15L10.11,5.22L16,8.61L17.96,7.5L12,4.15M6.04,7.5L12,10.85L13.96,9.75L8.08,6.35L6.04,7.5M5,15.91L11,19.29V12.58L5,9.21V15.91M19,15.91V9.21L13,12.58V19.29L19,15.91Z"; } diff --git a/src/MkPFS.Gui/ViewModels/MainWindowViewModel.cs b/src/MkPFS.Gui/ViewModels/MainWindowViewModel.cs index 3bcd353..e52a7ac 100644 --- a/src/MkPFS.Gui/ViewModels/MainWindowViewModel.cs +++ b/src/MkPFS.Gui/ViewModels/MainWindowViewModel.cs @@ -56,9 +56,7 @@ public sealed record NavSection(string TitleKey, IReadOnlyList Items) /// Main window state: sidebar sections, the current page, and the language picker. public sealed partial class MainWindowViewModel : ObservableObject { - // Python theme.py neon palette; Repair is new and takes the error red. - private static readonly Color Teal = Color.Parse("#00E5A0"); - private static readonly Color Blue = Color.Parse("#00C8FF"); + // Python theme.py neon palette; Repair is new and takes the error red, AMPR Packs a new lime. private static readonly Color Orange = Color.Parse("#FF6B3D"); private static readonly Color Cyan = Color.Parse("#00FFD4"); private static readonly Color Green = Color.Parse("#39FF8A"); @@ -66,6 +64,7 @@ public sealed partial class MainWindowViewModel : ObservableObject private static readonly Color Purple = Color.Parse("#B560FF"); private static readonly Color Amber = Color.Parse("#FFB800"); private static readonly Color Pink = Color.Parse("#FF5CAA"); + private static readonly Color Lime = Color.Parse("#C6FF3D"); /// Create the main window state with every page. public MainWindowViewModel() @@ -74,10 +73,9 @@ public MainWindowViewModel() [ new("section_build", [ - Item("batch", "nav_batch", Icons.Batch, new BatchPanelViewModel(Teal)), - Item("pack_folder", "nav_pack_folder", Icons.PackFolder, new PackFolderPanelViewModel(Blue)), Item("pack_exfat", "nav_pack_exfat", Icons.PackExfat, new PackExfatPanelViewModel(Orange)), Item("pack_file", "nav_pack_file", Icons.PackFile, new PackFilePanelViewModel(Cyan)), + Item("ampr", "nav_ampr", Icons.AmprPack, new AmprPackPanelViewModel(Lime)), ]), new("section_check", [ diff --git a/src/MkPFS.Gui/ViewModels/RepairPanelViewModel.cs b/src/MkPFS.Gui/ViewModels/RepairPanelViewModel.cs index ce39d17..1c7cc1f 100644 --- a/src/MkPFS.Gui/ViewModels/RepairPanelViewModel.cs +++ b/src/MkPFS.Gui/ViewModels/RepairPanelViewModel.cs @@ -125,7 +125,7 @@ protected override async Task ExecuteAsync(IReadOnlyList arg PFSCRepairOptions options = RepairCommand.Options(ctx, scan, badBlocks, recompress, mode, reportDir, cleanSlack, cpuCount: 0, progress: true); int code = RepairCommand.Execute(ctx, image, options, job.Token, r => result = r); return scan && code == RepairCommand.ExitRepairNeeded ? 0 : code; // "repair needed" is a scan result, not a failure - }).ConfigureAwait(true); + }, ProgressPlan.For(args)).ConfigureAwait(true); // After a repair the scanned map is out of date; scan again to see the new state. Selection = null; diff --git a/src/MkPFS.Gui/Views/BatchQueue.axaml b/src/MkPFS.Gui/Views/BatchQueue.axaml deleted file mode 100644 index 7dd2a58..0000000 --- a/src/MkPFS.Gui/Views/BatchQueue.axaml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/MkPFS.Gui/Views/BatchQueue.axaml.cs b/src/MkPFS.Gui/Views/BatchQueue.axaml.cs deleted file mode 100644 index 722ad23..0000000 --- a/src/MkPFS.Gui/Views/BatchQueue.axaml.cs +++ /dev/null @@ -1,13 +0,0 @@ -using Avalonia.Controls; - -namespace MkPFS.Gui.Views; - -/// Batch queue preview list. -public sealed partial class BatchQueue : UserControl -{ - /// Create the view. - public BatchQueue() - { - InitializeComponent(); - } -} diff --git a/src/MkPFS.Gui/Views/CompressionSettings.axaml b/src/MkPFS.Gui/Views/CompressionSettings.axaml index 3a5c34f..d8672b2 100644 --- a/src/MkPFS.Gui/Views/CompressionSettings.axaml +++ b/src/MkPFS.Gui/Views/CompressionSettings.axaml @@ -6,7 +6,7 @@ x:DataType="vm:CompressionSettingsViewModel"> - + @@ -14,16 +14,20 @@ - + - - - + + + + + + + - + @@ -31,20 +35,20 @@ - + - + - + - diff --git a/src/MkPFS.Gui/Views/PFSOptions.axaml b/src/MkPFS.Gui/Views/PFSOptions.axaml index d8e39d6..5fb5c7c 100644 --- a/src/MkPFS.Gui/Views/PFSOptions.axaml +++ b/src/MkPFS.Gui/Views/PFSOptions.axaml @@ -6,7 +6,7 @@ x:DataType="vm:PFSOptionsViewModel"> - + @@ -14,7 +14,7 @@ - + @@ -24,13 +24,13 @@ - - - - + + + - + diff --git a/src/MkPFS.Gui/Views/PanelView.axaml b/src/MkPFS.Gui/Views/PanelView.axaml index 20d0739..c67c5f6 100644 --- a/src/MkPFS.Gui/Views/PanelView.axaml +++ b/src/MkPFS.Gui/Views/PanelView.axaml @@ -21,8 +21,6 @@ page template on MainWindow, which also matches it. --> - - @@ -30,6 +28,7 @@ + diff --git a/src/MkPFS.Gui/Views/Panels/AmprPackForm.axaml b/src/MkPFS.Gui/Views/Panels/AmprPackForm.axaml new file mode 100644 index 0000000..7fe6d3a --- /dev/null +++ b/src/MkPFS.Gui/Views/Panels/AmprPackForm.axaml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/MkPFS.Gui/Views/Panels/BatchForm.axaml.cs b/src/MkPFS.Gui/Views/Panels/AmprPackForm.axaml.cs similarity index 73% rename from src/MkPFS.Gui/Views/Panels/BatchForm.axaml.cs rename to src/MkPFS.Gui/Views/Panels/AmprPackForm.axaml.cs index e570ba2..a4dfdf7 100644 --- a/src/MkPFS.Gui/Views/Panels/BatchForm.axaml.cs +++ b/src/MkPFS.Gui/Views/Panels/AmprPackForm.axaml.cs @@ -3,10 +3,10 @@ namespace MkPFS.Gui.Views.Panels; /// Options form of the page with the same name. -public sealed partial class BatchForm : UserControl +public sealed partial class AmprPackForm : UserControl { /// Create the form. - public BatchForm() + public AmprPackForm() { InitializeComponent(); } diff --git a/src/MkPFS.Gui/Views/Panels/BatchForm.axaml b/src/MkPFS.Gui/Views/Panels/BatchForm.axaml deleted file mode 100644 index a07a32e..0000000 --- a/src/MkPFS.Gui/Views/Panels/BatchForm.axaml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/MkPFS.Gui/Views/Panels/PackExfatForm.axaml b/src/MkPFS.Gui/Views/Panels/PackExfatForm.axaml index 2d57fe4..f5ea36c 100644 --- a/src/MkPFS.Gui/Views/Panels/PackExfatForm.axaml +++ b/src/MkPFS.Gui/Views/Panels/PackExfatForm.axaml @@ -23,7 +23,7 @@ - + @@ -31,7 +31,7 @@ - + diff --git a/src/MkPFS.Gui/Views/Panels/PackFileForm.axaml b/src/MkPFS.Gui/Views/Panels/PackFileForm.axaml index 3600c2a..17938d8 100644 --- a/src/MkPFS.Gui/Views/Panels/PackFileForm.axaml +++ b/src/MkPFS.Gui/Views/Panels/PackFileForm.axaml @@ -24,13 +24,13 @@ - - - - - - - + + + + + + + diff --git a/src/MkPFS.Gui/Views/Panels/PackFolderForm.axaml b/src/MkPFS.Gui/Views/Panels/PackFolderForm.axaml deleted file mode 100644 index b8f5b90..0000000 --- a/src/MkPFS.Gui/Views/Panels/PackFolderForm.axaml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/MkPFS.Gui/Views/Panels/PackFolderForm.axaml.cs b/src/MkPFS.Gui/Views/Panels/PackFolderForm.axaml.cs deleted file mode 100644 index ca8c2aa..0000000 --- a/src/MkPFS.Gui/Views/Panels/PackFolderForm.axaml.cs +++ /dev/null @@ -1,13 +0,0 @@ -using Avalonia.Controls; - -namespace MkPFS.Gui.Views.Panels; - -/// Options form of the page with the same name. -public sealed partial class PackFolderForm : UserControl -{ - /// Create the form. - public PackFolderForm() - { - InitializeComponent(); - } -} diff --git a/src/MkPFS.Gui/Views/Panels/TreeForm.axaml b/src/MkPFS.Gui/Views/Panels/TreeForm.axaml index 096ee94..a9ba853 100644 --- a/src/MkPFS.Gui/Views/Panels/TreeForm.axaml +++ b/src/MkPFS.Gui/Views/Panels/TreeForm.axaml @@ -23,7 +23,7 @@ - + diff --git a/src/MkPFS.Gui/Views/Panels/UnpackForm.axaml b/src/MkPFS.Gui/Views/Panels/UnpackForm.axaml index d9ef484..c58d581 100644 --- a/src/MkPFS.Gui/Views/Panels/UnpackForm.axaml +++ b/src/MkPFS.Gui/Views/Panels/UnpackForm.axaml @@ -31,7 +31,7 @@ - + diff --git a/src/MkPFS.Gui/Views/Panels/VerifyForm.axaml b/src/MkPFS.Gui/Views/Panels/VerifyForm.axaml index 2a1ae27..b9f7afa 100644 --- a/src/MkPFS.Gui/Views/Panels/VerifyForm.axaml +++ b/src/MkPFS.Gui/Views/Panels/VerifyForm.axaml @@ -34,7 +34,7 @@ - + @@ -42,7 +42,7 @@ - + diff --git a/tests/MkPFS.Gui.Tests/AmprPackPanelTests.cs b/tests/MkPFS.Gui.Tests/AmprPackPanelTests.cs new file mode 100644 index 0000000..869fa41 --- /dev/null +++ b/tests/MkPFS.Gui.Tests/AmprPackPanelTests.cs @@ -0,0 +1,296 @@ +using Avalonia.Headless.XUnit; +using Avalonia.Media; +using MkPFS.Build; +using MkPFS.Gui.Jobs; +using MkPFS.Gui.ViewModels; + +namespace MkPFS.Gui.Tests; + +[Collection(GuiCollection.Name)] +public sealed class AmprPackPanelTests +{ + private static JobRunner Sync() => new(action => action()); + + private static AmprPackPanelViewModel Panel(string action) + { + AmprPackPanelViewModel panel = new(Colors.YellowGreen, Sync()); + panel.Action = panel.Actions.Single(a => a.Value == action); + return panel; + } + + [Fact] + public void Pack_builds_arguments_and_validates() + { + AmprPackPanelViewModel panel = Panel("pack"); + Assert.Null(panel.BuildArguments(out string? error)); + Assert.Equal("✗ Game folder and output folder are both required.", error); + + panel.Root = "D:/game"; + panel.Output = "D:/out"; + Assert.Equal(Path.Combine("D:/out", "ampr_assets.index"), panel.Manifest); + Assert.Null(panel.BuildArguments(out error)); + Assert.Equal("✗ Choose a TOML profile or a trace folder: without rules nothing is packed.", error); + panel.Config = "D:/rules.toml"; + Assert.Equal( + ["ampr", "pack", "--root", "D:/game", "--ampr-index", Path.Combine("D:/game", "ampr_emu.index"), "--output", "D:/out", "--config", "D:/rules.toml"], + panel.BuildArguments(out _)); + + panel.AmprIndex = "D:/idx/ampr_emu.index"; + panel.Config = "D:/rules.toml"; + panel.Workers = "4"; + panel.SelfContained = true; + panel.AllowMissing = true; + Assert.Equal( + ["ampr", "pack", "--root", "D:/game", "--ampr-index", "D:/idx/ampr_emu.index", "--output", "D:/out", "--config", "D:/rules.toml", + "--workers", "4", "--self-contained", "--allow-missing"], + panel.BuildArguments(out _)); + + panel.Workers = "0"; + Assert.Null(panel.BuildArguments(out error)); + Assert.Equal("✗ Workers must be a whole number from 1 to 256.", error); + } + + [Fact] + public void Game_is_the_default_action_and_builds_arguments() + { + AmprPackPanelViewModel panel = new(Colors.YellowGreen, Sync()); + Assert.Equal("game", panel.Action.Value); + Assert.Null(panel.BuildArguments(out string? error)); + Assert.Equal("✗ Game folder and output folder are both required.", error); + + panel.Root = "D:/game"; + panel.Output = "D:/games/out"; + Assert.Equal(Path.Combine("D:/games/out", "ampr_assets.index"), panel.Manifest); + Assert.Null(panel.BuildArguments(out error)); + Assert.Equal("✗ Choose a TOML profile or a trace folder: without rules nothing is packed.", error); + panel.Config = "D:/rules.toml"; + Assert.Equal(["ampr", "game", "--root", "D:/game", "--output", "D:/games/out", "--config", "D:/rules.toml"], panel.BuildArguments(out _)); + + panel.Libs = "D:/emu"; + panel.Workers = "4"; + panel.SelfContained = true; + panel.VerifyGame = false; + panel.Exfat = true; + Assert.Equal( + ["ampr", "game", "--root", "D:/game", "--output", "D:/games/out", "--fakelib", "D:/emu", "--config", "D:/rules.toml", "--workers", "4", + "--self-contained", "--skip-verify", "--exfat", Path.GetDirectoryName(Path.GetFullPath("D:/games/out"))!], + panel.BuildArguments(out _)); + + panel.ExfatPath = "E:/PPSA01234.exfat"; + Assert.Equal(["--exfat", "E:/PPSA01234.exfat"], panel.BuildArguments(out _)!.TakeLast(2)); + } + + [Fact] + public void Other_actions_build_arguments_and_validate() + { + AmprPackPanelViewModel verify = Panel("verify"); + Assert.Null(verify.BuildArguments(out string? error)); + Assert.Equal("✗ Pack manifest is required.", error); + verify.Manifest = "D:/out/ampr_assets.index"; + Assert.Equal(["ampr", "verify", "--index", "D:/out/ampr_assets.index"], verify.BuildArguments(out _)); + verify.Root = "D:/game"; + Assert.Equal(["ampr", "verify", "--index", "D:/out/ampr_assets.index", "--root", "D:/game"], verify.BuildArguments(out _)); + + AmprPackPanelViewModel unpack = Panel("unpack"); + unpack.Manifest = "m.index"; + Assert.Null(unpack.BuildArguments(out error)); + Assert.Equal("✗ Pack manifest and output folder are both required.", error); + unpack.Output = "D:/x"; + unpack.Overwrite = true; + Assert.Equal(["ampr", "unpack", "--index", "m.index", "--output", "D:/x", "--overwrite"], unpack.BuildArguments(out _)); + + AmprPackPanelViewModel list = Panel("list"); + list.Manifest = "m.index"; + list.Json = true; + Assert.Equal(["ampr", "list", "--index", "m.index", "--json"], list.BuildArguments(out _)); + Assert.Equal(["ampr", "inspect", "--index", "m.index"], Panel("inspect") is var inspect && (inspect.Manifest = "m.index") is not null ? inspect.BuildArguments(out _) : null); + + AmprPackPanelViewModel runtime = Panel("runtime-config"); + runtime.Manifest = "m.index"; + Assert.Null(runtime.BuildArguments(out error)); + Assert.Equal("✗ Pack manifest and TOML configuration are both required.", error); + runtime.Config = "r.toml"; + Assert.Equal(["ampr", "runtime-config", "--index", "m.index", "--config", "r.toml"], runtime.BuildArguments(out _)); + + AmprPackPanelViewModel remove = Panel("remove-sources"); + remove.Manifest = "m.index"; + Assert.Null(remove.BuildArguments(out error)); + Assert.Equal("✗ Pack manifest and game folder are both required.", error); + remove.Root = "D:/game"; + Assert.Equal(["ampr", "remove-sources", "--index", "m.index", "--root", "D:/game"], remove.BuildArguments(out _)); + remove.Confirm = true; + Assert.Equal(["ampr", "remove-sources", "--index", "m.index", "--root", "D:/game", "--confirm"], remove.BuildArguments(out _)); + } + + [Theory] + [InlineData("game", true, false, true, false, true, true, false, false, false)] + [InlineData("pack", true, true, true, false, true, true, false, false, false)] + [InlineData("verify", true, false, false, true, false, false, false, false, false)] + [InlineData("unpack", false, false, true, true, false, false, true, false, false)] + [InlineData("list", false, false, false, true, false, false, false, true, false)] + [InlineData("inspect", false, false, false, true, false, false, false, false, false)] + [InlineData("runtime-config", false, false, false, true, true, false, false, false, false)] + [InlineData("remove-sources", true, false, false, true, false, false, false, false, true)] + public void Each_action_shows_only_its_fields( + string action, bool root, bool packPaths, bool output, bool manifest, bool config, bool packOptions, bool overwrite, bool json, bool confirm) + { + AmprPackPanelViewModel panel = Panel(action); + Assert.Equal( + (root, packPaths, output, manifest, config, packOptions, overwrite, json, confirm), + (panel.ShowRoot, panel.ShowPackPaths, panel.ShowOutput, panel.ShowManifest, panel.ShowConfig, panel.ShowPackOptions, + panel.ShowOverwrite, panel.ShowJson, panel.ShowConfirm)); + Assert.Equal(action == "game", panel.ShowGameOptions); + Assert.Equal(action == "pack", panel.ShowAllowMissing); + Assert.Equal(action == "pack", panel.ShowPackNote); + Assert.False(panel.ShowExfatPath); + panel.Exfat = true; + Assert.Equal(action == "game", panel.ShowExfatPath); + } + + [Fact] + public void Page_asks_for_a_profile_or_traces() + { + AmprPackPanelViewModel panel = Panel("game"); + Assert.True(panel.ShowRulesMissing); + Assert.False(panel.ShowRulesConfig); + + panel.Config = "rules.toml"; + Assert.False(panel.ShowRulesMissing); + Assert.True(panel.ShowRulesConfig); + + panel.Config = string.Empty; + panel.Traces = "traces"; + Assert.False(panel.ShowRulesMissing); + + panel.Action = panel.Actions.Single(a => a.Value == "verify"); + panel.Traces = string.Empty; + Assert.False(panel.ShowRulesMissing || panel.ShowRulesConfig); + } + + [Fact] + public async Task Page_takes_rules_from_a_trace_folder() + { + using TempDir dir = new(); + AmprPackPanelViewModel panel = Panel("game"); + panel.TraceCountDebounce = TimeSpan.Zero; + panel.Root = Directory.CreateDirectory(Path.Combine(dir.Path, "game")).FullName; + panel.Output = Path.Combine(dir.Path, "out"); + string traces = Directory.CreateDirectory(Path.Combine(dir.Path, "traces")).FullName; + + panel.Traces = traces; + await panel.TraceCount; + Assert.True(panel.ShowNoTraces); + Assert.False(panel.ShowRulesTraces || panel.ShowRulesMissing); + + // Runs copied in after the folder was picked are found when the page runs; hidden files count. + dir.File("traces/startup/ampr_commands.bin"); + string index = dir.File("traces/startup/ampr_emu.index"); + File.SetAttributes(index, FileAttributes.Hidden); + Assert.NotNull(panel.BuildArguments(out _)); + Assert.True(panel.ShowRulesTraces); + Assert.StartsWith("✓ Rules from traces: 1 recorded run(s) found.", panel.RulesTracesText, StringComparison.Ordinal); + + dir.File("traces/level1/ampr_commands.bin"); + dir.File("traces/level1/ampr_emu.index"); + panel.Traces = traces + Path.DirectorySeparatorChar; + await panel.TraceCount; + Assert.True(panel.ShowRulesTraces); + Assert.False(panel.ShowNoTraces); + Assert.StartsWith("✓ Rules from traces: 2 recorded run(s) found.", panel.RulesTracesText, StringComparison.Ordinal); + Assert.Equal(["--traces", traces + Path.DirectorySeparatorChar], panel.BuildArguments(out _)!.SkipWhile(a => a != "--traces").Take(2)); + + Assert.DoesNotContain("--pack-untraced-types", panel.BuildArguments(out _)!); + panel.UntracedTypes = true; + Assert.Contains("--pack-untraced-types", panel.BuildArguments(out _)!); + + panel.Action = panel.Actions.Single(a => a.Value == "pack"); + Assert.Equal(["--traces", traces + Path.DirectorySeparatorChar], panel.BuildArguments(out _)!.SkipWhile(a => a != "--traces").Take(2)); + Assert.Contains("--pack-untraced-types", panel.BuildArguments(out _)!); + + panel.Config = "rules.toml"; + Assert.False(panel.ShowRulesTraces || panel.ShowRulesConfig); + Assert.Null(panel.BuildArguments(out string? error)); + Assert.Equal("Use either a TOML file or a trace folder, not both.", error); + } + + // Compress every file but executables, modules, system files and the libraries. + private static string CompressAll(TempDir dir) => dir.File( + "compress-all.toml", + "[pack]\ndefault_action = \"compress\"\n[[rule]]\naction = \"loose\"\n" + + "include = [\"eboot.bin\", \"*.prx\", \"*.sprx\", \"sce_sys/*\", \"sce_module/*\", \"fakelib/*\", \"ampr_emu.index\"]\n"); + + [AvaloniaFact] + public async Task Page_packs_with_a_toml_profile_and_logs_cli_progress() + { + using TempDir dir = new(); + string game = BuildPanelTests.Game(dir); + dir.File("PPSA01234-app/data/level.dat", string.Concat(Enumerable.Repeat("terrain mesh texture ", 20_000))); + dir.File("PPSA01234-app/sce_module/libc.prx", "module"); + AmprIndex.Build(game, Path.Combine(game, AmprIndex.IndexName)); + + AmprPackPanelViewModel panel = Panel("pack"); + panel.Root = game; + panel.Output = Path.Combine(dir.Path, "packs"); + panel.Config = CompressAll(dir); + await panel.RunCommand.ExecuteAsync(null); + + Assert.Equal("✓ Completed successfully.", panel.Job.Lines[^1].Text); + Assert.Contains(panel.Job.Lines, l => l.Text.Contains("\"files_packed\": 1", StringComparison.Ordinal)); + Assert.Contains(panel.Job.Lines, l => l.Text.StartsWith("[pack 100%] complete: ", StringComparison.Ordinal)); + // One overall phase: no per-phase "✓ planning: N%" lines that look like finished steps. + Assert.DoesNotContain(panel.Job.Lines, l => l.Text.StartsWith("✓ planning", StringComparison.Ordinal) || l.Text.StartsWith("✓ packing", StringComparison.Ordinal)); + } + + [AvaloniaFact] + public async Task Page_builds_a_playable_game_folder() + { + using TempDir dir = new(); + string game = BuildPanelTests.Game(dir); + dir.File("PPSA01234-app/data/level.dat", string.Concat(Enumerable.Repeat("terrain mesh texture ", 20_000))); + dir.File("emu/libSceAmpr.sprx", "AMPR Emu with AMPRPAK4 support"); + + AmprPackPanelViewModel panel = new(Colors.YellowGreen, Sync()); + panel.Root = game; + panel.Output = Path.Combine(dir.Path, "playable"); + panel.Libs = Path.Combine(dir.Path, "emu"); + panel.Config = CompressAll(dir); + await panel.RunCommand.ExecuteAsync(null); + + Assert.True(panel.Job.Lines[^1].Text == "✓ Completed successfully.", string.Join('\n', panel.Job.Lines.Select(l => l.Text))); + Assert.Contains(panel.Job.Lines, l => l.Text == "[5/5] Verifying"); + Assert.True(File.Exists(Path.Combine(panel.Output, "fakelib", "libSceAmpr.sprx"))); + Assert.True(File.Exists(Path.Combine(panel.Output, "ampr_emu.index"))); + Assert.True(File.Exists(panel.Manifest)); + Assert.False(File.Exists(Path.Combine(panel.Output, "data", "level.dat"))); + } + + [AvaloniaFact] + public async Task Page_packs_verifies_and_plans_removal_for_a_real_folder() + { + using TempDir dir = new(); + string game = BuildPanelTests.Game(dir); + dir.File("PPSA01234-app/data/level.dat", string.Concat(Enumerable.Repeat("terrain mesh texture ", 20_000))); + AmprIndex.Build(game, Path.Combine(game, AmprIndex.IndexName)); + string config = dir.File("rules.toml", "[pack]\ndefault_action = \"loose\"\n[[rule]]\ninclude = \"data/**\"\n"); + + AmprPackPanelViewModel panel = Panel("pack"); + panel.Root = game; + panel.Output = Path.Combine(dir.Path, "packs"); + panel.Config = config; + await panel.RunCommand.ExecuteAsync(null); + Assert.Equal("✓ Completed successfully.", panel.Job.Lines[^1].Text); + Assert.Contains(panel.Job.Lines, l => l.Text.Contains("\"files_packed\": 1", StringComparison.Ordinal)); + Assert.True(File.Exists(panel.Manifest)); + + panel.Action = panel.Actions.Single(a => a.Value == "verify"); + await panel.RunCommand.ExecuteAsync(null); + Assert.Equal("✓ Completed successfully.", panel.Job.Lines[^1].Text); + Assert.Contains(panel.Job.Lines, l => l.Text.Contains("\"source_compare\"", StringComparison.Ordinal)); + + panel.Action = panel.Actions.Single(a => a.Value == "remove-sources"); + await panel.RunCommand.ExecuteAsync(null); + Assert.Equal("✓ Completed successfully.", panel.Job.Lines[^1].Text); + Assert.Contains(panel.Job.Lines, l => l.Text.Contains("\"dry_run\": true", StringComparison.Ordinal)); + Assert.True(File.Exists(Path.Combine(game, "data", "level.dat"))); + } +} diff --git a/tests/MkPFS.Gui.Tests/BuildPanelTests.cs b/tests/MkPFS.Gui.Tests/BuildPanelTests.cs index 69d6fb5..4490629 100644 --- a/tests/MkPFS.Gui.Tests/BuildPanelTests.cs +++ b/tests/MkPFS.Gui.Tests/BuildPanelTests.cs @@ -23,28 +23,6 @@ internal static string Game(TempDir dir, string name = "PPSA01234-app") return Path.Combine(dir.Path, name); } - [Fact] - public void Pack_folder_builds_python_arguments_and_validates() - { - PackFolderPanelViewModel panel = new(Colors.Blue, Sync()); - - Assert.Null(Build(panel, out string? error)); - Assert.Equal("✗ Source folder is required.", error); - panel.Source = "D:/games/x"; - Assert.Null(Build(panel, out error)); - Assert.Equal("✗ Output image path is required.", error); - - panel.Output = " D:/out/x.ffpfsc "; - Assert.Equal(["pack", "folder", "D:/games/x", "D:/out/x.ffpfsc"], Build(panel, out _)); - panel.Compress = false; - panel.Signed = panel.VerifyAfter = panel.DryRun = true; - panel.TempFolder = "T:/tmp"; - panel.Ampr.LibsDir = "L:/ampr"; - Assert.Equal( - ["pack", "folder", "D:/games/x", "D:/out/x.ffpfsc", "--no-compress", "--signed", "--verify", "--dry-run", "--temp-folder", "T:/tmp", "--ampr-libs", "L:/ampr"], - Build(panel, out _)); - } - [Fact] public void Pack_exfat_output_is_optional() { @@ -61,7 +39,7 @@ public void Pack_exfat_output_is_optional() } [Fact] - public void Pack_file_and_batch_build_python_arguments() + public void Pack_file_builds_python_arguments() { PackFilePanelViewModel file = new(Colors.Cyan, Sync()) { Source = "D:/a.exfat" }; Assert.Null(Build(file, out string? error)); @@ -70,36 +48,11 @@ public void Pack_file_and_batch_build_python_arguments() file.Compress = false; file.TempFolder = "T:/"; Assert.Equal(["pack", "file", "D:/a.exfat", "D:/a.ffpfsc", "--no-compress", "--temp-folder", "T:/"], Build(file, out _)); - - BatchPanelViewModel batch = new(Colors.Teal, Sync()) { Output = "D:/out" }; - Assert.Null(Build(batch, out error)); - Assert.Equal("✗ Source folder is required.", error); - batch.Source = "D:/in"; - batch.Compress = false; - batch.Overwrite = batch.DryRun = batch.VerifyAfter = true; - batch.Ampr.LibsDir = "L:/ampr"; - Assert.Equal(["batch", "D:/in", "D:/out", "--no-compress", "--overwrite", "--dry-run", "--verify", "--ampr-libs", "L:/ampr"], Build(batch, out _)); } [Fact] public void Advanced_options_map_to_cli_flags() { - PackFolderPanelViewModel folder = new(Colors.Blue, Sync()) { Source = "D:/g", Output = "D:/g.ffpfs" }; - folder.PFS.InodeBits = folder.PFS.InodeWidths[1]; - folder.PFS.Version = folder.PFS.Versions[1]; - folder.PFS.Encrypted = folder.PFS.CaseSensitive = folder.PFS.Verbose = true; - folder.PFS.EkpfsKey = " ab "; - folder.RequireGameFiles = folder.SkipVerification = folder.KeepExtension = true; - Assert.Equal( - ["pack", "folder", "D:/g", "D:/g.ffpfs", "--require-game-files", "--skip-verification", "--no-adjust-output-file-extension", - "--version", "PS4", "--case-sensitive", "--encrypted", "--ekpfs-key", "ab", "--verbose"], - Build(folder, out _)); // 64-bit inodes need --raw - folder.Raw = folder.VerifyAfter = true; - Assert.Equal( - ["pack", "folder", "D:/g", "D:/g.ffpfs", "--verify", "--raw", "--require-game-files", "--no-adjust-output-file-extension", - "--version", "PS4", "--inode-bits", "64", "--case-sensitive", "--encrypted", "--ekpfs-key", "ab", "--verbose"], - Build(folder, out _)); // --skip-verification conflicts with --verify - PackExfatPanelViewModel exfat = new(Colors.Orange, Sync()) { Source = "D:/g", Output = "D:/g.exfat" }; Assert.Equal("auto", exfat.ClusterSize.Value); exfat.ClusterSize = exfat.ClusterSizes.Single(c => c.Label == "32 MiB"); @@ -110,15 +63,8 @@ public void Advanced_options_map_to_cli_flags() file.Signed = file.DryRun = file.UseSpool = file.KeepInnerName = true; file.PFS.InodeBits = file.PFS.InodeWidths[1]; Assert.Equal( - ["pack", "file", "D:/a.exfat", "D:/a.ffpfsc", "--signed", "--dry-run", "--use-spool", "--no-rename-inner-image", "--inode-bits", "64"], + ["pack", "file", "D:/a.exfat", "D:/a.ffpfsc", "--signed", "--dry-run", "--use-spool", "--no-rename-inner-image", "--inode-bits", "64", .. CompressionSettingsTests.AutoCpu], Build(file, out _)); - - BatchPanelViewModel batch = new(Colors.Teal, Sync()) { Source = "D:/in", Output = "D:/out" }; - batch.PFS.NewCrypt = true; // only with --encrypted - Assert.Equal(["batch", "D:/in", "D:/out"], Build(batch, out _)); - batch.PFS.Encrypted = true; - batch.PFS.InodeBits = batch.PFS.InodeWidths[1]; // not offered by batch - Assert.Equal(["batch", "D:/in", "D:/out", "--encrypted", "--new-crypt"], Build(batch, out _)); } [Fact] @@ -155,16 +101,11 @@ public void A_source_with_an_index_keeps_it_by_default() exfat.Source = plain; // the automatic choice follows the source Assert.False(exfat.Ampr.HasExistingIndex); Assert.False(exfat.Ampr.KeepValidIndex); - - PackFolderPanelViewModel folder = new(Colors.Blue, Sync()); - folder.Ampr.KeepValidIndex = true; // the user's choice stays - folder.Source = indexed; - folder.Source = plain; - Assert.True(folder.Ampr.KeepValidIndex); - - BatchPanelViewModel batch = new(Colors.Teal, Sync()) { Source = dir.Path }; - Assert.True(batch.Ampr.HasExistingIndex); - Assert.True(batch.Ampr.KeepValidIndex); + PackExfatPanelViewModel chosen = new(Colors.Orange, Sync()); + chosen.Ampr.KeepValidIndex = true; // the user's choice stays + chosen.Source = indexed; + chosen.Source = plain; + Assert.True(chosen.Ampr.KeepValidIndex); } [AvaloniaFact] @@ -174,16 +115,14 @@ public void Choosing_a_source_suggests_a_sanitized_output_once() string game = Game(dir, "My Game (EU)!"); string exfat = dir.File("Data [v2].exfat"); - PackFolderPanelViewModel folder = new(Colors.Blue, Sync()) { Source = game }; - Assert.Equal(Path.Combine(dir.Path, "My Game EU.ffpfsc"), folder.Output); - folder.Source = Path.Combine(dir.Path, "other"); - Assert.Equal(Path.Combine(dir.Path, "My Game EU.ffpfsc"), folder.Output); // a filled output is kept + PackExfatPanelViewModel image = new(Colors.Orange, Sync()) { Source = game + Path.DirectorySeparatorChar }; + Assert.Equal(Path.Combine(dir.Path, "My Game EU.exfat"), image.Output); + image.Source = Path.Combine(dir.Path, "other"); + Assert.Equal(Path.Combine(dir.Path, "My Game EU.exfat"), image.Output); // a filled output is kept - Assert.Equal(Path.Combine(dir.Path, "My Game EU.exfat"), new PackExfatPanelViewModel(Colors.Orange, Sync()) { Source = game + Path.DirectorySeparatorChar }.Output); Assert.Equal(Path.Combine(dir.Path, "Data v2.ffpfsc"), new PackFilePanelViewModel(Colors.Cyan, Sync()) { Source = exfat }.Output); Assert.Equal(string.Empty, new PackFilePanelViewModel(Colors.Cyan, Sync()) { Source = game }.Output); // folder: not a file - Assert.Equal(dir.Path, new BatchPanelViewModel(Colors.Teal, Sync()) { Source = dir.Path }.Output); - Assert.Equal(string.Empty, new PackFolderPanelViewModel(Colors.Blue, Sync()) { Source = Path.Combine(dir.Path, "missing") }.Output); + Assert.Equal(string.Empty, new PackExfatPanelViewModel(Colors.Orange, Sync()) { Source = Path.Combine(dir.Path, "missing") }.Output); } [Fact] diff --git a/tests/MkPFS.Gui.Tests/CompressionSettingsTests.cs b/tests/MkPFS.Gui.Tests/CompressionSettingsTests.cs index 0406715..a0596ea 100644 --- a/tests/MkPFS.Gui.Tests/CompressionSettingsTests.cs +++ b/tests/MkPFS.Gui.Tests/CompressionSettingsTests.cs @@ -7,6 +7,9 @@ namespace MkPFS.Gui.Tests; [Collection(GuiCollection.Name)] public sealed class CompressionSettingsTests { + // Auto passes every core of this machine (the CLI's own default leaves one free and stops at 16). + internal static readonly string[] AutoCpu = ["--cpu-count", MkPFS.Core.Util.CpuTopology.PhysicalCores.ToString(System.Globalization.CultureInfo.InvariantCulture)]; + private static List Args(CompressionSettingsViewModel settings, bool compress = true) { List args = []; @@ -15,26 +18,43 @@ private static List Args(CompressionSettingsViewModel settings, bool com } [Fact] - public void Defaults_add_no_arguments() + public void Defaults_add_only_the_core_count() { CompressionSettingsViewModel settings = new(allowAutoFit: true, offerSkipExecutables: true); Assert.Equal("balanced", settings.Preset.Value); - Assert.Empty(Args(settings)); + Assert.Equal(AutoCpu, Args(settings)); + } + + [Fact] + public void Cpu_picker_offers_auto_then_every_core() + { + CompressionSettingsViewModel settings = new(allowAutoFit: true, offerSkipExecutables: true); + + Assert.Equal(MkPFS.Core.Util.CpuTopology.PhysicalCores + 1, settings.CpuChoices.Count); + Assert.Equal($"Auto ({MkPFS.Core.Util.CpuTopology.PhysicalCores} cores)", settings.CpuChoices[0].Label); + Assert.Equal("1", settings.CpuChoices[1].Label); + Assert.Same(settings.CpuChoices[0], settings.CpuChoice); + + settings.CpuChoice = settings.CpuChoices[1]; + Assert.Equal(1, settings.CpuCount); + Assert.Equal("low_ram", settings.Preset.Value); + settings.Preset = settings.Presets.Single(p => p.Value == "balanced"); + Assert.Same(settings.CpuChoices[0], settings.CpuChoice); } [Theory] - [InlineData("fast", new[] { "--compression-level", "1" })] - [InlineData("balanced", new string[0])] - [InlineData("max", new[] { "--compression-level", "9" })] - [InlineData("low_ram", new[] { "--cpu-count", "1" })] - public void Presets_set_level_and_cores(string preset, string[] expected) + [InlineData("fast", new[] { "--compression-level", "1" }, true)] + [InlineData("balanced", new string[0], true)] + [InlineData("max", new[] { "--compression-level", "9" }, true)] + [InlineData("low_ram", new[] { "--cpu-count", "1" }, false)] + public void Presets_set_level_and_cores(string preset, string[] expected, bool autoCpu) { CompressionSettingsViewModel settings = new(allowAutoFit: true, offerSkipExecutables: true); settings.Preset = settings.Presets.Single(p => p.Value == preset); - Assert.Equal(expected, Args(settings)); + Assert.Equal(autoCpu ? [.. expected, .. AutoCpu] : expected, Args(settings)); } [Fact] @@ -65,7 +85,7 @@ public void Block_size_and_keep_rules_become_arguments() settings.BlockSize = settings.BlockSizes.Single(b => b.Value == "auto-fit"); Assert.Equal( - ["--threshold-gain", "5", "--max-compressed-ratio", "90", "--min-compress-size", "32768", "--skip-executable-compression", "--block-size", "auto-fit"], + [.. AutoCpu, "--threshold-gain", "5", "--max-compressed-ratio", "90", "--min-compress-size", "32768", "--skip-executable-compression", "--block-size", "auto-fit"], Args(settings)); // Without compression only the block size still applies. @@ -73,7 +93,7 @@ public void Block_size_and_keep_rules_become_arguments() } [Fact] - public void Batch_offers_no_auto_fit_and_files_no_executables_option() + public void Auto_fit_and_the_executables_option_can_be_left_out() { CompressionSettingsViewModel batch = new(allowAutoFit: false, offerSkipExecutables: true); CompressionSettingsViewModel file = new(allowAutoFit: true, offerSkipExecutables: false) { SkipExecutables = true }; @@ -81,7 +101,7 @@ public void Batch_offers_no_auto_fit_and_files_no_executables_option() Assert.DoesNotContain(batch.BlockSizes, b => b.Value == "auto-fit"); Assert.Equal(["auto", "4096", "8192", "16384", "32768", "65536", "131072", "262144", "524288", "1048576", "2097152"], batch.BlockSizes.Select(b => b.Value)); Assert.Equal("64 KiB", batch.BlockSizes.Single(b => b.Value == "65536").Label); - Assert.Empty(Args(file)); + Assert.Equal(AutoCpu, Args(file)); } [Fact] @@ -99,15 +119,13 @@ public void Pages_append_the_tuning_to_their_arguments() JobRunner sync = new(action => action()); PackFilePanelViewModel file = new(Colors.Cyan, sync) { Source = "D:/a.exfat", Output = "D:/a.ffpfsc" }; file.Compression.Preset = file.Compression.Presets.Single(p => p.Value == "fast"); - Assert.Equal(["pack", "file", "D:/a.exfat", "D:/a.ffpfsc", "--compression-level", "1"], file.BuildArguments(out _)); + Assert.Equal(["pack", "file", "D:/a.exfat", "D:/a.ffpfsc", "--compression-level", "1", .. AutoCpu], file.BuildArguments(out _)); - BatchPanelViewModel batch = new(Colors.Teal, sync) { Source = "D:/in", Output = "D:/out" }; - batch.Compression.CpuCount = 2; - Assert.Equal(["batch", "D:/in", "D:/out", "--cpu-count", "2"], batch.BuildArguments(out _)); + file.Compression.CpuCount = 2; + Assert.Equal(["pack", "file", "D:/a.exfat", "D:/a.ffpfsc", "--compression-level", "1", "--cpu-count", "2"], file.BuildArguments(out _)); - PackFolderPanelViewModel folder = new(Colors.Blue, sync) { Source = "D:/g", Output = "D:/g.ffpfsc" }; - folder.Compression.MinCompressSize = "x"; - Assert.Null(folder.BuildArguments(out string? error)); + file.Compression.MinCompressSize = "x"; + Assert.Null(file.BuildArguments(out string? error)); Assert.Equal("✗ Store Files Smaller Than must be a whole number of bytes.", error); } } diff --git a/tests/MkPFS.Gui.Tests/JobRunnerTests.cs b/tests/MkPFS.Gui.Tests/JobRunnerTests.cs index d5454e4..43c8d74 100644 --- a/tests/MkPFS.Gui.Tests/JobRunnerTests.cs +++ b/tests/MkPFS.Gui.Tests/JobRunnerTests.cs @@ -52,6 +52,53 @@ public async Task Phase_changes_log_the_finished_phase_and_the_bar_ends_full() Assert.False(runner.IsIndeterminate); } + [Fact] + public async Task With_a_plan_one_bar_covers_every_phase_and_never_moves_back() + { + JobRunner runner = Runner(); + List seen = []; + runner.PropertyChanged += (_, e) => + { + if (e.PropertyName == nameof(JobRunner.Progress)) + { + seen.Add(runner.Progress); + } + }; + List labels = []; + IReadOnlyList plan = ProgressPlan.For(["pack", "file", "a.exfat", "a.ffpfsc", "--verify"])!; + + await runner.RunAsync( + job => + { + job.Progress.Report("compress", 30, 60); + labels.Add(runner.PhaseText); + job.Progress.Report("compress", 60, 60); + job.Progress.Report("verify", 0, 10); // a new phase starts at 0%, the bar does not + labels.Add(runner.PhaseText); + job.Progress.Report("verify", 10, 10); + job.Progress.Report("compare", 5, 10); + return 0; + }, + plan); + + // compress weighs 3 of 5, verify and compare 1 each. + Assert.Equal([0.3, 0.6, 0.8, 0.9, 1], seen.Where(p => p > 0).Select(p => Math.Round(p, 3)).Distinct()); + Assert.Equal(["compress (1/3)", "verify (2/3)"], labels); + } + + [Theory] + [InlineData(new[] { "pack", "file", "a", "b", "--verify" }, "compress,verify,compare")] + [InlineData(new[] { "pack", "file", "a", "b" }, "")] + [InlineData(new[] { "pack", "exfat", "a" }, "")] + [InlineData(new[] { "verify", "a.ffpfsc", "--source-file", "a.exfat" }, "verify,compare")] + [InlineData(new[] { "verify", "a.ffpfsc" }, "")] + [InlineData(new[] { "repair", "a.ffpfsc" }, "scan,repair,verify")] + [InlineData(new[] { "repair", "a.ffpfsc", "--scan" }, "")] + [InlineData(new[] { "ampr", "game", "--root", "g", "--output", "o", "--exfat", "." }, "pack,exfat")] + [InlineData(new[] { "ampr", "game", "--root", "g", "--output", "o" }, "")] + public void Plans_list_the_phases_each_command_reports(string[] args, string phases) => + Assert.Equal(phases, string.Join(',', ProgressPlan.For(args)?.Select(p => p.Name) ?? [])); + [Fact] public async Task Cancel_stops_the_job_at_its_next_progress_report() { diff --git a/tests/MkPFS.Gui.Tests/LocalizerTests.cs b/tests/MkPFS.Gui.Tests/LocalizerTests.cs index 5d76ae6..5379976 100644 --- a/tests/MkPFS.Gui.Tests/LocalizerTests.cs +++ b/tests/MkPFS.Gui.Tests/LocalizerTests.cs @@ -34,7 +34,7 @@ public void Lookup_falls_back_to_the_key_and_formats_like_python() { Assert.Equal("no_such_key", Localizer.Instance["no_such_key"]); Assert.Equal("✗ Process exited with code 3.", Localizer.Instance.Format("err_process", 3)); - Assert.Equal("3 item(s): 1 folder(s), 2 file(s)", Localizer.Instance.Format("bt_preview_count", 3, 1, 2)); + Assert.StartsWith("✓ Rules from traces: 2 recorded run(s) found.", Localizer.Instance.Format("ap_rules_traces", 2), StringComparison.Ordinal); } [AvaloniaTheory] diff --git a/tests/MkPFS.Gui.Tests/MainWindowTests.cs b/tests/MkPFS.Gui.Tests/MainWindowTests.cs index 56adaea..be9388f 100644 --- a/tests/MkPFS.Gui.Tests/MainWindowTests.cs +++ b/tests/MkPFS.Gui.Tests/MainWindowTests.cs @@ -41,10 +41,11 @@ public void Sidebar_lists_every_operation_grouped_by_section() Assert.Equal(["section_build", "section_check", "section_read"], model.Sections.Select(s => s.TitleKey)); Assert.Equal( - ["batch", "pack_folder", "pack_exfat", "pack_file", "verify", "repair", "inspect", "tree", "unpack"], + ["pack_exfat", "pack_file", "ampr", "verify", "repair", "inspect", "tree", "unpack"], NavButtons(window).Select(AutomationProperties.GetName)); - Assert.True(Nav(window, "batch").IsChecked); - Assert.Equal("Batch Convert", PageTitle(window)); + Assert.True(Nav(window, "pack_exfat").IsChecked); + Assert.Equal("Pack exFAT", PageTitle(window)); + Assert.True(Shows(window, "Pack FFPFSC")); Assert.True(Shows(window, "BUILD")); Assert.Equal("MkPFS.CSharp", window.Title); Assert.True(Shows(window, "MkPFS.CSharp")); @@ -56,7 +57,7 @@ public void Packing_pages_link_to_the_apr_emu_downloads() { (MainWindow window, _) = Open(); - foreach (string page in new[] { "batch", "pack_folder", "pack_exfat" }) + foreach (string page in new[] { "pack_exfat" }) { Nav(window, page).IsChecked = true; Dispatcher.UIThread.RunJobs(); @@ -75,7 +76,7 @@ public void Run_sits_between_the_options_and_the_log_and_advanced_starts_collaps { (MainWindow window, _) = Open(); - foreach (string page in new[] { "batch", "pack_folder", "pack_exfat", "pack_file" }) + foreach (string page in new[] { "pack_exfat", "pack_file" }) { Nav(window, page).IsChecked = true; Dispatcher.UIThread.RunJobs(); @@ -152,6 +153,24 @@ public void Closing_without_a_job_does_not_ask() Assert.Empty(window.OwnedWindows); } + [AvaloniaFact] + public void Tuning_and_advanced_options_explain_themselves_on_hover() + { + (MainWindow window, _) = Open(); + Nav(window, "pack_file").IsChecked = true; + Dispatcher.UIThread.RunJobs(); + window.GetVisualDescendants().OfType().Single(e => e.Name == "AdvancedExpander").IsExpanded = true; + Dispatcher.UIThread.RunJobs(); + + string[] tips = [.. window.GetVisualDescendants().OfType().Select(ToolTip.GetTip).OfType()]; + + // 8 compression tuning fields, 7 packing options and 7 PFS options. + Assert.Equal(22, tips.Length); + Assert.All(tips, t => Assert.DoesNotContain("tip_", t, StringComparison.Ordinal)); + Assert.Contains(tips, t => t.StartsWith("How many CPU cores compress blocks", StringComparison.Ordinal)); + window.Close(); + } + [AvaloniaFact] public void Clicking_an_entry_shows_its_page() { @@ -163,7 +182,7 @@ public void Clicking_an_entry_shows_its_page() Assert.Equal("Repair", PageTitle(window)); Assert.Same(model.Items.Single(i => i.Key == "repair").Page, model.Current); Assert.Equal(["repair"], model.Items.Where(i => i.IsSelected).Select(i => i.Key)); - Assert.False(Nav(window, "batch").IsChecked); + Assert.False(Nav(window, "pack_exfat").IsChecked); window.Close(); } @@ -177,9 +196,9 @@ public void Language_picker_relabels_the_window() picker.SelectedIndex = 1; Dispatcher.UIThread.RunJobs(); - Assert.Equal("Converter em Lote", PageTitle(window)); + Assert.Equal("Empacotar EXFAT", PageTitle(window)); Assert.True(Shows(window, "CONSTRUIR")); - Assert.True(Shows(window, "Empacotar Pasta")); + Assert.True(Shows(window, "Empacotar FFPFSC")); Assert.True(Shows(window, "Executar")); } finally @@ -195,13 +214,10 @@ public async Task Every_page_renders() using TempDir dir = new(); string game = BuildPanelTests.Game(dir, "games/PPSA01234-app"); (MainWindow window, MainWindowViewModel model) = Open(); - PackFolderPanelViewModel folder = (PackFolderPanelViewModel)model.Items.Single(i => i.Key == "pack_folder").Page; - folder.Metadata.Debounce = TimeSpan.Zero; - folder.Source = game; - await folder.Metadata.LoadAsync(game); - BatchPanelViewModel batch = (BatchPanelViewModel)model.Items.Single(i => i.Key == "batch").Page; - batch.Source = Path.GetDirectoryName(game)!; - await batch.Queue.LoadAsync(batch.Source); + PackExfatPanelViewModel exfat = (PackExfatPanelViewModel)model.Items.Single(i => i.Key == "pack_exfat").Page; + exfat.Metadata.Debounce = TimeSpan.Zero; + exfat.Source = game; + await exfat.Metadata.LoadAsync(game); window.Height = 1500; // whole pages in the snapshots RepairPanelViewModel repair = (RepairPanelViewModel)model.Items.Single(i => i.Key == "repair").Page; repair.Image = RepairPanelTests.RiskyImage(dir); diff --git a/tests/MkPFS.Gui.Tests/PreviewTests.cs b/tests/MkPFS.Gui.Tests/PreviewTests.cs index 2fbc6bb..81b6828 100644 --- a/tests/MkPFS.Gui.Tests/PreviewTests.cs +++ b/tests/MkPFS.Gui.Tests/PreviewTests.cs @@ -44,42 +44,4 @@ public async Task A_newer_path_supersedes_a_pending_one() Assert.Equal("-", preview.TitleId); Assert.Equal("-", preview.Title); } - - [AvaloniaFact] - public async Task Batch_queue_lists_folders_and_images_with_metadata() - { - using TempDir dir = new(); - string games = Path.Combine(dir.Path, "games"); - string game = BuildPanelTests.Game(dir, "games/PPSA01234-app"); - Assert.Equal(0, MkPFSCli.Run(["pack", "exfat", game, Path.Combine(games, "Data.exfat")], new CliContext(TextWriter.Null, TextWriter.Null, false, false, false))); - dir.File("games/readme.txt"); - BatchQueueViewModel queue = new(Brushes.Teal); - - await queue.LoadAsync(games); - - Assert.Equal("2 item(s): 1 folder(s), 1 file(s)", queue.Summary); - Assert.False(queue.IsEmpty); - Assert.Equal(["Astro Test", "Astro Test"], queue.Rows.Select(r => r.Title)); // the exFAT holds the same game - Assert.Equal(["FILE", "FOLDER"], queue.Rows.Select(r => r.Kind)); - BatchQueueRow folder = queue.Rows[1]; - Assert.StartsWith("Title ID: PPSA01234 | Version: 01.004.000 | ", folder.Details, StringComparison.Ordinal); - Assert.Equal("Content ID: UP0001-PPSA01234_00-ASTROBOT00000000", folder.ContentId); - Assert.True(folder.HasCover); - Assert.Null(folder.AprEmu); - } - - [AvaloniaFact] - public async Task Batch_queue_explains_empty_and_invalid_sources() - { - using TempDir dir = new(); - BatchQueueViewModel queue = new(Brushes.Teal); - - await queue.LoadAsync(string.Empty); - Assert.Equal("Select a source folder to preview the batch queue.", queue.Summary); - await queue.LoadAsync(Path.Combine(dir.Path, "missing")); - Assert.Equal("Select an existing folder to preview the batch queue.", queue.Summary); - await queue.LoadAsync(dir.Path); - Assert.Equal("No packable items found.", queue.Summary); - Assert.True(queue.IsEmpty); - } } diff --git a/tests/MkPFS.Parity/AMPRCliParityTests.cs b/tests/MkPFS.Parity/AMPRCliParityTests.cs new file mode 100644 index 0000000..47055bc --- /dev/null +++ b/tests/MkPFS.Parity/AMPRCliParityTests.cs @@ -0,0 +1,111 @@ +using System.Text; +using System.Text.Json; +using MkPFS.Cli; +using MkPFS.Cli.Output; + +namespace MkPFS.Parity; + +/// +/// Replays every ampr_pack command recorded by tools/oracle/build_ampr_goldens.py through mkpfs ampr, in +/// the oracle's order and folder layout, and compares stdout, stderr and the exit code with each log. +/// +public sealed class AMPRCliParityTests +{ + private const long FixedMTime = 1_600_000_000; + + // Execution order in build_ampr_goldens.py (the manifest stores steps with sorted keys). + private static readonly string[] StepOrder = ["pack", "verify", "list", "list_text", "inspect", "unpack", "remove_plan", "runtime_config"]; + + public static TheoryData Cases() + { + TheoryData data = []; + string? manifest = Fixtures.GeneratedRoot is { } root ? Path.Combine(root, "ampr", "goldens", "manifest.json") : null; + if (manifest is null || !File.Exists(manifest)) + { + data.Add("(corpus missing)"); + return data; + } + + using JsonDocument document = JsonDocument.Parse(File.ReadAllBytes(manifest)); + foreach (JsonProperty entry in document.RootElement.GetProperty("cases").EnumerateObject()) + { + data.Add(entry.Name); + } + + return data; + } + + [Theory] + [MemberData(nameof(Cases))] + public void Every_step_matches_the_oracle_log(string caseName) + { + string caseDir = Fixtures.PathOrSkip("ampr", "goldens", caseName); + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", "manifest.json"))); + JsonElement entry = manifest.RootElement.GetProperty("cases").GetProperty(caseName); + string work = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-cli-" + Guid.NewGuid().ToString("N")); + try + { + Prepare(caseDir, entry, work); + JsonElement steps = entry.GetProperty("steps"); + Assert.All(steps.EnumerateObject(), step => Assert.Contains(step.Name, StepOrder)); + foreach (string stepName in StepOrder.Where(name => steps.TryGetProperty(name, out _))) + { + string[] argv = [.. steps.GetProperty(stepName).EnumerateArray().Select(a => a.GetString()!)]; + string expected = File.ReadAllText(Path.Combine(caseDir, stepName + ".log"), Encoding.UTF8).Replace("\r\n", "\n", StringComparison.Ordinal); + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + CliContext ctx = new(stdout, stderr, useColor: false, utf8: false, progress: true) { WorkingDirectory = work }; + int exit = MkPFSCli.Run(["ampr", .. argv], ctx); + + // The port warns when a pack run packs nothing (ampr_pack.py stays silent): exactly where the oracle + // reported files_packed 0. + string warning = Cli.Commands.AmprCommand.NothingPackedWarning + "\n"; + string errors = stderr.ToString(); + Assert.Equal(expected.Contains("\"files_packed\": 0,", StringComparison.Ordinal), errors.Contains(warning, StringComparison.Ordinal)); + errors = errors.Replace(warning, string.Empty, StringComparison.Ordinal); + string actual = CliParityTests.Normalize( + $"$ ampr_pack.py {string.Join(' ', argv)}\nexit={exit}\n--- stdout\n{stdout}\n--- stderr\n{errors}", work); + + Assert.True(expected == actual, $"{caseName}/{stepName}:\n--- expected\n{expected}\n--- actual\n{actual}"); + } + } + finally + { + Directory.Delete(work, recursive: true); + } + } + + // The oracle's case folder: app0 (fixture copy, pinned mtimes, its index, then removals and touches) plus the + // configuration files. + private static void Prepare(string caseDir, JsonElement entry, string work) + { + string tree = Fixtures.PathOrSkip("ampr", "trees", "ampr_assets"); + string app0 = Path.Combine(work, "app0"); + foreach (string file in Directory.EnumerateFiles(tree, "*", SearchOption.AllDirectories)) + { + string target = Path.Combine(app0, Path.GetRelativePath(tree, file)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(file, target); + File.SetLastWriteTimeUtc(target, DateTime.UnixEpoch.AddSeconds(FixedMTime)); + } + + File.Copy(Path.Combine(caseDir, "ampr_emu.index"), Path.Combine(app0, "ampr_emu.index")); + foreach (JsonElement removed in entry.GetProperty("remove_before_pack").EnumerateArray()) + { + File.Delete(Path.Combine(app0, removed.GetString()!)); + } + + foreach (JsonProperty touched in entry.GetProperty("touch_after_index").EnumerateObject()) + { + File.SetLastWriteTimeUtc(Path.Combine(app0, touched.Name), DateTime.UnixEpoch.AddSeconds(touched.Value.GetInt64())); + } + + foreach (string name in (string[])["config.toml", "list.txt", "runtime_alt.toml"]) + { + if (File.Exists(Path.Combine(caseDir, name))) + { + File.Copy(Path.Combine(caseDir, name), Path.Combine(work, name)); + } + } + } +} diff --git a/tests/MkPFS.Parity/AMPRPackBuilderParityTests.cs b/tests/MkPFS.Parity/AMPRPackBuilderParityTests.cs new file mode 100644 index 0000000..0da9e72 --- /dev/null +++ b/tests/MkPFS.Parity/AMPRPackBuilderParityTests.cs @@ -0,0 +1,215 @@ +using System.Security.Cryptography; +using System.Text.Json; +using MkPFS.Build.AMPRPack; +using MkPFS.Core.AMPR; + +namespace MkPFS.Parity; + +/// +/// Rebuilds every golden pack set (tools/oracle/build_ampr_goldens.py) with and +/// compares each output file byte for byte (SHA-256), plus the stats and warnings ampr_pack printed. +/// +public sealed class AMPRPackBuilderParityTests +{ + private const long FixedMTime = 1_600_000_000; + + public static TheoryData BuiltCases => AMPRPackFormatParityTests.BuiltCases; + + private sealed record CaseInfo( + List Flags, List Removed, Dictionary Touched, bool HasConfig, bool HasRuntimeConfigStep); + + private static CaseInfo ReadCase(string caseName) + { + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", "manifest.json"))); + JsonElement entry = manifest.RootElement.GetProperty("cases").GetProperty(caseName); + return new CaseInfo( + [.. entry.GetProperty("flags").EnumerateArray().Select(e => e.GetString()!)], + [.. entry.GetProperty("remove_before_pack").EnumerateArray().Select(e => e.GetString()!)], + entry.GetProperty("touch_after_index").EnumerateObject().ToDictionary(p => p.Name, p => p.Value.GetInt64()), + entry.GetProperty("toml").GetBoolean(), + entry.GetProperty("steps").TryGetProperty("runtime_config", out _)); + } + + // The oracle copies trees/ampr_assets, pins mtimes, builds the index, then removes or touches files. + private static string PrepareRoot(CaseInfo info, string work) + { + string tree = Fixtures.PathOrSkip("ampr", "trees", "ampr_assets"); + if (info.Removed.Count == 0 && info.Touched.Count == 0) + { + return tree; + } + + string root = Path.Combine(work, "app0"); + foreach (string file in Directory.EnumerateFiles(tree, "*", SearchOption.AllDirectories)) + { + string target = Path.Combine(root, Path.GetRelativePath(tree, file)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(file, target); + File.SetLastWriteTimeUtc(target, DateTime.UnixEpoch.AddSeconds(FixedMTime)); + } + + foreach (string removed in info.Removed) + { + File.Delete(Path.Combine(root, removed)); + } + + foreach ((string relative, long mtime) in info.Touched) + { + File.SetLastWriteTimeUtc(Path.Combine(root, relative), DateTime.UnixEpoch.AddSeconds(mtime)); + } + + return root; + } + + private static AMPRBuildResult Run(string caseName, string outDir, string work) + { + CaseInfo info = ReadCase(caseName); + string root = PrepareRoot(info, work); + string caseDir = Fixtures.PathOrSkip("ampr", "goldens", caseName); + AMPRPackConfig config = AMPRPackConfig.Load(info.HasConfig ? Path.Combine(caseDir, "config.toml") : null); + List include = []; + List exclude = []; + bool allowMissing = false; + for (int i = 0; i < info.Flags.Count; i++) + { + switch (info.Flags[i]) + { + case "--workers": + config.Workers = long.Parse(info.Flags[++i], System.Globalization.CultureInfo.InvariantCulture); + break; + case "--self-contained": + config.SelfContained = true; + break; + case "--require-packed": + config.RequiredPacked = [.. config.RequiredPacked, info.Flags[++i]]; + break; + case "--include": + include.Add(info.Flags[++i]); + break; + case "--exclude": + exclude.Add(info.Flags[++i]); + break; + case "--allow-missing": + allowMissing = true; + break; + default: + throw new InvalidOperationException($"unhandled flag {info.Flags[i]}"); + } + } + + return AMPRPackBuilder.Build(root, Path.Combine(caseDir, "ampr_emu.index"), outDir, config, include, exclude, allowMissing); + } + + private static JsonElement OracleOutput(string caseName) + { + string log = File.ReadAllText(Fixtures.PathOrSkip("ampr", "goldens", caseName, "pack.log")); + int start = log.IndexOf("--- stdout\n", StringComparison.Ordinal) + "--- stdout\n".Length; + int end = log.IndexOf("\n--- stderr", StringComparison.Ordinal); + using JsonDocument document = JsonDocument.Parse(log[start..end]); + return document.RootElement.Clone(); + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Pack_set_matches_the_oracle_byte_for_byte(string caseName) + { + string work = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-build-" + Guid.NewGuid().ToString("N")); + string outDir = Path.Combine(work, "out"); + try + { + AMPRBuildResult result = Run(caseName, outDir, work); + + Dictionary expected = OracleHashesUnder(caseName, "out/"); + Dictionary actual = Directory.EnumerateFiles(outDir, "*", SearchOption.AllDirectories).ToDictionary( + p => "out/" + Path.GetRelativePath(outDir, p).Replace('\\', '/'), + p => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(p))), + StringComparer.Ordinal); + if (ReadCase(caseName).HasRuntimeConfigStep) + { + // The oracle's later runtime-config step rewrote .runtime; check the build's own settings instead. + const string runtime = "out/ampr_assets.index.runtime"; + AMPRPackConfig config = AMPRPackConfig.Load(Fixtures.PathOrSkip("ampr", "goldens", caseName, "config.toml")); + byte[] buildId = Core.AMPR.AMPRPackManifest.Load(result.IndexPath).BuildId; + Assert.Equal(config.Runtime!.Encode(buildId), File.ReadAllBytes(Path.Combine(outDir, "ampr_assets.index.runtime"))); + expected.Remove(runtime); + actual.Remove(runtime); + } + + Assert.Equal(expected.OrderBy(kv => kv.Key, StringComparer.Ordinal), actual.OrderBy(kv => kv.Key, StringComparer.Ordinal)); + + JsonElement oracle = OracleOutput(caseName); + AMPRBuildStats stats = result.Stats; + Assert.Equal( + Numbers(oracle), + new Dictionary + { + ["files_total"] = stats.FilesTotal, + ["files_packed"] = stats.FilesPacked, + ["files_loose"] = stats.FilesLoose, + ["files_auto_loose"] = stats.FilesAutoLoose, + ["auto_loose_logical_bytes"] = stats.AutoLooseLogicalBytes, + ["auto_loose_sampled_bytes"] = stats.AutoLooseSampledBytes, + ["chunks"] = stats.Chunks, + ["chunks_lz4"] = stats.ChunksLZ4, + ["chunks_raw"] = stats.ChunksRaw, + ["chunks_shared"] = stats.ChunksShared, + ["logical_bytes"] = stats.LogicalBytes, + ["stored_bytes"] = stats.StoredBytes, + ["padding_bytes"] = stats.PaddingBytes, + ["io_pages_touched"] = stats.IOPagesTouched, + ["io_page_safe_chunks"] = stats.IOPageSafeChunks, + ["dense_streaming_chunks"] = stats.DenseStreamingChunks, + }); + Assert.Equal(Strings(oracle, "loose_paths"), stats.LoosePaths); + Assert.Equal(Strings(oracle, "warnings"), result.Warnings); + } + finally + { + // Absent when a missing fixture skipped the case before the build created it. + if (Directory.Exists(work)) + { + Directory.Delete(work, recursive: true); + } + } + } + + [Theory] + [InlineData("neg_missing")] + [InlineData("neg_require_packed")] + public void Build_errors_match_the_oracle_and_leave_no_output(string caseName) + { + string log = File.ReadAllText(Fixtures.PathOrSkip("ampr", "goldens", caseName, "pack.log")); + string expected = log.Split('\n').Single(line => line.StartsWith("error: ", StringComparison.Ordinal))["error: ".Length..]; + string work = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-build-" + Guid.NewGuid().ToString("N")); + string outDir = Path.Combine(work, "out"); + try + { + AMPRPackException error = Assert.Throws(() => Run(caseName, outDir, work)); + Assert.Equal(expected, error.Message); + Assert.Empty(Directory.EnumerateFileSystemEntries(outDir)); + } + finally + { + // Absent when a missing fixture skipped the case before the build created it. + if (Directory.Exists(work)) + { + Directory.Delete(work, recursive: true); + } + } + } + + private static Dictionary OracleHashesUnder(string caseName, string prefix) + { + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", "manifest.json"))); + return manifest.RootElement.GetProperty("cases").GetProperty(caseName).GetProperty("sha256").EnumerateObject() + .Where(p => p.Name.StartsWith(prefix, StringComparison.Ordinal)) + .ToDictionary(p => p.Name, p => p.Value.GetString()!, StringComparer.Ordinal); + } + + private static Dictionary Numbers(JsonElement oracle) => oracle.EnumerateObject() + .Where(p => p.Value.ValueKind == JsonValueKind.Number && p.Name != "compression_ratio") + .ToDictionary(p => p.Name, p => p.Value.GetInt64()); + + private static List Strings(JsonElement oracle, string name) => + [.. oracle.GetProperty(name).EnumerateArray().Select(e => e.GetString()!)]; +} diff --git a/tests/MkPFS.Parity/AMPRPackConfigParityTests.cs b/tests/MkPFS.Parity/AMPRPackConfigParityTests.cs new file mode 100644 index 0000000..0d58dc7 --- /dev/null +++ b/tests/MkPFS.Parity/AMPRPackConfigParityTests.cs @@ -0,0 +1,42 @@ +using MkPFS.Build.AMPRPack; +using MkPFS.Core.AMPR; + +namespace MkPFS.Parity; + +/// +/// Loads every golden pack configuration (ampr/goldens/<case>/config.toml) and compares the build-id input +/// with the oracle's _canonical_config_bytes dump (config.canonical.json). +/// +public sealed class AMPRPackConfigParityTests +{ + public static TheoryData ConfiguredCases => new( + "default", "unsafe_default", "example", "fast", "fast_accel8", "hc9", "block16k_random", "block1m", "streaming", + "store_movies", "dedup_off", "dedup_group", "dedup_streaming", "lanes_balanced", "lanes_hash", "lanes_round_robin", + "stripe_rollover", "auto_loose", "self_contained", "runtime", "io_page_4k", "mtime_preserved", "no_preserve_mtime", + "cli_include_exclude", "include_from", "allow_missing", "workers1", "workers8"); + + [Theory] + [MemberData(nameof(ConfiguredCases))] + public void Canonical_config_bytes_match_the_oracle(string caseName) + { + string config = Fixtures.PathOrSkip("ampr", "goldens", caseName, "config.toml"); + byte[] expected = File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", caseName, "config.canonical.json")); + + Assert.Equal(System.Text.Encoding.UTF8.GetString(expected), System.Text.Encoding.UTF8.GetString(AMPRPackConfig.Load(config).CanonicalBytes())); + } + + [Theory] + [InlineData("neg_bad_block")] + [InlineData("neg_unknown_group")] + [InlineData("neg_runtime_keys")] + public void Config_errors_match_the_oracle_message(string caseName) + { + string config = Fixtures.PathOrSkip("ampr", "goldens", caseName, "config.toml"); + string log = File.ReadAllText(Fixtures.PathOrSkip("ampr", "goldens", caseName, "pack.log")); + string expected = log.Split('\n').Single(line => line.StartsWith("error: ", StringComparison.Ordinal))["error: ".Length..]; + + Exception error = Assert.ThrowsAny(() => AMPRPackConfig.Load(config)); + Assert.True(error is AMPRPackException or ArgumentException, error.GetType().Name); + Assert.Equal(expected, error.Message); + } +} diff --git a/tests/MkPFS.Parity/AMPRPackFormatParityTests.cs b/tests/MkPFS.Parity/AMPRPackFormatParityTests.cs new file mode 100644 index 0000000..3d6f960 --- /dev/null +++ b/tests/MkPFS.Parity/AMPRPackFormatParityTests.cs @@ -0,0 +1,126 @@ +using System.Security.Cryptography; +using System.Text.Json; +using MkPFS.Core.AMPR; +using MkPFS.Core.Compression; +using MkPFS.Core.Util; + +namespace MkPFS.Parity; + +/// +/// Reads every pack set built by ampr_emu ampr_pack.py (ampr/goldens, see +/// tools/oracle/build_ampr_goldens.py), re-serializes each structure byte for byte, and decodes every +/// packed file against the oracle's unpack hashes and the decoded-chunk CRC sidecar. +/// +public sealed class AMPRPackFormatParityTests +{ + public static TheoryData BuiltCases => new( + "default", "unsafe_default", "no_config", "example", "fast", "fast_accel8", "hc9", "block16k_random", "block1m", + "streaming", "store_movies", "dedup_off", "dedup_group", "dedup_streaming", "lanes_balanced", "lanes_hash", + "lanes_round_robin", "stripe_rollover", "auto_loose", "self_contained", "runtime", "io_page_4k", "mtime_preserved", + "no_preserve_mtime", "cli_include_exclude", "include_from", "allow_missing", "workers1", "workers8"); + + private static Dictionary OracleHashes(string caseName) + { + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", "manifest.json"))); + return manifest.RootElement.GetProperty("cases").GetProperty(caseName).GetProperty("sha256") + .EnumerateObject().ToDictionary(p => p.Name, p => p.Value.GetString()!, StringComparer.Ordinal); + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Manifest_sidecars_and_volume_headers_round_trip(string caseName) + { + string outDir = Fixtures.PathOrSkip("ampr", "goldens", caseName, "out"); + string indexPath = Path.Combine(outDir, "ampr_assets.index"); + byte[] indexBytes = File.ReadAllBytes(indexPath); + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + Assert.Equal(indexBytes, manifest.Serialize()); + + byte[] crcBytes = File.ReadAllBytes(AMPRPackFormat.ChunkCrcPath(indexPath)); + uint[] crcs = AMPRChunkCrcs.Parse(crcBytes, manifest.BuildId, manifest.Chunks.Count); + Assert.Equal(crcBytes, AMPRChunkCrcs.Build(manifest.BuildId, crcs)); + + string runtimePath = AMPRPackFormat.RuntimePath(indexPath); + AMPRRuntimeSettings? runtime = AMPRRuntimeSettings.Read(runtimePath, manifest.BuildId); + Assert.Equal(File.Exists(runtimePath), runtime is not null); + if (runtime is not null) + { + Assert.Equal(File.ReadAllBytes(runtimePath), runtime.Encode(manifest.BuildId)); + } + + for (int packId = 0; packId < manifest.Packs.Count; packId++) + { + AMPRPackRecord record = manifest.Packs[packId]; + string volume = AMPRAssetPath.SafeOutputPath(outDir, manifest.PackName(packId)); + byte[] head = new byte[AMPRPackFormat.DataHeaderSize]; + using (FileStream stream = File.OpenRead(volume)) + { + stream.ReadExactly(head); + } + + AMPRDataHeader header = AMPRDataHeader.Parse(head, (uint)packId, manifest.BuildId, record.Flags); + Assert.Equal(head, header.ToBytes()); + Assert.Equal(record.FileSize, (ulong)new FileInfo(volume).Length); + Assert.Equal(record.PayloadBytes, header.PayloadBytes); + Assert.Equal(record.FileSize, header.PayloadOffset + header.PayloadBytes); + } + + // AMPRIDX3 rows map one-to-one onto manifest file records (file id = row + 1). + List index = AMPRIndexReader.Read(Fixtures.PathOrSkip("ampr", "goldens", caseName, "ampr_emu.index")); + Assert.Equal(index.Count, manifest.Files.Count); + for (int i = 0; i < index.Count; i++) + { + Assert.Equal(index[i].Path, manifest.FilePath(i + 1)); + Assert.Equal(AMPRAssetPath.Hash(index[i].Path), manifest.Files[i].PathHash); + } + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Decoded_files_match_oracle_unpack_hashes_and_chunk_crcs(string caseName) + { + string outDir = Fixtures.PathOrSkip("ampr", "goldens", caseName, "out"); + string indexPath = Path.Combine(outDir, "ampr_assets.index"); + AMPRPackManifest manifest = AMPRPackManifest.Load(indexPath); + uint[] crcs = AMPRChunkCrcs.Load(AMPRPackFormat.ChunkCrcPath(indexPath), manifest.BuildId, manifest.Chunks.Count); + Dictionary oracle = OracleHashes(caseName); + byte[][] volumes = [.. Enumerable.Range(0, manifest.Packs.Count) + .Select(id => File.ReadAllBytes(AMPRAssetPath.SafeOutputPath(outDir, manifest.PackName(id))))]; + + int packed = 0; + for (int fileId = 1; fileId <= manifest.Files.Count; fileId++) + { + AMPRFileRecord record = manifest.Files[fileId - 1]; + if (!record.IsPacked) + { + continue; + } + + packed++; + using IncrementalHash sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + for (uint local = 0; local < record.ChunkCount; local++) + { + int chunkIndex = (int)(record.FirstChunk + local); + AMPRChunkRecord chunk = manifest.Chunks[chunkIndex]; + ReadOnlySpan stored = volumes[chunk.PackId].AsSpan((int)chunk.Offset, chunk.StoredSize); + byte[] raw = new byte[chunk.RawSize]; + if (chunk.Codec == AMPRPackFormat.ChunkCodecLZ4) + { + Assert.Equal(chunk.RawSize, LZ4Codec.Decompress(stored, raw)); + } + else + { + stored.CopyTo(raw); + } + + Assert.Equal(crcs[chunkIndex], Crc32.Update(0, raw)); + sha.AppendData(raw); + } + + string relative = AMPRAssetPath.Relative(manifest.FilePath(fileId)); + Assert.Equal(oracle[$"unpacked/{relative}"], Convert.ToHexStringLower(sha.GetHashAndReset())); + } + + Assert.Equal(oracle.Keys.Count(k => k.StartsWith("unpacked/", StringComparison.Ordinal)), packed); + } +} diff --git a/tests/MkPFS.Parity/AMPRPackOperationsParityTests.cs b/tests/MkPFS.Parity/AMPRPackOperationsParityTests.cs new file mode 100644 index 0000000..0b6cc75 --- /dev/null +++ b/tests/MkPFS.Parity/AMPRPackOperationsParityTests.cs @@ -0,0 +1,218 @@ +using System.Security.Cryptography; +using System.Text.Json; +using MkPFS.Build.AMPRPack; +using MkPFS.Core.AMPR; + +namespace MkPFS.Parity; + +/// +/// Runs verify, list, inspect, unpack, the removal plan and runtime-config on every golden pack set and compares +/// the results with the JSON ampr_pack printed (verify.log, list.log, ...). +/// +public sealed class AMPRPackOperationsParityTests +{ + public static TheoryData BuiltCases => AMPRPackFormatParityTests.BuiltCases; + + private static string Index(string caseName) => Fixtures.PathOrSkip("ampr", "goldens", caseName, "out", "ampr_assets.index"); + + // Removed files only ever affect loose entries, so the shared fixture tree serves as every case's /app0. + private static string Root() => Fixtures.PathOrSkip("ampr", "trees", "ampr_assets"); + + private static JsonElement Stdout(string caseName, string step) + { + string log = File.ReadAllText(Fixtures.PathOrSkip("ampr", "goldens", caseName, step + ".log")); + int start = log.IndexOf("--- stdout\n", StringComparison.Ordinal) + "--- stdout\n".Length; + int end = log.IndexOf("\n--- stderr", StringComparison.Ordinal); + using JsonDocument document = JsonDocument.Parse(log[start..end]); + return document.RootElement.Clone(); + } + + private static string? Error(string caseName, string step) => + File.ReadAllText(Fixtures.PathOrSkip("ampr", "goldens", caseName, step + ".log")).Split('\n') + .FirstOrDefault(line => line.StartsWith("error: ", StringComparison.Ordinal))?["error: ".Length..]; + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Verify_and_source_compare_match(string caseName) + { + JsonElement oracle = Stdout(caseName, "verify"); + AMPRVerifyResult verify = AMPRPackTools.Verify(Index(caseName)); + AMPRSourceCompareResult compare = AMPRPackTools.VerifyAgainstRoot(Index(caseName), Root()); + + Assert.Equal( + (oracle.GetProperty("files").GetInt64(), oracle.GetProperty("physical_chunks").GetInt64(), + oracle.GetProperty("stored_bytes").GetInt64(), oracle.GetProperty("raw_bytes").GetInt64()), + (verify.Files, verify.PhysicalChunks, verify.StoredBytes, verify.RawBytes)); + JsonElement source = oracle.GetProperty("source_compare"); + Assert.Equal( + (source.GetProperty("files").GetInt64(), source.GetProperty("chunks").GetInt64(), source.GetProperty("bytes").GetInt64()), + (compare.Files, compare.Chunks, compare.Bytes)); + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void List_matches(string caseName) + { + JsonElement[] oracle = [.. Stdout(caseName, "list").EnumerateArray()]; + List rows = AMPRPackTools.List(Index(caseName)); + + Assert.Equal(oracle.Length, rows.Count); + for (int i = 0; i < rows.Count; i++) + { + JsonElement o = oracle[i]; + AMPRListEntry r = rows[i]; + Assert.Equal(o.GetProperty("file_id").GetInt32(), r.FileId); + Assert.Equal(o.GetProperty("path").GetString(), r.Path); + Assert.Equal(o.GetProperty("packed").GetBoolean(), r.Packed); + Assert.Equal(o.GetProperty("logical_size").GetUInt64(), r.LogicalSize); + Assert.Equal(o.GetProperty("stored_size").GetInt64(), r.StoredSize); + Assert.Equal(o.GetProperty("block_size").GetInt64(), r.BlockSize); + Assert.Equal(o.GetProperty("chunks").GetUInt32(), r.Chunks); + Assert.Equal(o.GetProperty("codecs").EnumerateArray().Select(e => e.GetString()!), r.Codecs); + Assert.Equal(o.GetProperty("packs").EnumerateArray().Select(e => e.GetInt32()), r.Packs); + Assert.Equal(o.GetProperty("io_page_sizes").EnumerateArray().Select(e => e.GetUInt32()), r.IOPageSizes); + Assert.Equal(o.GetProperty("layout").GetString(), r.Layout); + Assert.Equal(o.GetProperty("streaming").GetBoolean(), r.Streaming); + Assert.Equal(o.GetProperty("random_access").GetBoolean(), r.RandomAccess); + Assert.Equal(o.GetProperty("hot").GetBoolean(), r.Hot); + } + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Inspect_matches(string caseName) + { + JsonElement oracle = Stdout(caseName, "inspect"); + AMPRInspectResult result = AMPRPackTools.Inspect(Index(caseName)); + + Assert.Equal(oracle.GetProperty("build_id").GetString(), result.BuildId); + Assert.Equal( + (oracle.GetProperty("files").GetInt64(), oracle.GetProperty("packed_files").GetInt64(), + oracle.GetProperty("loose_files").GetInt64(), oracle.GetProperty("chunks").GetInt64()), + (result.Files, result.PackedFiles, result.LooseFiles, result.Chunks)); + // The oracle ran inspect before its runtime-config step rewrote .runtime; expect the final settings. + JsonElement runtime = File.Exists(Path.Combine(Fixtures.PathOrSkip("ampr", "goldens", caseName), "runtime_config.log")) + ? Stdout(caseName, "runtime_config").GetProperty("runtime") + : oracle.GetProperty("runtime"); + if (runtime.ValueKind == JsonValueKind.Null) + { + Assert.Null(result.Runtime); + } + else + { + Assert.Equal( + new AMPRRuntimeSettings( + runtime.GetProperty("decoded_cache_bytes").GetInt64(), + runtime.GetProperty("physical_cache_bytes").GetInt64(), + runtime.GetProperty("workers").GetInt64(), + runtime.GetProperty("latency_reserve_workers").GetInt64()), + result.Runtime); + } + + Assert.Equal( + oracle.GetProperty("packs").EnumerateArray().Select(p => new AMPRInspectPack( + p.GetProperty("id").GetInt32(), + p.GetProperty("name").GetString()!, + p.GetProperty("file_size").GetUInt64(), + p.GetProperty("payload_bytes").GetUInt64(), + p.GetProperty("io_page_size").GetUInt32(), + p.GetProperty("io_pages").GetUInt64(), + p.GetProperty("flags").GetUInt32())), + result.Packs); + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Unpack_matches_counts_hashes_and_mtimes(string caseName) + { + JsonElement oracle = Stdout(caseName, "unpack"); + string output = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-unpack-" + Guid.NewGuid().ToString("N")); + try + { + AMPRExtractResult result = AMPRPackTools.Extract(Index(caseName), output); + Assert.Equal((oracle.GetProperty("files").GetInt64(), oracle.GetProperty("bytes").GetInt64()), (result.Files, result.Bytes)); + + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", "manifest.json"))); + Dictionary expected = manifest.RootElement.GetProperty("cases").GetProperty(caseName).GetProperty("sha256") + .EnumerateObject().Where(p => p.Name.StartsWith("unpacked/", StringComparison.Ordinal)) + .ToDictionary(p => p.Name["unpacked/".Length..], p => p.Value.GetString()!, StringComparer.Ordinal); + Dictionary actual = Directory.Exists(output) + ? Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories).ToDictionary( + p => Path.GetRelativePath(output, p).Replace('\\', '/'), + p => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(p))), + StringComparer.Ordinal) + : []; + Assert.Equal(expected.OrderBy(kv => kv.Key, StringComparer.Ordinal), actual.OrderBy(kv => kv.Key, StringComparer.Ordinal)); + + AMPRPackManifest pack = AMPRPackManifest.Load(Index(caseName)); + for (int fileId = 1; fileId <= pack.Files.Count; fileId++) + { + if (pack.Files[fileId - 1].IsPacked) + { + string path = Path.Combine(output, AMPRAssetPath.Relative(pack.FilePath(fileId))); + Assert.Equal(DateTime.UnixEpoch.AddSeconds(pack.Files[fileId - 1].MTime), File.GetLastWriteTimeUtc(path)); + } + } + } + finally + { + if (Directory.Exists(output)) + { + Directory.Delete(output, recursive: true); + } + } + } + + [Theory] + [MemberData(nameof(BuiltCases))] + public void Removal_plan_matches(string caseName) + { + string? error = Error(caseName, "remove_plan"); + if (error is not null) + { + Assert.Equal(error, Assert.Throws(() => AMPRPackMaintenance.RemovalPlan(Index(caseName), Root())).Message); + return; + } + + JsonElement oracle = Stdout(caseName, "remove_plan"); + AMPRRemovalPlan plan = AMPRPackMaintenance.RemovalPlan(Index(caseName), Root()); + Assert.True(oracle.GetProperty("dry_run").GetBoolean()); + Assert.Equal( + (oracle.GetProperty("files").GetInt64(), oracle.GetProperty("present_files").GetInt64(), + oracle.GetProperty("missing_files").GetInt64(), oracle.GetProperty("bytes").GetInt64()), + (plan.Files, plan.PresentFiles, plan.MissingFiles, plan.Bytes)); + Assert.Equal(oracle.GetProperty("paths").EnumerateArray().Select(e => e.GetString()!), plan.Paths); + } + + [Fact] + public void Runtime_config_rewrites_the_runtime_like_the_oracle() + { + string caseDir = Fixtures.PathOrSkip("ampr", "goldens", "runtime"); + JsonElement oracle = Stdout("runtime", "runtime_config"); + string work = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-runtime-" + Guid.NewGuid().ToString("N")); + try + { + Directory.CreateDirectory(work); + string index = Path.Combine(work, "ampr_assets.index"); + File.Copy(Path.Combine(caseDir, "out", "ampr_assets.index"), index); + + AMPRRuntimeConfigResult result = AMPRPackMaintenance.WriteRuntimeConfig(index, Path.Combine(caseDir, "runtime_alt.toml")); + + Assert.Equal(File.ReadAllBytes(Path.Combine(caseDir, "out", "ampr_assets.index.runtime")), File.ReadAllBytes(index + ".runtime")); + Assert.Equal(oracle.GetProperty("build_id").GetString(), result.BuildId); + JsonElement runtime = oracle.GetProperty("runtime"); + Assert.Equal( + new AMPRRuntimeSettings( + runtime.GetProperty("decoded_cache_bytes").GetInt64(), + runtime.GetProperty("physical_cache_bytes").GetInt64(), + runtime.GetProperty("workers").GetInt64(), + runtime.GetProperty("latency_reserve_workers").GetInt64()), + result.Runtime); + Assert.Empty(Directory.GetFiles(work, ".runtime-*")); + } + finally + { + Directory.Delete(work, recursive: true); + } + } +} diff --git a/tests/MkPFS.Parity/LZ4VectorTests.cs b/tests/MkPFS.Parity/LZ4VectorTests.cs new file mode 100644 index 0000000..d14b569 --- /dev/null +++ b/tests/MkPFS.Parity/LZ4VectorTests.cs @@ -0,0 +1,82 @@ +using System.Buffers.Binary; +using System.Text.Json; +using MkPFS.Core.Compression; + +namespace MkPFS.Parity; + +/// +/// Byte-for-byte check of the native lz4 against the AMPR pack oracle codec (ampr_emu +/// ampr_pack_format.Lz4Codec, python-lz4 4.4.5), recorded by tools/oracle/build_ampr_goldens.py +/// in ampr/vectors/lz4_vectors.bin (format L4VEC001). +/// +public sealed class LZ4VectorTests +{ + private sealed record Vector(int Index, bool HighCompression, int Parameter, byte[] Raw, byte[] Compressed) + { + public override string ToString() => $"#{Index} {(HighCompression ? "hc" : "fast")}{Parameter}/{Raw.Length}"; + } + + private static List LoadVectors() + { + byte[] data = File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "vectors", "lz4_vectors.bin")); + Assert.Equal("L4VEC001"u8.ToArray(), data[..8]); + int count = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan(8)); + List vectors = new(count); + int offset = 12; + for (int i = 0; i < count; i++) + { + byte mode = data[offset]; + int parameter = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan(offset + 4)); + int rawLength = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan(offset + 8)); + int compressedLength = BinaryPrimitives.ReadInt32LittleEndian(data.AsSpan(offset + 12)); + offset += 16; + byte[] raw = data.AsSpan(offset, rawLength).ToArray(); + offset += rawLength; + byte[] compressed = data.AsSpan(offset, compressedLength).ToArray(); + offset += compressedLength; + vectors.Add(new Vector(i, mode == 1, parameter, raw, compressed)); + } + + Assert.Equal(data.Length, offset); + return vectors; + } + + [Fact] + public void Native_lz4_output_matches_ampr_pack_for_every_vector() + { + List vectors = LoadVectors(); + Assert.NotEmpty(vectors); + using LZ4Encoder encoder = new(); + List mismatches = []; + foreach (Vector vector in vectors) + { + byte[] produced = vector.HighCompression + ? encoder.CompressHC(vector.Raw, vector.Parameter) + : encoder.CompressFast(vector.Raw, vector.Parameter); + if (!produced.AsSpan().SequenceEqual(vector.Compressed)) + { + mismatches.Add(vector); + } + } + + Assert.True(mismatches.Count == 0, $"vectors differing from ampr_pack: {string.Join(", ", mismatches)}"); + } + + [Fact] + public void Native_lz4_decodes_every_ampr_pack_vector() + { + foreach (Vector vector in LoadVectors()) + { + byte[] output = new byte[vector.Raw.Length]; + Assert.Equal(vector.Raw.Length, LZ4Codec.Decompress(vector.Compressed, output)); + Assert.Equal(vector.Raw, output); + } + } + + [Fact] + public void Oracle_used_the_same_lz4_release() + { + using JsonDocument manifest = JsonDocument.Parse(File.ReadAllBytes(Fixtures.PathOrSkip("ampr", "goldens", "manifest.json"))); + Assert.Equal(LZ4Codec.NativeVersion, manifest.RootElement.GetProperty("versions").GetProperty("liblz4").GetString()); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRGameBuilderTests.cs b/tests/MkPFS.Tests/AMPR/AMPRGameBuilderTests.cs new file mode 100644 index 0000000..3327d45 --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRGameBuilderTests.cs @@ -0,0 +1,230 @@ +using System.Security.Cryptography; +using MkPFS.Build; +using MkPFS.Build.AMPRPack; +using MkPFS.Build.Exfat; +using MkPFS.Build.PFS; +using MkPFS.Cli; +using MkPFS.Cli.Output; +using MkPFS.Core.AMPR; +using MkPFS.Core.Diagnostics; +using MkPFS.Core.Exfat; + +namespace MkPFS.Tests.AMPR; + +/// ampr game: a folder that runs from packs as is (an extension over ampr_pack.py). +public sealed class AMPRGameBuilderTests +{ + private sealed class ListLog : IMkPFSLog + { + public List Lines { get; } = []; + + public void Log(LogLevel level, string message, LogIcon icon = LogIcon.None) => Lines.Add(message); + } + + // A stand-in for AMPR Emu 0.4.2.1+: what matters is the embedded manifest magic. + private const string PackCapable = "ELF...AMPRPAK4.../app0/ampr_assets.index"; + + private static readonly string Level = string.Concat(Enumerable.Repeat("terrain mesh texture vertex ", 20_000)); + + private static string Game(TempDir dir, string library = "fakelib") + { + dir.File("game/sce_sys/param.json", """{"titleId":"PPSA01234","contentVersion":"01.000.000"}"""); + dir.File("game/eboot.bin", "eboot"); + dir.File("game/sce_module/libc.prx", "module"); + dir.File("game/data/level0.dat", Level); + dir.File("game/data/Niveau_é.dat", Level + "é"); + dir.File($"game/{library}/libSceAmpr.sprx", "old emulator without packs"); + dir.File($"game/{library}/libkernel.sprx", "kernel"); + dir.Dir("game/empty/dir"); + dir.File("libs/libSceAmpr.sprx", PackCapable); + dir.File("libs/libScePlayGo.sprx", "playgo"); + dir.File("libs/libkernel.sprx", "kernel"); + dir.File("libs/.DS_Store", "ignored"); + return Path.Combine(dir.Path, "game"); + } + + private static AMPRGameOptions Options(TempDir dir, string? image = null) => new() + { + Config = CompressAll(), + LibsDir = Path.Combine(dir.Path, "libs"), + ExfatImage = image, + }; + + // Compress every file but executables, modules, system files and the libraries. + private static AMPRPackConfig CompressAll() => AMPRProfiler.ToConfig(""" + [pack] + default_action = "compress" + + [[rule]] + action = "loose" + include = ["eboot.bin", "*/eboot.bin", "*.prx", "*.sprx", "sce_sys/*", "sce_module/*", "fakelib/*", "fakelib2/*", "ampr_emu.index"] + """); + + private static Dictionary Hashes(string root) => + Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories).ToDictionary( + f => Path.GetRelativePath(root, f), f => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(f)))); + + [Fact] + public void Output_holds_libraries_index_packs_and_loose_files_and_the_source_is_untouched() + { + using TempDir dir = new(); + string game = Game(dir); + Dictionary before = Hashes(game); + string output = Path.Combine(dir.Path, "out"); + ListLog log = new(); + + AMPRGameResult result = AMPRGameBuilder.Build(game, output, Options(dir), log); + + Assert.Equal(before, Hashes(game)); + Assert.Equal("fakelib", result.LibraryDir); + Assert.Equal(2, result.Pack.Stats.FilesPacked); + Assert.Contains(" libSceAmpr.sprx: replaced", log.Lines); + Assert.Contains(" libScePlayGo.sprx: added", log.Lines); + Assert.Contains(" libkernel.sprx: already up to date", log.Lines); + Assert.Equal(PackCapable, File.ReadAllText(Path.Combine(output, "fakelib", "libSceAmpr.sprx"))); + Assert.False(File.Exists(Path.Combine(output, "fakelib", ".DS_Store"))); + + // Packed files are only in the packs; everything else is copied; empty folders are kept. + Assert.False(File.Exists(Path.Combine(output, "data", "level0.dat"))); + Assert.False(File.Exists(Path.Combine(output, "data", "Niveau_é.dat"))); + foreach (string loose in (string[])["eboot.bin", "sce_module/libc.prx", "sce_sys/param.json"]) + { + Assert.Equal(File.ReadAllBytes(Path.Combine(game, loose)), File.ReadAllBytes(Path.Combine(output, loose))); + } + + Assert.True(Directory.Exists(Path.Combine(output, "empty", "dir"))); + Assert.Contains(log.Lines, l => l.StartsWith(" Loose files: ", StringComparison.Ordinal)); + + // The index is exactly the index of the final tree: unpack the packs over a copy and index that. + string final = dir.Dir("final"); + foreach (string file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories)) + { + string relative = Path.GetRelativePath(output, file); + if (!relative.StartsWith("ampr_", StringComparison.Ordinal)) + { + Directory.CreateDirectory(Path.GetDirectoryName(Path.Combine(final, relative))!); + File.Copy(file, Path.Combine(final, relative)); + File.SetLastWriteTimeUtc(Path.Combine(final, relative), File.GetLastWriteTimeUtc(file)); + } + } + + AMPRPackTools.Extract(result.Pack.IndexPath, final, null, overwrite: false, preserveMTime: true); + AmprIndex.Build(final, Path.Combine(dir.Path, "final.index")); + Assert.Equal(File.ReadAllBytes(Path.Combine(dir.Path, "final.index")), File.ReadAllBytes(Path.Combine(output, AmprIndex.IndexName))); + } + + [Fact] + public void A_game_with_fakelib2_gets_its_libraries_there() + { + using TempDir dir = new(); + string game = Game(dir, "fakelib2"); + string output = Path.Combine(dir.Path, "out"); + + AMPRGameResult result = AMPRGameBuilder.Build(game, output, Options(dir), new ListLog()); + + Assert.Equal("fakelib2", result.LibraryDir); + Assert.Equal(PackCapable, File.ReadAllText(Path.Combine(output, "fakelib2", "libSceAmpr.sprx"))); + Assert.False(Directory.Exists(Path.Combine(output, "fakelib"))); + } + + [Fact] + public void Inputs_that_cannot_give_a_playable_folder_are_rejected_before_anything_is_written() + { + using TempDir dir = new(); + string game = Game(dir); + string output = Path.Combine(dir.Path, "out"); + string Error(AMPRGameOptions options, string target) => + Assert.Throws(() => AMPRGameBuilder.Build(game, target, options, new ListLog())).Message; + + // The game's own emulator cannot read packs. + Assert.StartsWith("fakelib/libSceAmpr.sprx cannot read asset packs", Error(Options(dir) with { LibsDir = null }, output), StringComparison.Ordinal); + Assert.False(Directory.Exists(output)); + + Assert.StartsWith("library folder not found", Error(Options(dir) with { LibsDir = Path.Combine(dir.Path, "nope") }, output), StringComparison.Ordinal); + Assert.Equal("the game folder and the output folder must not contain each other", Error(Options(dir), Path.Combine(game, "out"))); + + dir.File("busy/file.txt"); + Assert.StartsWith("the output folder must be new or empty", Error(Options(dir), Path.Combine(dir.Path, "busy")), StringComparison.Ordinal); + + Assert.Equal("the exFAT image must be outside the game and output folders", Error(Options(dir, Path.Combine(output, "x.exfat")), output)); + + dir.File("game/ampr_assets.index", "old manifest"); + Assert.StartsWith("the game folder already has ampr_assets.index", Error(Options(dir), output), StringComparison.Ordinal); + } + + [Fact] + public void A_failed_build_leaves_the_output_folder_empty() + { + using TempDir dir = new(); + string game = Game(dir); + string output = dir.Dir("out"); + AMPRGameOptions options = Options(dir) with { Config = CompressAll() }; + options.Config.RequiredPacked = ["eboot.bin"]; // the rules keep it loose, so the packer fails + + Assert.ThrowsAny(() => AMPRGameBuilder.Build(game, output, options, new ListLog())); + + Assert.True(Directory.Exists(output)); + Assert.Empty(Directory.EnumerateFileSystemEntries(output)); + } + + [Fact] + public void The_exfat_image_holds_the_playable_folder() + { + using TempDir dir = new(); + string game = Game(dir); + string output = Path.Combine(dir.Path, "out"); + + AMPRGameResult result = AMPRGameBuilder.Build(game, output, Options(dir, dir.Path), new ListLog()); + + Assert.Equal(Path.Combine(dir.Path, "PPSA01234.exfat"), result.ExfatImage); + using FileStream stream = File.OpenRead(result.ExfatImage!); + ExfatReader reader = new(stream); + Assert.Equal(64 * 1024, reader.Geometry.ClusterSize); + List names = [.. reader.RootEntries().Select(e => e.Name).Order(StringComparer.Ordinal)]; + Assert.Equal(["ampr_assets-000.pak", "ampr_assets.index", "ampr_assets.index.crc", "ampr_emu.index", "data", "eboot.bin", "empty", "fakelib", "sce_module", "sce_sys"], names); + } + + [Fact] + public void The_exfat_image_can_keep_free_space_for_writes() + { + using TempDir dir = new(); + string game = Game(dir); + string output = Path.Combine(dir.Path, "out"); + + AMPRGameResult result = AMPRGameBuilder.Build(game, output, Options(dir, dir.Path) with { ExfatFreeBytes = 3 * 1024 * 1024 + 1 }, new ListLog()); + + using FileStream stream = File.OpenRead(result.ExfatImage!); + ExfatReader reader = new(stream); + Assert.Contains(reader.RootEntries(), e => e.Name == "eboot.bin"); + ExfatImageWriter tight = ExfatImageWriter.Plan(output); + ExfatImageWriter roomy = ExfatImageWriter.Plan(output, null, 3 * 1024 * 1024 + 1); + Assert.Equal(0, tight.FreeBytes); + Assert.Equal(49L * 64 * 1024, roomy.FreeBytes); // 3 MiB + 1 byte, rounded up to 64 KiB clusters + Assert.Equal(stream.Length, roomy.ImageSize); + Assert.True(roomy.ImageSize >= tight.ImageSize + roomy.FreeBytes); + } + + [Fact] + public void Cli_game_builds_the_folder_and_reports_each_step() + { + using TempDir dir = new(); + string game = Game(dir); + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + CliContext ctx = new(stdout, stderr, useColor: false, utf8: false, progress: false); + + string toml = dir.File("rules.toml", "[pack]\ndefault_action = \"compress\"\n[[rule]]\naction = \"loose\"\ninclude = [\"eboot.bin\", \"*.prx\", \"*.sprx\", \"sce_sys/*\"]\n"); + int exit = MkPFSCli.Run(["ampr", "game", "--root", game, "--output", Path.Combine(dir.Path, "out"), "--fakelib", Path.Combine(dir.Path, "libs"), "--config", toml], ctx); + + Assert.True(exit == 0, stdout.ToString() + stderr.ToString()); + Assert.StartsWith(global::MkPFS.Cli.Commands.AmprCommand.ExperimentalWarning + $"\nRules: TOML profile {toml}\n", stdout.ToString(), StringComparison.Ordinal); + foreach (string step in (string[])["[1/5] Libraries: fakelib/", "[2/5] Writing ampr_emu.index", "[3/5] Packing", "[4/5] Copying loose files", "[5/5] Verifying"]) + { + Assert.Contains(step + "\n", stdout.ToString(), StringComparison.Ordinal); + } + + exit = MkPFSCli.Run(["ampr", "game", "--root", game, "--output", Path.Combine(dir.Path, "out2"), "--config", toml], ctx); + Assert.Equal(2, exit); + Assert.EndsWith("error: fakelib/libSceAmpr.sprx cannot read asset packs; use AMPR Emu 0.4.2.1 or newer from https://github.com/drakmor/ampr_emu/releases\n", stderr.ToString(), StringComparison.Ordinal); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRGlobTests.cs b/tests/MkPFS.Tests/AMPR/AMPRGlobTests.cs new file mode 100644 index 0000000..6965675 --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRGlobTests.cs @@ -0,0 +1,100 @@ +using MkPFS.Core.AMPR; +using MkPFS.Core.Util; + +namespace MkPFS.Tests.AMPR; + +/// fnmatch, float repr, splitlines and JSON helpers; expected values come from CPython 3.11. +public sealed class AMPRGlobTests +{ + [Theory] + [InlineData("a/b.txt", "*", true)] + [InlineData("a/b.txt", "*.txt", true)] + [InlineData("a/b.txt", "a/*", true)] + [InlineData("a/b/c", "a/**", true)] + [InlineData("a/b/c", "**/c", true)] + [InlineData("c", "**/c", false)] + [InlineData("x.BIN", "*.bin", false)] + [InlineData("a[1]", "a[[]1]", true)] + [InlineData("a1", "a[0-9]", true)] + [InlineData("a-", "a[a-]", true)] + [InlineData("ab", "a[!b]", false)] + [InlineData("ac", "a[!b]", true)] + [InlineData("a]", "a[]]", true)] + [InlineData("a!", "a[!]", false)] + [InlineData("a[!]", "a[!]", true)] + [InlineData("a", "a[]", false)] + [InlineData("ab", "a?", true)] + [InlineData("a.b", "a.b", true)] + [InlineData("axb", "a.b", false)] + [InlineData("a+b", "a+b", true)] + [InlineData("a\\b", "a\\b", true)] + [InlineData("a^", "a[^]", true)] + [InlineData("a^", "a[\\^]", true)] + [InlineData("a\\", "a[\\^]", true)] + [InlineData("ab", "a[z-a]", false)] + [InlineData("a-", "a[z-a-]", true)] + [InlineData("a&", "a[&&]", true)] + [InlineData("a~", "a[~~]", true)] + [InlineData("a\nb", "a*b", true)] + [InlineData("a/b", "a?b", true)] + [InlineData("a", "[a-c-e]", true)] + [InlineData("d", "[a-c-e]", false)] + [InlineData("-", "[a-c-e]", true)] + [InlineData("\u00e9", "?", true)] + [InlineData("a", "a**", true)] + [InlineData("", "*", true)] + [InlineData("a[", "a[", true)] + [InlineData("a[b", "a[b", true)] + [InlineData("a]x", "a[]x]", false)] + [InlineData("a]", "a[!]x]", false)] + [InlineData("ay", "a[!]x]", true)] + public void FnMatchCase_matches_python(string name, string pattern, bool expected) => + Assert.Equal(expected, AMPRGlob.FnMatchCase(name, pattern)); + + [Fact] + public void Matches_normalizes_separators_and_leading_slashes() + { + Assert.True(AMPRGlob.Matches("\\assets\\a.bin", ["/assets/*.bin"])); + Assert.True(AMPRGlob.Matches("assets/a.bin", ["x", "assets\\*"])); + Assert.False(AMPRGlob.Matches("assets/a.bin", [])); + } + + [Theory] + [InlineData(0.01, "0.01")] + [InlineData(0.125, "0.125")] + [InlineData(1e-05, "1e-05")] + [InlineData(0.0001, "0.0001")] + [InlineData(1e16, "1e+16")] + [InlineData(1e15, "1000000000000000.0")] + [InlineData(123456789012345678.0, "1.2345678901234568e+17")] + [InlineData(0.0, "0.0")] + [InlineData(-0.0, "-0.0")] + [InlineData(1.0, "1.0")] + [InlineData(0.30000000000000004, "0.30000000000000004")] + [InlineData(5e-324, "5e-324")] + [InlineData(1.7976931348623157e308, "1.7976931348623157e+308")] + [InlineData(2.5, "2.5")] + [InlineData(100.0, "100.0")] + [InlineData(1e22, "1e+22")] + [InlineData(-12.5, "-12.5")] + public void FloatRepr_matches_python(double value, string expected) => Assert.Equal(expected, PythonText.FloatRepr(value)); + + [Fact] + public void SplitLines_matches_python() + { + Assert.Equal(["a", "b", "", "c", "d", "e"], PythonText.SplitLines("a\nb\r\n\rc\u2028d\u0085e\n")); + Assert.Empty(PythonText.SplitLines(string.Empty)); + } + + [Fact] + public void Json_matches_python_dumps_sorted_compact() + { + Dictionary value = new() + { + ["b"] = new List { 1L, 0.5, true, null, "\u00e9\"\\\n\u007f" }, + ["a"] = new Dictionary { ["y"] = 1e-05, ["x"] = 0.0 }, + }; + + Assert.Equal("{\"a\":{\"x\":0.0,\"y\":1e-05},\"b\":[1,0.5,true,null,\"\\u00e9\\\"\\\\\\n\\u007f\"]}", PythonSortedJson.Compact(value)); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRPackBuilderTests.cs b/tests/MkPFS.Tests/AMPR/AMPRPackBuilderTests.cs new file mode 100644 index 0000000..ce11804 --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRPackBuilderTests.cs @@ -0,0 +1,185 @@ +using System.Text; +using MkPFS.Build; +using MkPFS.Build.AMPRPack; +using MkPFS.Core.AMPR; + +namespace MkPFS.Tests.AMPR; + +/// Builder behavior the golden corpus does not cover: placement, sampling, republishing and failures. +public sealed class AMPRPackBuilderTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-builder-" + Guid.NewGuid().ToString("N")); + + public AMPRPackBuilderTests() => Directory.CreateDirectory(_dir); + + public void Dispose() => Directory.Delete(_dir, recursive: true); + + [Theory] + [InlineData(0, 5, new long[0])] + [InlineData(3, 5, new long[] { 0, 1, 2 })] + [InlineData(5, 5, new long[] { 0, 1, 2, 3, 4 })] + [InlineData(10, 1, new long[] { 5 })] + [InlineData(10, 3, new long[] { 0, 4, 9 })] + [InlineData(7, 2, new long[] { 0, 6 })] + [InlineData(1000, 7, new long[] { 0, 166, 333, 499, 666, 832, 999 })] + public void Sample_block_indices_match_python(long total, long samples, long[] expected) => + Assert.Equal(expected, AMPRPackPlanner.SampleBlockIndices(total, samples)); + + [Fact] + public void Placement_keeps_small_chunks_inside_one_page_and_large_chunks_page_aligned() + { + AMPRPackConfig config = AMPRPackConfig.Load(null); + AMPRPackLayout layout = new(config, _dir); + AMPRGroupConfig group = new("default"); + try + { + AMPRPackVolumeWriter writer = layout.WriterFor(group, 0, 100, false, "mixed", false); + Assert.Equal(65536, writer.PayloadOffset); + + (long first, int firstFlags) = writer.Write(new byte[60000], "mixed"); + Assert.Equal((65536L, AMPRPackFormat.ChunkFlagPageContained | AMPRPackFormat.ChunkFlagPageAligned), (first, firstFlags)); + + // 60000 rounds up to 60032; 10000 more would cross the page at 131072, so it moves to the next page. + (long second, _) = writer.Write(new byte[10000], "mixed"); + Assert.Equal(131072, second); + + // A full-page chunk starts on a page boundary; a streaming chunk stays dense at 64-byte alignment. + (long third, int thirdFlags) = writer.Write(new byte[65536], "mixed"); + Assert.Equal((196608L, AMPRPackFormat.ChunkFlagPageContained | AMPRPackFormat.ChunkFlagPageAligned), (third, thirdFlags)); + (long fourth, int fourthFlags) = writer.Write(new byte[100], "streaming"); + Assert.Equal((262144L, AMPRPackFormat.ChunkFlagPageContained | AMPRPackFormat.ChunkFlagPageAligned), (fourth, fourthFlags)); + (long fifth, int fifthFlags) = writer.Write(new byte[65536], "streaming"); + Assert.Equal((262144L + 128, 0), (fifth, fifthFlags)); + } + finally + { + layout.Abort(); + } + } + + private string MakeTree(params (string Path, byte[] Data)[] files) + { + string root = Path.Combine(_dir, "app0"); + foreach ((string path, byte[] data) in files) + { + string full = Path.Combine(root, path); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + File.WriteAllBytes(full, data); + } + + AmprIndex.Build(root, Path.Combine(root, AmprIndex.IndexName)); + return root; + } + + private string WriteConfig(string toml) + { + string path = Path.Combine(_dir, "config.toml"); + File.WriteAllText(path, toml, new UTF8Encoding(false)); + return path; + } + + private static byte[] Text(int size) => Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("asset texture mesh ", (size / 19) + 1)))[..size]; + + [Fact] + public void Rebuild_replaces_the_set_removes_a_stale_runtime_and_leaves_no_temporaries() + { + string root = MakeTree(("data/a.txt", Text(200_000)), ("data/b.txt", Text(1000))); + string index = Path.Combine(root, AmprIndex.IndexName); + string output = Path.Combine(_dir, "out"); + AMPRPackConfig withRuntime = AMPRPackConfig.Load(WriteConfig( + "[pack]\ndefault_action = \"compress\"\n[runtime]\ndecoded_cache_bytes = \"64MiB\"\nphysical_cache_bytes = 0\nworkers = 2\nlatency_reserve_workers = 0\n")); + + AMPRBuildResult first = AMPRPackBuilder.Build(root, index, output, withRuntime); + Assert.True(File.Exists(first.IndexPath + ".runtime")); + byte[] firstBuildId = AMPRPackManifest.Load(first.IndexPath).BuildId; + + AMPRPackConfig plain = AMPRPackConfig.Load(WriteConfig("[pack]\ndefault_action = \"compress\"\ndeduplicate = false\n")); + AMPRBuildResult second = AMPRPackBuilder.Build(root, index, output, plain); + + Assert.False(File.Exists(second.IndexPath + ".runtime")); + Assert.Equal(["ampr_assets-000.pak", "ampr_assets.index", "ampr_assets.index.crc"], Directory.GetFiles(output).Select(Path.GetFileName).Order(StringComparer.Ordinal)); + AMPRPackManifest manifest = AMPRPackManifest.Load(second.IndexPath); + Assert.Equal(2, manifest.Files.Count(f => f.IsPacked)); + Assert.NotEqual(firstBuildId, manifest.BuildId); + } + + [Fact] + public void Failed_build_keeps_the_previous_set_and_removes_temporaries() + { + string root = MakeTree(("data/a.txt", Text(100_000)), ("data/b.txt", Text(5000))); + string index = Path.Combine(root, AmprIndex.IndexName); + string output = Path.Combine(_dir, "out"); + AMPRPackConfig config = AMPRPackConfig.Load(WriteConfig("[pack]\ndefault_action = \"compress\"\n")); + AMPRPackBuilder.Build(root, index, output, config); + Dictionary before = Directory.GetFiles(output).ToDictionary(p => p, File.ReadAllBytes); + + File.Delete(Path.Combine(root, "data", "b.txt")); + AMPRPackException error = Assert.Throws(() => AMPRPackBuilder.Build(root, index, output, config)); + + Assert.Equal("source is missing or unsafe for file id 2: data/b.txt", error.Message); + Assert.Equal(before.Keys.Order(StringComparer.Ordinal), Directory.GetFiles(output).Order(StringComparer.Ordinal)); + Assert.All(before, kv => Assert.Equal(kv.Value, File.ReadAllBytes(kv.Key))); + } + + [Fact] + public void Missing_source_is_left_loose_with_allow_missing() + { + string root = MakeTree(("data/a.txt", Text(100_000)), ("data/b.txt", Text(5000))); + File.Delete(Path.Combine(root, "data", "b.txt")); + AMPRPackConfig config = AMPRPackConfig.Load(WriteConfig("[pack]\ndefault_action = \"compress\"\n")); + + AMPRBuildResult result = AMPRPackBuilder.Build(root, Path.Combine(root, AmprIndex.IndexName), Path.Combine(_dir, "out"), config, allowMissing: true); + + Assert.Equal(["file id 2 left loose because source is missing: data/b.txt"], result.Warnings); + Assert.Equal(["data/b.txt"], result.Stats.LoosePaths); + } + + [Fact] + public void Size_mismatch_with_the_index_is_rejected() + { + string root = MakeTree(("data/a.txt", Text(1000))); + File.WriteAllBytes(Path.Combine(root, "data", "a.txt"), Text(999)); + AMPRPackConfig config = AMPRPackConfig.Load(WriteConfig("[pack]\ndefault_action = \"compress\"\n")); + + AMPRPackException error = Assert.Throws( + () => AMPRPackBuilder.Build(root, Path.Combine(root, AmprIndex.IndexName), Path.Combine(_dir, "out"), config)); + Assert.Equal("AMPRIDX3 size mismatch for data/a.txt: index=1000, disk=999", error.Message); + } + + [Fact] + public void Block_larger_than_the_volume_limit_is_rejected() + { + string root = MakeTree(("data/r.bin", [.. Enumerable.Range(0, 1 << 20).Select(i => (byte)(i * 2654435761U >> 24))])); + AMPRPackConfig config = AMPRPackConfig.Load(WriteConfig( + "[pack]\ndefault_action = \"store\"\ndefault_block_size = \"1MiB\"\n[groups.default]\nmax_pack_size = \"128KiB\"\n")); + + AMPRPackException error = Assert.Throws( + () => AMPRPackBuilder.Build(root, Path.Combine(root, AmprIndex.IndexName), Path.Combine(_dir, "out"), config)); + Assert.Equal("one block (1048576 bytes) exceeds max pack size for group default", error.Message); + Assert.Empty(Directory.GetFiles(Path.Combine(_dir, "out"))); + } + + [Fact] + public void Pattern_without_a_volume_field_fails_on_rollover() + { + string root = MakeTree(("data/a.bin", new byte[300_000]), ("data/b.bin", new byte[300_000])); + AMPRPackConfig config = AMPRPackConfig.Load(WriteConfig( + "[pack]\ndefault_action = \"store\"\npack_pattern = \"same.pak\"\ndeduplicate = false\n[groups.default]\nmax_pack_size = \"256KiB\"\n")); + + AMPRPackException error = Assert.Throws( + () => AMPRPackBuilder.Build(root, Path.Combine(root, AmprIndex.IndexName), Path.Combine(_dir, "out"), config)); + Assert.Equal("pack_pattern produced duplicate name: same.pak", error.Message); + } + + [Fact] + public void Previous_outputs_inside_the_root_stay_loose() + { + string root = MakeTree(("data/a.txt", Text(10_000)), ("ampr_assets.index", [1, 2, 3]), ("ampr_assets-000.pak", [4]), ("x/ampr_assets.index.crc", [5])); + AMPRPackConfig config = AMPRPackConfig.Load(WriteConfig("[pack]\ndefault_action = \"compress\"\n")); + + AMPRBuildResult result = AMPRPackBuilder.Build(root, Path.Combine(root, AmprIndex.IndexName), Path.Combine(_dir, "out"), config); + + Assert.Equal(["ampr_assets-000.pak", "ampr_assets.index"], result.Stats.LoosePaths.Order(StringComparer.Ordinal)); + Assert.Equal(2, result.Stats.FilesPacked); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRPackCliTests.cs b/tests/MkPFS.Tests/AMPR/AMPRPackCliTests.cs new file mode 100644 index 0000000..2dd91bb --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRPackCliTests.cs @@ -0,0 +1,133 @@ +using MkPFS.Build; +using MkPFS.Build.AMPRPack; +using MkPFS.Cli; +using MkPFS.Cli.Output; +using MkPFS.Core.AMPR; + +namespace MkPFS.Tests.AMPR; + +/// The rules options of ampr pack and ampr game and their warnings (extensions over ampr_pack.py). +public sealed class AMPRPackCliTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-cli-" + Guid.NewGuid().ToString("N")); + + public AMPRPackCliTests() => Directory.CreateDirectory(_dir); + + public void Dispose() => Directory.Delete(_dir, recursive: true); + + // Compress every file but the ones remove-sources protects (the generic profile shared outside MkPFS). + private const string CompressAll = """ + [pack] + default_action = "compress" + + [[rule]] + action = "loose" + include = ["eboot.bin", "*.prx", "*.sprx", "sce_sys/*", "fakelib/*", "fakelib2/*", "ampr_emu.index"] + """; + + private string Profile() + { + string path = Path.Combine(_dir, "profile.toml"); + File.WriteAllText(path, CompressAll); + return path; + } + + private string Game(string? emulator = "fakelib") + { + string root = Path.Combine(_dir, "app0"); + List<(string, string)> files = [("eboot.bin", "eboot"), ("sce_sys/param.json", "{}"), ("lib.prx", "prx"), ("data/a.dat", new string('a', 50_000))]; + if (emulator is not null) + { + files.Add(($"{emulator}/libSceAmpr.sprx", "sprx")); + } + + foreach ((string path, string text) in files) + { + string full = Path.Combine(root, path); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + File.WriteAllText(full, text); + } + + AmprIndex.Build(root, Path.Combine(root, AmprIndex.IndexName)); + return root; + } + + private static (int Exit, string Out, string Err) Run(params string[] args) + { + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + int exit = MkPFSCli.Run(["ampr", .. args], new CliContext(stdout, stderr, useColor: false, utf8: false, progress: false)); + return (exit, stdout.ToString(), stderr.ToString()); + } + + [Fact] + public void Cli_profile_packs_and_the_removal_plan_accepts_it() + { + string root = Game(); + string output = Path.Combine(_dir, "out"); + + (int exit, string stdout, string stderr) = Run( + "pack", "--root", root, "--ampr-index", Path.Combine(root, AmprIndex.IndexName), "--output", output, "--config", Profile()); + + Assert.Equal(0, exit); + Assert.Contains("\"files_packed\": 1,", stdout, StringComparison.Ordinal); + Assert.Equal(string.Empty, stderr); + Assert.Equal(["data/a.dat"], AMPRPackMaintenance.RemovalPlan(Path.Combine(output, "ampr_assets.index"), root).Paths); + } + + [Fact] + public void Cli_warns_when_nothing_is_packed_and_rejects_config_with_traces() + { + string root = Game(); + string index = Path.Combine(root, AmprIndex.IndexName); + string output = Path.Combine(_dir, "out"); + + (int exit, string stdout, string stderr) = Run("pack", "--root", root, "--ampr-index", index, "--output", output); + Assert.Equal(0, exit); + Assert.Contains("\"files_packed\": 0,", stdout, StringComparison.Ordinal); + Assert.Equal(global::MkPFS.Cli.Commands.AmprCommand.NothingPackedWarning + "\n", stderr); + + (exit, stdout, stderr) = Run("pack", "--root", root, "--ampr-index", index, "--output", output, "--config", Profile(), "--traces", _dir); + Assert.Equal((2, string.Empty, "error: --traces cannot be used with --config\n"), (exit, stdout, stderr)); + } + + [Fact] + public void Cli_game_needs_a_profile_or_traces() + { + (int exit, string stdout, string stderr) = Run("game", "--root", Game(), "--output", Path.Combine(_dir, "out")); + + Assert.Equal(2, exit); + Assert.Equal(global::MkPFS.Cli.Commands.AmprCommand.ExperimentalWarning + "\n", stdout); + Assert.EndsWith("error: " + global::MkPFS.Cli.Commands.AmprCommand.NoRulesError + "\n", stderr, StringComparison.Ordinal); + Assert.False(Directory.Exists(Path.Combine(_dir, "out"))); + } + + [Theory] + [InlineData(null, true)] + [InlineData("fakelib", false)] + [InlineData("fakelib2", false)] + public void Cli_warns_when_packs_are_built_for_a_folder_without_ampr_emu(string? emulator, bool warns) + { + string root = Game(emulator); + + (int exit, _, string stderr) = Run( + "pack", "--root", root, "--ampr-index", Path.Combine(root, AmprIndex.IndexName), "--output", Path.Combine(_dir, "out"), "--config", Profile()); + + Assert.Equal(0, exit); + Assert.Equal(warns ? global::MkPFS.Cli.Commands.AmprCommand.NoEmulatorWarning + "\n" : string.Empty, stderr); + } + + [Fact] + public void Runtime_limits_of_the_default_emulator_build_are_reported() + { + AMPRBuildStats stats = new() { FilesTotal = AMPRBuildResult.RuntimeMaxFiles, Chunks = AMPRBuildResult.RuntimeMaxChunks }; + Assert.Empty(new AMPRBuildResult("i", stats, [], AMPRBuildResult.RuntimeMaxVolumes).RuntimeLimitWarnings()); + + stats = new() { FilesTotal = AMPRBuildResult.RuntimeMaxFiles + 1, Chunks = AMPRBuildResult.RuntimeMaxChunks + 1 }; + IReadOnlyList warnings = new AMPRBuildResult("i", stats, [], AMPRBuildResult.RuntimeMaxVolumes + 1).RuntimeLimitWarnings(); + Assert.Equal(3, warnings.Count); + Assert.StartsWith("2000001 files exceed", warnings[0], StringComparison.Ordinal); + Assert.StartsWith("16000001 chunks exceed", warnings[1], StringComparison.Ordinal); + Assert.StartsWith("1025 pack volumes exceed", warnings[2], StringComparison.Ordinal); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRPackConfigTests.cs b/tests/MkPFS.Tests/AMPR/AMPRPackConfigTests.cs new file mode 100644 index 0000000..02cd616 --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRPackConfigTests.cs @@ -0,0 +1,204 @@ +using System.Text; +using MkPFS.Build.AMPRPack; +using MkPFS.Core.AMPR; + +namespace MkPFS.Tests.AMPR; + +/// ampr_pack load_config, select_rule and _pack_output_glob; expectations come from Python. +public sealed class AMPRPackConfigTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-config-" + Guid.NewGuid().ToString("N")); + + public AMPRPackConfigTests() => Directory.CreateDirectory(_dir); + + public void Dispose() => Directory.Delete(_dir, recursive: true); + + private AMPRPackConfig Load(string toml) + { + string path = Path.Combine(_dir, "c.toml"); + File.WriteAllText(path, toml + "\n", new UTF8Encoding(false)); + return AMPRPackConfig.Load(path); + } + + [Theory] + [InlineData("ampr_assets-{id:03d}.pak", "ampr_assets-*.pak", "ampr_assets-000.pak", "ampr_assets-007.pak")] + [InlineData("ampr_assets-{group}-lane{lane:02d}-vol{volume:02d}-{id:03d}.pak", "ampr_assets-*-lane*-vol*-*.pak", "ampr_assets-g-lane00-vol00-000.pak", "ampr_assets-assets-lane03-vol12-007.pak")] + [InlineData("{id}", "*", "0", "7")] + [InlineData("a{{b}}{id}", "a{b}*", "a{b}0", "a{b}7")] + [InlineData("{id!r}.pak", "*.pak", "0.pak", "7.pak")] + [InlineData("{group!r}.pak", "*.pak", "'g'.pak", "'assets'.pak")] + [InlineData("{id:>5}", "*", " 0", " 7")] + [InlineData("{group:05}", "*", "g0000", "assets")] + [InlineData("{id:+d}", "*", "+0", "+7")] + [InlineData("{id:x}", "*", "0", "7")] + [InlineData("{id:#x}", "*", "0x0", "0x7")] + [InlineData("{id:.2f}", "*", "0.00", "7.00")] + [InlineData("a/{id}.pak", "a/*.pak", "a/0.pak", "a/7.pak")] + [InlineData("{id:_>4}", "*", "___0", "___7")] + [InlineData("{id:,}", "*", "0", "7")] + [InlineData("sub\\{id}.pak", "sub/*.pak", "sub\\0.pak", "sub\\7.pak")] + [InlineData("{id: 3d}", "*", " 0", " 7")] + [InlineData("{id:=+5d}", "*", "+ 0", "+ 7")] + [InlineData("{id:^5}", "*", " 0 ", " 7 ")] + [InlineData("{group:^5}", "*", " g ", "assets")] + [InlineData("{id!s:>3}", "*", " 0", " 7")] + [InlineData("{id!a}", "*", "0", "7")] + [InlineData("{id:%}", "*", "0.000000%", "700.000000%")] + [InlineData("{id:e}", "*", "0.000000e+00", "7.000000e+00")] + [InlineData("{id:b}", "*", "0", "111")] + [InlineData("{id:#o}", "*", "0o0", "0o7")] + [InlineData("{lane:08_b}", "*", "000_0000", "000_0011")] + public void Pack_pattern_globs_and_renders_like_python(string pattern, string glob, string zero, string sample) + { + AMPRPackPattern parsed = AMPRPackPattern.Parse(pattern); + Assert.Equal(glob, parsed.OutputGlob); + Assert.Equal(zero, parsed.Render("g", 0, 0, 0)); + Assert.Equal(sample, parsed.Render("assets", 3, 12, 7)); + } + + [Theory] + [InlineData("{}", "pack_pattern uses unsupported field ''; allowed fields are group, lane, volume and id")] + [InlineData("{0}", "pack_pattern uses unsupported field '0'; allowed fields are group, lane, volume and id")] + [InlineData("{id.real}", "pack_pattern uses unsupported field 'id.real'; allowed fields are group, lane, volume and id")] + [InlineData("{id[0]}", "pack_pattern uses unsupported field 'id[0]'; allowed fields are group, lane, volume and id")] + [InlineData("{ id }", "pack_pattern uses unsupported field ' id '; allowed fields are group, lane, volume and id")] + [InlineData("x}y", "invalid pack_pattern: 'x}y'")] + [InlineData("x{", "invalid pack_pattern: 'x{'")] + [InlineData("{id", "invalid pack_pattern: '{id'")] + [InlineData("{id!}", "invalid pack_pattern: '{id!}'")] + [InlineData("{id!rr}", "invalid pack_pattern: '{id!rr}'")] + [InlineData("{id!x}.pak", "unsafe or invalid pack_pattern: '{id!x}.pak'")] + [InlineData("{id:{x}}", "unsafe or invalid pack_pattern: '{id:{x}}'")] + [InlineData("{id:s}", "unsafe or invalid pack_pattern: '{id:s}'")] + [InlineData("{group:d}", "unsafe or invalid pack_pattern: '{group:d}'")] + [InlineData("/abs/{id}", "unsafe or invalid pack_pattern: '/abs/{id}'")] + [InlineData("../{id}", "unsafe or invalid pack_pattern: '../{id}'")] + [InlineData("", "pack_pattern cannot be empty")] + public void Pack_pattern_errors_match_python(string pattern, string message) => + Assert.Equal(message, Assert.Throws(() => AMPRPackPattern.Parse(pattern)).Message); + + [Theory] + [InlineData("pack = 1", "[pack] must be a table")] + [InlineData("[pack]\nworkers = 0", "workers must be between 1 and 256")] + [InlineData("[pack]\nworkers = \"4\"\ncompression_level = 13", "compression_level must be between 1 and 12")] + [InlineData("[pack]\nio_page_size = \"3KiB\"", "io_page_size must be a power of two between 4 KiB and 1 MiB")] + [InlineData("[pack]\npayload_alignment = 32", "payload_alignment must be a power of two between 64 B and 1 MiB; the writer automatically raises it to the selected io_page_size")] + [InlineData("[pack]\nchunk_alignment = \"128KiB\"", "chunk_alignment must be a power of two between 64 B and io_page_size")] + [InlineData("[pack]\ncompression_mode = \"zstd\"", "compression_mode must be fast or hc")] + [InlineData("[pack]\ndeduplicate_scope = \"all\"", "deduplicate_scope must be lane or group")] + [InlineData("[pack]\nacceleration = 0", "acceleration must be at least 1")] + [InlineData("[pack]\nmin_savings_ratio = 1.0", "min_savings_ratio must be in [0, 1)")] + [InlineData("[pack]\nio_neutral_min_savings_ratio = -0.1", "io_neutral_min_savings_ratio must be in [0, 1)")] + [InlineData("[pack]\nauto_loose_sample_blocks = 5000", "auto_loose_sample_blocks must be between 1 and 4096")] + [InlineData("[pack]\nauto_loose_sample_bytes = 0", "auto_loose_sample_bytes must be at least 1 byte")] + [InlineData("[pack]\nauto_loose_min_savings_ratio = 2", "auto_loose_min_savings_ratio must be in [0, 1)")] + [InlineData("[pack]\nauto_loose_max_raw_ratio = 1.5", "auto_loose_max_raw_ratio must be in [0, 1]")] + [InlineData("[pack]\nindex_name = \"../x.index\"", "unsafe index_name: '../x.index'")] + [InlineData("[pack]\npack_pattern = \"{name}.pak\"", "pack_pattern uses unsupported field 'name'; allowed fields are group, lane, volume and id")] + [InlineData("[pack]\ndefault_action = \"zip\"", "action must be compress, store or loose; got 'zip'")] + [InlineData("[pack]\ndefault_action = true", "action must be compress, store or loose; got 'true'")] + [InlineData("[pack]\nrequired_packed = [1]", "pack.required_packed must be a string or list of strings")] + [InlineData("groups = 5", "[groups] must be a table")] + [InlineData("[groups]\nbulk = 1", "[groups.bulk] must be a table")] + [InlineData("[groups.bulk]\npack_count = 0", "groups.bulk.pack_count must be between 1 and 65535")] + [InlineData("[groups.bulk]\nassignment = \"random\"", "assignment must be balanced, hash or round_robin; got 'random'")] + [InlineData("[groups.bulk]\nio_page_size = \"3KiB\"", "groups.bulk.io_page_size must be a power of two between 4 KiB and 1 MiB and a multiple of chunk_alignment")] + [InlineData("[groups.bulk]\nmax_pack_size = \"64KiB\"", "groups.bulk.max_pack_size must be at least one complete I/O page beyond the payload offset (131072 bytes)")] + [InlineData("[groups.bulk]\nstripe_group_blocks = 0", "groups.bulk.stripe_group_blocks must be at least 1")] + [InlineData("rule = \"x\"", "[[rule]] entries must form an array")] + [InlineData("rule = [1]", "rule 0 must be a table")] + [InlineData("[[rule]]\ngroup = \"nope\"", "rule 0 references unknown group 'nope'")] + [InlineData("[[rule]]\ninclude = []", "rule 0 include list cannot be empty")] + [InlineData("[[rule]]\ninclude = [1, 2]", "include must be a string or list of strings")] + [InlineData("[[rule]]\ncompression_mode = \"x\"", "rule 0 compression_mode must be fast or hc")] + [InlineData("[[rule]]\ncompression_level = 0", "rule 0 compression_level must be between 1 and 12")] + [InlineData("[[rule]]\nacceleration = 0", "rule 0 acceleration must be at least 1")] + [InlineData("[[rule]]\nmin_savings_ratio = 1", "rule 0 min_savings_ratio must be in [0, 1)")] + [InlineData("[[rule]]\nio_neutral_min_savings_ratio = 1", "rule 0 io_neutral_min_savings_ratio must be in [0, 1)")] + [InlineData("[[rule]]\nlayout = \"linear\"", "layout must be auto, random, mixed or streaming; got 'linear'")] + [InlineData("[[rule]]\naction = \"skip\"", "action must be compress, store or loose; got 'skip'")] + [InlineData("[[rule]]\ninclude_from = [\"../outside.txt\"]", "include_from entry escapes the TOML directory: ../outside.txt")] + [InlineData("[[rule]]\ninclude_from = 5", "include_from must be a string or list of strings")] + public void Config_pack_errors_match_python(string toml, string message) => + Assert.Equal(message, Assert.Throws(() => Load(toml)).Message); + + [Theory] + [InlineData("[pack]\nworkers = \"abc\"", "invalid literal for int() with base 10: 'abc'")] + [InlineData("[pack]\nmin_savings_ratio = \"x\"", "could not convert string to float: 'x'")] + [InlineData("[pack]\nmin_savings_bytes = 1.5", "'float' object has no attribute 'strip'")] + [InlineData("[runtime]\ndecoded_cache_bytes = \"64MiB\"\nphysical_cache_bytes = \"16MiB\"\nworkers = \"4\"\nlatency_reserve_workers = 1", "runtime settings must be integers")] + [InlineData("[runtime]\ndecoded_cache_bytes = \"64MiB\"\nphysical_cache_bytes = 1000\nworkers = 4\nlatency_reserve_workers = 1", "runtime cache sizes must be nonnegative 16 KiB multiples")] + [InlineData("[runtime]\ndecoded_cache_bytes = \"64MiB\"\nphysical_cache_bytes = 0\nworkers = 4\nlatency_reserve_workers = 4", "runtime workers must be 1..16 and reserve must be smaller")] + [InlineData("[[rule]]\nblock_size = \"8KiB\"", "block size must be between 16384 and 1048576 bytes")] + public void Config_value_errors_match_python(string toml, string message) => + Assert.Equal(message, Assert.Throws(() => Load(toml)).Message); + + [Theory] + [InlineData("a = 1\na = 2")] + [InlineData("[pack]\nworkers = 4\n[pack]\nx = 1")] + [InlineData("a.b = 1\n[a]\nc = 2")] + [InlineData("a = {b = 1}\n[a.c]\nd = 1")] + [InlineData("[[r]]\nx = 1\n[r]\ny = 2")] + [InlineData("x = ")] + public void Invalid_toml_is_rejected_like_tomllib(string toml) => + Assert.Throws(() => Load(toml)); + + [Fact] + public void Missing_include_from_file_names_the_file() => + Assert.StartsWith( + "cannot read include_from file missing.txt: ", + Assert.Throws(() => Load("[[rule]]\ninclude_from = [\"missing.txt\"]")).Message); + + [Fact] + public void Default_config_canonical_bytes_match_python() => + Assert.Equal( + "{\"acceleration\":1,\"auto_loose_hot_files\":false,\"auto_loose_large_files\":true,\"auto_loose_max_raw_ratio\":0.9," + + "\"auto_loose_min_file_size\":67108864,\"auto_loose_min_savings_ratio\":0.05,\"auto_loose_sample_blocks\":32," + + "\"auto_loose_sample_bytes\":16777216,\"chunk_alignment\":64,\"compression_level\":12,\"compression_mode\":\"hc\"," + + "\"deduplicate\":true,\"deduplicate_scope\":\"lane\",\"deduplicate_streaming\":false,\"default_action\":\"loose\"," + + "\"default_block_shift\":16,\"groups\":[{\"assignment\":\"balanced\",\"io_page_size\":0,\"max_pack_size\":0," + + "\"name\":\"default\",\"pack_count\":1,\"stripe_group_blocks\":8,\"stripe_large_files\":false,\"stripe_threshold\":268435456}]," + + "\"index_name\":\"ampr_assets.index\",\"io_neutral_min_savings_bytes\":8192,\"io_neutral_min_savings_ratio\":0.125," + + "\"io_page_size\":65536,\"min_savings_bytes\":64,\"min_savings_ratio\":0.01,\"pack_pattern\":\"ampr_assets-{id:03d}.pak\"," + + "\"payload_alignment\":65536,\"rules\":[],\"tool_version\":\"4.0\"}", + Encoding.UTF8.GetString(AMPRPackConfig.Load(null).CanonicalBytes())); + + [Fact] + public void Groups_sort_by_code_point_and_rules_inherit_pack_defaults() + { + AMPRPackConfig config = Load( + "[pack]\nmin_savings_ratio = 0.00001\nworkers = 2\n[groups.\"b\u00e9\"]\npack_count = 2\n[groups.a]\n[[rule]]\ninclude = \"x/\u00e9*\"\nmin_savings_ratio = 0"); + + string canonical = Encoding.UTF8.GetString(config.CanonicalBytes()); + Assert.Contains("\"groups\":[{\"assignment\":\"balanced\",\"io_page_size\":0,\"max_pack_size\":0,\"name\":\"a\"", canonical, StringComparison.Ordinal); + Assert.Contains("\"name\":\"b\\u00e9\",\"pack_count\":2", canonical, StringComparison.Ordinal); + Assert.Contains("\"min_savings_ratio\":1e-05,\"pack_pattern\"", canonical, StringComparison.Ordinal); + Assert.Contains("\"include\":[\"x/\\u00e9*\"]", canonical, StringComparison.Ordinal); + Assert.Contains("\"min_savings_ratio\":0.0,\"mode\":\"hc\"", canonical, StringComparison.Ordinal); + Assert.Equal(["a", "b\u00e9", "default"], config.SortedGroupNames()); + Assert.Equal(2, config.Workers); + } + + [Fact] + public void Last_matching_rule_wins_and_unmatched_files_use_the_default_rule() + { + AMPRPackConfig config = Load( + "[pack]\ndefault_action = \"store\"\n[[rule]]\ninclude = [\"assets/**\"]\nblock_size = \"16KiB\"\n" + + "[[rule]]\naction = \"loose\"\ninclude = [\"assets/*.bik\"]\n[[rule]]\ninclude = \"**/*.lua\"\nhot = true\nexclude = \"x/**\""); + + Assert.Equal(("compress", 14), (config.SelectRule("assets/a.bin").Action, config.SelectRule("assets/a.bin").BlockShift)); + Assert.Equal("loose", config.SelectRule("assets/intro.bik").Action); + Assert.Equal("random", config.SelectRule("scripts/a.lua").ResolvedLayout()); + Assert.Equal(("store", "mixed"), (config.SelectRule("x/a.lua").Action, config.SelectRule("x/a.lua").ResolvedLayout())); + } + + [Fact] + public void Include_from_reads_stripped_patterns_relative_to_the_toml() + { + File.WriteAllText(Path.Combine(_dir, "list.txt"), "# comment\n scripts/** \r\n\r\nassets/text_*.dat\u2028data/*\n", new UTF8Encoding(false)); + AMPRPackConfig config = Load("[[rule]]\ninclude_from = [\"list.txt\"]\nexclude = [\"x\"]\nexclude_from = \"list.txt\""); + + Assert.Equal(["scripts/**", "assets/text_*.dat", "data/*"], config.Rules[0].Include); + Assert.Equal(["x", "scripts/**", "assets/text_*.dat", "data/*"], config.Rules[0].Exclude); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRPackFormatTests.cs b/tests/MkPFS.Tests/AMPR/AMPRPackFormatTests.cs new file mode 100644 index 0000000..309cff6 --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRPackFormatTests.cs @@ -0,0 +1,348 @@ +using System.Buffers.Binary; +using MkPFS.Core.AMPR; +using MkPFS.Core.Util; + +namespace MkPFS.Tests.AMPR; + +/// AMPR pack format helpers (ampr_emu ampr_pack_format.py); expected values come from Python. +public sealed class AMPRPackFormatTests +{ + private static readonly byte[] BuildId = [.. Enumerable.Range(0, 16).Select(i => (byte)i)]; + + [Theory] + [InlineData("64KiB", 65536L)] + [InlineData("1.5MB", 1500000L)] + [InlineData("1_024", 1024L)] + [InlineData(" 8 KiB ", 8192L)] + [InlineData("2g", 2000000000L)] + [InlineData("0", 0L)] + [InlineData("1e3", 1000L)] + [InlineData("10TiB", 10995116277760L)] + [InlineData("3.99b", 3L)] + public void ParseSize_matches_python(string text, long expected) => Assert.Equal(expected, AMPRSize.Parse(text)); + + [Theory] + [InlineData("", "empty size")] + [InlineData("KiB", "invalid size: KiB")] + [InlineData("12xb", "unknown size suffix: xb")] + [InlineData("-1", "size must be non-negative")] + [InlineData("1e400", "invalid size: 1e400")] + [InlineData("abc5", "invalid size: abc5")] + public void ParseSize_errors_match_python(string text, string message) => + Assert.Equal(message, Assert.Throws(() => AMPRSize.Parse(text)).Message); + + [Theory] + [InlineData("16KiB", 14)] + [InlineData("64KiB", 16)] + [InlineData("1MiB", 20)] + public void BlockShift_accepts_supported_sizes(string text, int shift) => Assert.Equal(shift, AMPRSize.BlockShift(text)); + + [Theory] + [InlineData("48KiB", "block size must be a power of two, got 49152")] + [InlineData("8KiB", "block size must be between 16384 and 1048576 bytes")] + [InlineData("2MiB", "block size must be between 16384 and 1048576 bytes")] + public void BlockShift_rejects_other_sizes(string text, string message) => + Assert.Equal(message, Assert.Throws(() => AMPRSize.BlockShift(text)).Message); + + [Fact] + public void Fnv1a64_matches_python() + { + Assert.Equal(0xcbf29ce484222325UL, AMPRAssetPath.Fnv1a64([])); + Assert.Equal(0xaf63dc4c8601ec8cUL, AMPRAssetPath.Fnv1a64("a"u8)); + } + + [Theory] + [InlineData("/app0/Assets/X.bin", "/app0/Assets/X.bin", 0xe7738fe70d6ed8f9UL)] + [InlineData("app0\\data\\a.txt", "/app0/data/a.txt", 0x6d353fb6a0bd22eaUL)] + [InlineData("/app0/donn\u00e9es.txt", "/app0/donn\u00e9es.txt", 0x19e1db4b3788e9f7UL)] + [InlineData("/APP0/./a//b/../c", "/APP0/a/c", 0)] + public void Canonical_path_and_hash_match_python(string path, string canonical, ulong hash) + { + Assert.Equal(canonical, AMPRAssetPath.Canonical(path)); + if (hash != 0) + { + Assert.Equal(hash, AMPRAssetPath.Hash(path)); + } + } + + [Theory] + [InlineData("/app0/../../x", "path escapes root: /app0/../../x")] + [InlineData("/other/x", "asset is outside /app0: /other/x")] + [InlineData("/app0/a\0b", "NUL in path")] + public void Canonical_path_errors_match_python(string path, string message) => + Assert.Equal(message, Assert.Throws(() => AMPRAssetPath.Canonical(path)).Message); + + [Fact] + public void Relative_path_strips_app0() + { + Assert.Equal("a/B.bin", AMPRAssetPath.Relative("/app0/a/B.bin")); + Assert.Equal(string.Empty, AMPRAssetPath.Relative("/App0")); + } + + [Theory] + [InlineData("")] + [InlineData("a\\b")] + [InlineData("/a")] + [InlineData("a/")] + [InlineData("a//b")] + [InlineData("a/./b")] + [InlineData("a/../b")] + public void SafeOutputPath_rejects_unsafe_names(string relative) => + Assert.StartsWith("unsafe output path: ", Assert.Throws(() => AMPRAssetPath.SafeOutputPath(".", relative)).Message); + + [Fact] + public void SafeOutputPath_joins_safe_names() => + Assert.Equal(Path.Combine("root", "a", "b.pak"), AMPRAssetPath.SafeOutputPath("root", "a/b.pak")); + + [Theory] + [InlineData("a'b", "\"a'b\"")] + [InlineData("a\"b", "'a\"b'")] + [InlineData("a'\"b", "'a\\'\"b'")] + [InlineData("\0\n\t\u00e9", "'\\x00\\n\\t\u00e9'")] + [InlineData("\u200b\U0001F600\u007f\u00a0\u2028", "'\\u200b\U0001F600\\x7f\\xa0\\u2028'")] + [InlineData("back\\slash", "'back\\\\slash'")] + public void Repr_matches_python(string value, string expected) => Assert.Equal(expected, PythonText.Repr(value)); + + [Fact] + public void BuildId_matches_python() => + Assert.Equal("481ee681277b00834eea5627b85c71b3", Convert.ToHexStringLower(AMPRBuildId.Compute([[(byte)'x'], []]))); + + [Fact] + public void Runtime_settings_encode_like_python_and_round_trip() + { + AMPRRuntimeSettings settings = new(64 * 1024 * 1024, 16 * 1024 * 1024, 4, 1); + byte[] encoded = settings.Encode(BuildId); + + Assert.Equal( + "414d5052434647310100000040000000000102030405060708090a0b0c0d0e0f0000000400000000000000010000000004000000010000000b640fd700000000", + Convert.ToHexStringLower(encoded)); + Assert.Equal(settings, AMPRRuntimeSettings.Parse(encoded, BuildId)); + } + + [Theory] + [InlineData(0, 0, "runtime workers must be 1..16 and reserve must be smaller")] + [InlineData(17, 0, "runtime workers must be 1..16 and reserve must be smaller")] + [InlineData(4, 4, "runtime workers must be 1..16 and reserve must be smaller")] + [InlineData(4, -1, "runtime workers must be 1..16 and reserve must be smaller")] + public void Runtime_settings_reject_bad_workers(long workers, long reserve, string message) => + Assert.Equal(message, Assert.Throws(() => new AMPRRuntimeSettings(0, 0, workers, reserve).Validate()).Message); + + [Fact] + public void Runtime_settings_reject_unaligned_cache() => + Assert.Equal( + "runtime cache sizes must be nonnegative 16 KiB multiples", + Assert.Throws(() => new AMPRRuntimeSettings(1000, 0, 4, 1).Validate()).Message); + + [Fact] + public void Runtime_settings_reject_other_build_and_corruption() + { + byte[] encoded = new AMPRRuntimeSettings(0, 16384, 1, 0).Encode(BuildId); + byte[] otherBuild = new byte[16]; + Assert.Throws(() => AMPRRuntimeSettings.Parse(encoded, otherBuild)); + encoded[33] ^= 1; + Assert.Equal( + "invalid runtime profile header, build ID or CRC", + Assert.Throws(() => AMPRRuntimeSettings.Parse(encoded, BuildId)).Message); + Assert.Equal("invalid runtime profile size", Assert.Throws(() => AMPRRuntimeSettings.Parse(encoded[..63], BuildId)).Message); + } + + [Fact] + public void Missing_runtime_settings_read_as_null() => + Assert.Null(AMPRRuntimeSettings.Read(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")), BuildId)); + + [Fact] + public void Data_header_encodes_like_python_and_round_trips() + { + AMPRDataHeader header = new(2, BuildId, 65536, 4096, 3); + byte[] bytes = header.ToBytes(); + + Assert.Equal( + "414d50524441543303000000400000000200000003000000000102030405060708090a0b0c0d0e0f00000100000000000010000000000000a71d34c200000000", + Convert.ToHexStringLower(bytes)); + AMPRDataHeader parsed = AMPRDataHeader.Parse(bytes, 2, BuildId, 3); + Assert.Equal((2U, 65536UL, 4096UL, 3U), (parsed.PackId, parsed.PayloadOffset, parsed.PayloadBytes, parsed.Flags)); + Assert.Equal(BuildId, parsed.BuildId); + } + + [Fact] + public void Data_header_rejects_mismatches_in_python_order() + { + byte[] bytes = new AMPRDataHeader(2, BuildId, 65536, 4096, 2).ToBytes(); + Assert.Equal("pack id mismatch", Assert.Throws(() => AMPRDataHeader.Parse(bytes, expectedPackId: 1)).Message); + Assert.Equal("pack build id mismatch", Assert.Throws(() => AMPRDataHeader.Parse(bytes, 2, new byte[16])).Message); + Assert.Equal("pack flags mismatch", Assert.Throws(() => AMPRDataHeader.Parse(bytes, 2, BuildId, 3)).Message); + Assert.Equal("pack data header is truncated", Assert.Throws(() => AMPRDataHeader.Parse(bytes[..63])).Message); + bytes[60] = 1; + Assert.Equal("invalid pack data header", Assert.Throws(() => AMPRDataHeader.Parse(bytes)).Message); + bytes[60] = 0; + bytes[41] ^= 1; + Assert.Equal("pack data header CRC mismatch", Assert.Throws(() => AMPRDataHeader.Parse(bytes)).Message); + } + + [Fact] + public void Chunk_crc_sidecar_encodes_like_python_and_round_trips() + { + byte[] bytes = AMPRChunkCrcs.Build(BuildId, [1, 0xFFFFFFFF]); + Assert.Equal( + "414d5052435243310100000030000000000102030405060708090a0b0c0d0e0f020000000000000014ff33775e9fbbb301000000ffffffff", + Convert.ToHexStringLower(bytes)); + Assert.Equal([1U, 0xFFFFFFFFU], AMPRChunkCrcs.Parse(bytes, BuildId, 2)); + Assert.Equal( + "chunk CRC sidecar build ID or chunk count mismatch", + Assert.Throws(() => AMPRChunkCrcs.Parse(bytes, BuildId, 3)).Message); + bytes[^1] ^= 1; + Assert.Equal( + "chunk CRC sidecar payload CRC mismatch", + Assert.Throws(() => AMPRChunkCrcs.Parse(bytes, BuildId, 2)).Message); + } + + [Fact] + public void Chunk_record_packs_bit_fields_and_round_trips() + { + AMPRChunkRecord chunk = new(0x123456789ABC, 65536, 0, 0x0102, AMPRPackFormat.ChunkCodecLZ4, AMPRPackFormat.ChunkFlagPageAligned); + byte[] bytes = new byte[12]; + chunk.Write(bytes); + + Assert.Equal(0x0102123456789ABCUL, BinaryPrimitives.ReadUInt64LittleEndian(bytes)); + Assert.Equal(0xFFFFU | (1U << 20) | (8U << 22), BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan(8))); + Assert.Equal(chunk, AMPRChunkRecord.Read(bytes)); + bytes[11] |= 0x40; + Assert.Equal("chunk descriptor contains reserved bits", Assert.Throws(() => AMPRChunkRecord.Read(bytes)).Message); + } + + [Theory] + [InlineData(1UL << 48, 1, 0, 0, 0, "chunk offset exceeds the 48-bit AMPRPAK4 domain")] + [InlineData(0UL, 0, 0, 0, 0, "chunk stored size is outside the AMPRPAK4 domain")] + [InlineData(0UL, (1 << 20) + 1, 0, 0, 0, "chunk stored size is outside the AMPRPAK4 domain")] + [InlineData(0UL, 1, 0x10000, 0, 0, "chunk pack id exceeds uint16")] + [InlineData(0UL, 1, 0, 4, 0, "chunk codec is outside the compact descriptor domain")] + [InlineData(0UL, 1, 0, 0, 0x10, "chunk contains unknown flags")] + public void Chunk_record_rejects_out_of_domain_fields(ulong offset, int stored, int packId, int codec, int flags, string message) => + Assert.Equal(message, Assert.Throws(() => new AMPRChunkRecord(offset, stored, 0, packId, codec, flags).Write(new byte[12])).Message); + + [Fact] + public void String_table_deduplicates_and_terminates() + { + AMPRStringTable table = new(); + Assert.Equal((0U, 3U), table.Add("abc")); + Assert.Equal((4U, 2U), table.Add("\u00e9")); + Assert.Equal((0U, 3U), table.Add("abc")); + Assert.Equal("abc\0\u00e9\0"u8.ToArray(), table.ToArray()); + Assert.Throws(() => table.Add("a\0")); + } + + // One volume with a 64 KiB page, one packed file of 70000 bytes in two LZ4 chunks, one loose file. + private static AMPRPackManifest SampleManifest( + Func? editPacked = null, + Func? editFirstChunk = null, + Func? editPack = null) + { + AMPRStringTable strings = new(); + (uint packedOffset, uint packedLength) = strings.Add("/app0/data/Big.bin"); + (uint looseOffset, uint looseLength) = strings.Add("/app0/eboot.bin"); + (uint nameOffset, uint nameLength) = strings.Add("ampr_assets-000.pak"); + const int contained = AMPRPackFormat.ChunkFlagPageContained | AMPRPackFormat.ChunkFlagPageAligned; + AMPRChunkRecord first = new(65536, 30000, 65536, 0, AMPRPackFormat.ChunkCodecLZ4, contained); + AMPRChunkRecord second = new(65536 + 30016, 4464, 4464, 0, AMPRPackFormat.ChunkCodecRaw, AMPRPackFormat.ChunkFlagPageContained); + AMPRFileRecord packed = new( + AMPRAssetPath.Hash("/app0/data/Big.bin"), 70000, 1600000000, 0, 2, packedOffset, packedLength, + AMPRPackFormat.FileFlagPacked, 16); + AMPRFileRecord loose = new(AMPRAssetPath.Hash("/app0/eboot.bin"), 1000, 1600000000, 0, 0, looseOffset, looseLength, 0, 0); + AMPRPackRecord pack = new(65536, 131072, nameOffset, nameLength, AMPRPackFormat.PackFlagIOPageLayout, 65536); + return new AMPRPackManifest( + BuildId, + 0, + [editPacked?.Invoke(packed) ?? packed, loose], + [editFirstChunk?.Invoke(first) ?? first, second], + [editPack?.Invoke(pack) ?? pack], + strings.ToArray()); + } + + [Fact] + public void Manifest_round_trips_and_derives_raw_sizes() + { + byte[] bytes = SampleManifest().Serialize(); + AMPRPackManifest parsed = AMPRPackManifest.Parse(bytes); + + Assert.Equal(bytes, parsed.Serialize()); + Assert.Equal([65536, 4464], parsed.Chunks.Select(c => c.RawSize)); + Assert.Equal("/app0/data/Big.bin", parsed.FilePath(1)); + Assert.Equal("ampr_assets-000.pak", parsed.PackName(0)); + } + + [Fact] + public void Manifest_rejects_header_and_payload_corruption() + { + byte[] bytes = SampleManifest().Serialize(); + Assert.Equal("pack index is truncated", Assert.Throws(() => AMPRPackManifest.Parse(bytes.AsSpan(0, 127))).Message); + + byte[] header = (byte[])bytes.Clone(); + header[40] ^= 1; + Assert.Equal("pack index header CRC mismatch", Assert.Throws(() => AMPRPackManifest.Parse(header)).Message); + + byte[] payload = (byte[])bytes.Clone(); + payload[^2] ^= 1; + Assert.Equal("pack index payload CRC mismatch", Assert.Throws(() => AMPRPackManifest.Parse(payload)).Message); + + byte[] magic = (byte[])bytes.Clone(); + magic[7] = (byte)'5'; + Assert.Equal("unsupported AMPR pack index", Assert.Throws(() => AMPRPackManifest.Parse(magic)).Message); + } + + public static TheoryData InvalidManifests => new() + { + { "hash", "path hash mismatch for file id 1" }, + { "shift", "invalid block shift for file id 1" }, + { "both", "file id 1 cannot be both streaming and random-access" }, + { "storeonly", "store-only file id 1 contains a compressed chunk" }, + { "misaligned", "chunk range/alignment is invalid for file id 1" }, + { "uncontained", "small page-aligned chunk lacks containment flag for file id 1" }, + { "pagesize", "invalid I/O page size for pack 0" }, + { "nolayout", "pack 0 does not declare page-aware layout" }, + { "payload", "invalid payload offset for pack 0" }, + { "streaming", "streaming flag mismatch for file id 1" }, + { "size", "logical size mismatch for file id 1" }, + }; + + [Theory] + [MemberData(nameof(InvalidManifests))] + public void Manifest_validation_messages_match_python(string edit, string message) + { + AMPRPackManifest manifest = edit switch + { + "hash" => SampleManifest(editPacked: f => f with { PathHash = f.PathHash + 1 }), + "shift" => SampleManifest(editPacked: f => f with { BlockShift = 13 }), + "both" => SampleManifest(editPacked: f => f with { Flags = f.Flags | AMPRPackFormat.FileFlagStreaming | AMPRPackFormat.FileFlagRandomAccess }), + "storeonly" => SampleManifest(editPacked: f => f with { Flags = f.Flags | AMPRPackFormat.FileFlagStoreOnly }), + "misaligned" => SampleManifest(editFirstChunk: c => c with { Offset = c.Offset + 1 }), + "uncontained" => SampleManifest(editFirstChunk: c => c with { Flags = AMPRPackFormat.ChunkFlagPageAligned }), + "pagesize" => SampleManifest(editPack: p => p with { IOPageSize = 3000 }), + "nolayout" => SampleManifest(editPack: p => p with { Flags = 0 }), + "payload" => SampleManifest(editPack: p => p with { PayloadBytes = 1000 }), + "streaming" => SampleManifest(editPacked: f => f with { Flags = f.Flags | AMPRPackFormat.FileFlagStreaming }), + "size" => SampleManifest(editPacked: f => f with { ChunkCount = 0 }), + _ => throw new ArgumentOutOfRangeException(nameof(edit)), + }; + + // Parse derives raw sizes like load_manifest, so the raw sizes always add up unless the chunk count is wrong. + Assert.Equal(message, Assert.Throws(() => AMPRPackManifest.Parse(manifest.Serialize())).Message); + } + + [Fact] + public void Manifest_rejects_extra_chunks_while_deriving_raw_sizes() => + Assert.Equal( + "AMPRPAK4 file has too many chunks", + Assert.Throws(() => AMPRPackManifest.Parse(SampleManifest(editPacked: f => f with { LogicalSize = 65536 }).Serialize())).Message); + + [Fact] + public void Index_reader_parses_the_build_writer_output() + { + byte[] index = MkPFS.Build.AmprIndex.Serialize( + [new(10, 1600000000, "/app0/a/B.bin"), new(0, 1600000001, "/app0/c.txt")]); + + Assert.Equal( + [new AMPRIndexEntry("/app0/a/B.bin", 10, 1600000000), new AMPRIndexEntry("/app0/c.txt", 0, 1600000001)], + AMPRIndexReader.Parse(index)); + Assert.Equal("AMPRIDX3 is truncated", Assert.Throws(() => AMPRIndexReader.Parse(index.AsSpan(0, 47))).Message); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRPackOperationsTests.cs b/tests/MkPFS.Tests/AMPR/AMPRPackOperationsTests.cs new file mode 100644 index 0000000..90fd220 --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRPackOperationsTests.cs @@ -0,0 +1,170 @@ +using System.Text; +using MkPFS.Build; +using MkPFS.Build.AMPRPack; +using MkPFS.Core.AMPR; + +namespace MkPFS.Tests.AMPR; + +/// verify, unpack and source removal on small pack sets built in a temporary directory. +public sealed class AMPRPackOperationsTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), "mkpfs-ampr-ops-" + Guid.NewGuid().ToString("N")); + + public AMPRPackOperationsTests() => Directory.CreateDirectory(_dir); + + public void Dispose() => Directory.Delete(_dir, recursive: true); + + private string Root => Path.Combine(_dir, "app0"); + + private static byte[] Text(int size, int seed = 0) => + Encoding.ASCII.GetBytes(string.Concat(Enumerable.Range(seed, (size / 8) + 1).Select(i => $"blk{i % 97:D4} ")))[..size]; + + // Packs data/** (two files in nested folders), keeps eboot.bin and sce_sys loose. + private string BuildSet(string extraToml = "") + { + Dictionary files = new() + { + ["eboot.bin"] = Text(3000, 1), + ["sce_sys/param.json"] = Text(100, 2), + ["data/a/one.dat"] = Text(150_000, 3), + ["data/a/b/two.dat"] = Text(70_000, 4), + }; + foreach ((string path, byte[] data) in files) + { + string full = Path.Combine(Root, path); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + File.WriteAllBytes(full, data); + } + + AmprIndex.Build(Root, Path.Combine(Root, AmprIndex.IndexName)); + string config = Path.Combine(_dir, "config.toml"); + File.WriteAllText(config, "[pack]\ndefault_action = \"loose\"\n[[rule]]\ninclude = \"data/**\"\n" + extraToml, new UTF8Encoding(false)); + return AMPRPackBuilder.Build(Root, Path.Combine(Root, AmprIndex.IndexName), Path.Combine(_dir, "out"), AMPRPackConfig.Load(config)).IndexPath; + } + + [Theory] + [InlineData("eboot.bin", true)] + [InlineData("sce_sys/param.json", true)] + [InlineData("SCE_MODULE/x.bin", true)] + [InlineData("data/lib.SPRX", true)] + [InlineData("data/x.self", true)] + [InlineData("data/ampr_assets-007.pak", true)] + [InlineData("data/ampr_emu.index", true)] + [InlineData("data/.prx", false)] + [InlineData("data/a.prx.bak", false)] + [InlineData("data/eboot.bin.txt", false)] + public void Protected_paths_match_python(string relative, bool expected) => + Assert.Equal(expected, AMPRPackMaintenance.IsProtected(relative)); + + [Fact] + public void Remove_packed_sources_verifies_then_deletes_only_packed_files() + { + string index = BuildSet(); + AMPRRemovalPlan plan = AMPRPackMaintenance.RemovalPlan(index, Root); + Assert.Equal(["data/a/b/two.dat", "data/a/one.dat"], plan.Paths.Order(StringComparer.Ordinal)); + Assert.True(File.Exists(Path.Combine(Root, "data", "a", "one.dat"))); + + AMPRRemovalResult result = AMPRPackMaintenance.RemovePackedSources(index, Root, removeEmptyDirs: true); + + Assert.Equal(new AMPRRemovalResult(2, 220_000, 3, true), result); + Assert.False(Directory.Exists(Path.Combine(Root, "data"))); + Assert.True(File.Exists(Path.Combine(Root, "eboot.bin"))); + Assert.True(File.Exists(Path.Combine(Root, "sce_sys", "param.json"))); + + // The packs still reproduce the removed files exactly. + string restored = Path.Combine(_dir, "restored"); + Assert.Equal(new AMPRExtractResult(2, 220_000), AMPRPackTools.Extract(index, restored)); + Assert.Equal(Text(150_000, 3), File.ReadAllBytes(Path.Combine(restored, "data", "a", "one.dat"))); + Assert.Equal(Text(70_000, 4), File.ReadAllBytes(Path.Combine(restored, "data", "a", "b", "two.dat"))); + } + + [Fact] + public void Remove_packed_sources_refuses_a_changed_source_and_deletes_nothing() + { + string index = BuildSet(); + string one = Path.Combine(Root, "data", "a", "one.dat"); + byte[] changed = File.ReadAllBytes(one); + changed[70_000] ^= 0xFF; + File.WriteAllBytes(one, changed); + + AMPRPackException error = Assert.Throws(() => AMPRPackMaintenance.RemovePackedSources(index, Root)); + + Assert.StartsWith("packed/source mismatch for data/a/one.dat: logical=0x11170, chunk=1, pack=0, physical=0x", error.Message, StringComparison.Ordinal); + Assert.True(File.Exists(one)); + Assert.True(File.Exists(Path.Combine(Root, "data", "a", "b", "two.dat"))); + } + + [Fact] + public void Remove_packed_sources_refuses_when_a_source_is_missing() + { + string index = BuildSet(); + File.Delete(Path.Combine(Root, "data", "a", "b", "two.dat")); + + Assert.Equal(1, AMPRPackMaintenance.RemovalPlan(index, Root).MissingFiles); + Assert.Equal( + "cannot verify before removal: 1 packed source files are missing", + Assert.Throws(() => AMPRPackMaintenance.RemovePackedSources(index, Root)).Message); + Assert.True(File.Exists(Path.Combine(Root, "data", "a", "one.dat"))); + } + + [Fact] + public void Removal_plan_refuses_a_manifest_that_packs_protected_files() + { + string index = BuildSet("[[rule]]\ninclude = \"eboot.bin\"\n"); + Assert.Equal( + "manifest marks a protected game/runtime file as packed: eboot.bin", + Assert.Throws(() => AMPRPackMaintenance.RemovalPlan(index, Root)).Message); + } + + [Fact] + public void Verify_detects_a_corrupted_volume() + { + string index = BuildSet(); + AMPRPackManifest manifest = AMPRPackManifest.Load(index); + AMPRChunkRecord chunk = manifest.Chunks[0]; + string volume = Path.Combine(Path.GetDirectoryName(index)!, manifest.PackName(chunk.PackId)); + byte[] data = File.ReadAllBytes(volume); + data[(int)chunk.Offset + chunk.StoredSize - 1] ^= 0x55; + File.WriteAllBytes(volume, data); + + AMPRPackException error = Assert.Throws(() => AMPRPackTools.Verify(index)); + Assert.Contains(error.Message, new[] { "raw chunk CRC/size mismatch", "LZ4 chunk decompression failed" }); + } + + [Fact] + public void Readers_require_the_crc_sidecar() + { + string index = BuildSet(); + File.Delete(AMPRPackFormat.ChunkCrcPath(index)); + + Assert.StartsWith( + "invalid or missing chunk CRC sidecar: ", + Assert.Throws(() => AMPRPackTools.Verify(index)).Message, + StringComparison.Ordinal); + } + + [Fact] + public void Unpack_selects_files_and_refuses_to_overwrite_unless_asked() + { + string index = BuildSet(); + string output = Path.Combine(_dir, "unpacked"); + + Assert.Equal(new AMPRExtractResult(1, 70_000), AMPRPackTools.Extract(index, output, ["data/a/b/*"])); + AMPRPackException error = Assert.Throws(() => AMPRPackTools.Extract(index, output)); + Assert.Equal($"destination exists: {Path.Combine(output, "data", "a", "b", "two.dat")}", error.Message); + Assert.Equal(new AMPRExtractResult(2, 220_000), AMPRPackTools.Extract(index, output, overwrite: true, preserveMTime: false)); + Assert.Empty(Directory.GetFiles(output, "*.tmp-*", SearchOption.AllDirectories)); + } + + [Fact] + public void Runtime_config_requires_a_runtime_section() + { + string index = BuildSet(); + string config = Path.Combine(_dir, "noruntime.toml"); + File.WriteAllText(config, "[pack]\n"); + + Assert.Equal( + "configuration must contain [runtime]", + Assert.Throws(() => AMPRPackMaintenance.WriteRuntimeConfig(index, config)).Message); + } +} diff --git a/tests/MkPFS.Tests/AMPR/AMPRProfilerTests.cs b/tests/MkPFS.Tests/AMPR/AMPRProfilerTests.cs new file mode 100644 index 0000000..bb2f08c --- /dev/null +++ b/tests/MkPFS.Tests/AMPR/AMPRProfilerTests.cs @@ -0,0 +1,317 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text.Json; +using MkPFS.Build; +using MkPFS.Build.AMPRPack; +using MkPFS.Cli; +using MkPFS.Cli.Output; +using MkPFS.Core.AMPR; + +namespace MkPFS.Tests.AMPR; + +/// +/// ampr profile and --traces: the port of ampr_pack_profile.py. Byte parity with the Python tool is +/// checked by tools/oracle/check_ampr_profile.py; these tests cover the decoder and the integration. +/// +public sealed class AMPRProfilerTests +{ + /// Writes AMPRCMD1 journals like the debug emulator build. + internal sealed class TraceWriter + { + private readonly MemoryStream _data = new(); + private ulong _sequence; + + public static byte[] ReadFile(int fileId, long offset, long length) + { + bool wide = offset >= 1L << 32; + byte[] command = new byte[wide ? 24 : 20]; + uint dwords = wide ? 6u : 5u; + BinaryPrimitives.WriteUInt32LittleEndian(command, 40 | ((dwords - 1) << 8) | (uint)((offset & 0x3FFFF) << 12)); + BinaryPrimitives.WriteUInt32LittleEndian(command.AsSpan(4), (uint)(length - 1)); + BinaryPrimitives.WriteUInt32LittleEndian(command.AsSpan(8), (uint)fileId); + BinaryPrimitives.WriteUInt32LittleEndian(command.AsSpan(16), (uint)(offset & 0xFFFC0000)); + if (wide) + { + BinaryPrimitives.WriteUInt32LittleEndian(command.AsSpan(20), (uint)((offset >> 32) & 0xFF)); + } + + return command; + } + + public TraceWriter Record(byte[] payload, uint priority = 0, uint domain = 1) + { + byte[] header = new byte[AMPRCommandLog.HeaderSize]; + "AMPRCMD1"u8.CopyTo(header); + BinaryPrimitives.WriteUInt16LittleEndian(header.AsSpan(8), 1); + BinaryPrimitives.WriteUInt16LittleEndian(header.AsSpan(10), AMPRCommandLog.HeaderSize); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(12), (uint)(AMPRCommandLog.HeaderSize + payload.Length)); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(16), ++_sequence); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(32), 1_000_000 * _sequence); + BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(56), AMPRCommandLog.Fnv1a64(payload)); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(64), (uint)payload.Length); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(76), priority); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(80), domain); + _data.Write(header); + _data.Write(payload); + return this; + } + + public void Save(string path) => File.WriteAllBytes(path, _data.ToArray()); + } + + private static readonly string Level = string.Concat(Enumerable.Repeat("terrain mesh texture vertex ", 20_000)); + + // A game folder and one trace run of it: the game read data/level0.dat (randomly) and eboot.bin, nothing else. + private static (string Game, string Traces) TracedGame(TempDir dir) + { + dir.File("game/sce_sys/param.json", """{"titleId":"PPSA01234","contentVersion":"01.000.000"}"""); + dir.File("game/eboot.bin", "eboot"); + dir.File("game/data/level0.dat", Level); + dir.File("game/data/level1.dat", Level + "1"); + dir.File("game/fakelib/libSceAmpr.sprx", "ELF...AMPRPAK4.../app0/ampr_assets.index"); + string game = Path.Combine(dir.Path, "game"); + string run = dir.Dir("traces/startup"); + AmprIndex.Build(game, Path.Combine(run, "ampr_emu.index")); + Dictionary index = AMPRCommandLog.LoadIndex(Path.Combine(run, "ampr_emu.index")); + int Id(string path) => index.Single(e => e.Value.Path == path).Key; + TraceWriter trace = new(); + Random rng = new(7); + for (int i = 0; i < 40; i++) + { + trace.Record(TraceWriter.ReadFile(Id("/app0/data/level0.dat"), rng.Next(0, Level.Length - 9000), 8192)); + } + + trace.Record(TraceWriter.ReadFile(Id("/app0/eboot.bin"), 0, 5)); + trace.Save(Path.Combine(run, "ampr_commands.bin")); + return (game, Path.Combine(dir.Path, "traces")); + } + + [Theory] + [InlineData(new uint[] { 0x12345428, 99, 7, 0, 0xABCC1234 }, "AprReadFile", 5, 7, 0xABCD2345L, 100)] + [InlineData(new uint[] { 0x1528, 0, 0x80000003, 0, 0x40000, 0x12 }, "AprReadFile", 6, 3, 0x12_0004_0001L, 1)] + [InlineData(new uint[] { 0x5229, 9, 3 }, "AprReadGather", 3, 0, 5 | (3L << 18), 10)] + [InlineData(new uint[] { 0x22A, 15, 0 }, "AprReadScatter", 3, 0, 0, 16)] + [InlineData(new uint[] { 0x32B, 0, 0, 0x40000 }, "AprReadGatherScatter", 4, 0, 0x40000, 1)] + [InlineData(new uint[] { 47 }, "AprResetGatherScatterState", 1, 0, 0, 0)] + [InlineData(new uint[] { 0x5452110F, 0x6968 }, "MarkerSet", 2, 0, 0, 0)] + [InlineData(new uint[] { 0x325, 1, 2, 3 }, "Amm", 4, 0, 0, 0)] + [InlineData(new uint[] { 0xBBC }, "Unknown", 1, 0, 0, 0)] + public void Commands_decode_like_parse_ampr_command_log(uint[] words, string name, int dwords, long fileId, long fileOffset, long length) + { + byte[] data = new byte[words.Length * 4]; + for (int i = 0; i < words.Length; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(i * 4), words[i]); + } + + Assert.Equal(new AMPRCommand(name, dwords, fileId, fileOffset, length), AMPRCommandLog.Decode(data, 0)); + } + + [Theory] + [InlineData(new uint[] { 0x228, 0, 0 }, "invalid AprReadFile length")] + [InlineData(new uint[] { 0x701 }, "invalid WaitOnAddress length")] + [InlineData(new uint[] { 0x302, 0, 0, 0 }, "invalid WaitOnCounter length")] + [InlineData(new uint[] { 1032, 0 }, "truncated WriteKernelEventQueue")] + [InlineData(new uint[] { 0x5452500F }, "marker with color has no color dword")] + public void Invalid_commands_raise_the_parser_message(uint[] words, string message) + { + byte[] data = new byte[words.Length * 4]; + for (int i = 0; i < words.Length; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian(data.AsSpan(i * 4), words[i]); + } + + Assert.Equal(message, Assert.Throws(() => AMPRCommandLog.Decode(data, 0)).Message); + Assert.Equal("unaligned or truncated command", Assert.Throws(() => AMPRCommandLog.Decode(data, 2)).Message); + } + + [Fact] + public void A_profile_packs_only_what_the_game_read_and_keeps_executables_loose() + { + using TempDir dir = new(); + (_, string traces) = TracedGame(dir); + List runs = AMPRProfiler.DiscoverTracePairs(traces); + Assert.Equal("startup", Assert.Single(runs).Name); + + AMPRProfileOptions options = new() { Name = "game" }; + AMPRProfileResult result = AMPRProfiler.Build(runs, options); + + Assert.Equal(2, result.ObservedFiles); + Assert.Equal(1, result.PackedFiles); + string toml = AMPRProfiler.RenderToml(result, options); + Assert.Contains("include = [\"data/level0.dat\"]", toml, StringComparison.Ordinal); + Assert.DoesNotContain("level1", toml, StringComparison.Ordinal); + AMPRPackConfig config = AMPRProfiler.ToConfig(result, options); + Assert.Equal("compress", config.SelectRule("data/level0.dat").Action); + Assert.Equal("loose", config.SelectRule("data/level1.dat").Action); + Assert.Equal("loose", config.SelectRule("eboot.bin").Action); + Assert.NotNull(config.Runtime); + } + + [Fact] + public void Trace_folders_are_found_in_path_order_and_named_after_their_folders() + { + using TempDir dir = new(); + foreach (string run in (string[])["b/2", "a", "b/10", "c"]) + { + dir.File($"t/{run}/ampr_commands.bin"); + if (run != "c") + { + dir.File($"t/{run}/ampr_emu.index"); + } + } + + Assert.Equal(["a", "b-10", "b-2"], AMPRProfiler.DiscoverTracePairs(Path.Combine(dir.Path, "t")).Select(r => r.Name)); + Assert.Empty(AMPRProfiler.DiscoverTracePairs(Path.Combine(dir.Path, "missing"))); + } + + [Fact] + public void Cli_game_with_traces_packs_the_traced_files_and_says_so() + { + using TempDir dir = new(); + (string game, string traces) = TracedGame(dir); + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + CliContext ctx = new(stdout, stderr, useColor: false, utf8: false, progress: false); + string output = Path.Combine(dir.Path, "out"); + + int exit = MkPFSCli.Run(["ampr", "game", "--root", game, "--output", output, "--traces", traces], ctx); + + Assert.True(exit == 0, stdout.ToString() + stderr.ToString()); + Assert.Contains($"\nRules: from 1 trace run in {traces}; 1 file the game read through APR are packed (", stdout.ToString(), StringComparison.Ordinal); + Assert.False(File.Exists(Path.Combine(output, "data", "level0.dat"))); + Assert.True(File.Exists(Path.Combine(output, "data", "level1.dat"))); + Assert.True(File.Exists(Path.Combine(output, "ampr_assets.index.runtime"))); + + exit = MkPFSCli.Run(["ampr", "game", "--root", game, "--output", Path.Combine(dir.Path, "out2"), "--traces", traces, "--config", traces], ctx); + Assert.Equal(2, exit); + Assert.EndsWith("error: --traces cannot be used with --config\n", stderr.ToString(), StringComparison.Ordinal); + + exit = MkPFSCli.Run(["ampr", "game", "--root", game, "--output", Path.Combine(dir.Path, "out3"), "--traces", game], ctx); + Assert.Equal(2, exit); + Assert.EndsWith($"error: no traces found in {game}: each run needs ampr_commands.bin and its ampr_emu.index side by side\n", stderr.ToString(), StringComparison.Ordinal); + } + + [Fact] + public void Cli_profile_generate_writes_the_toml_report_and_header_and_refuses_to_overwrite() + { + using TempDir dir = new(); + (_, string traces) = TracedGame(dir); + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + CliContext ctx = new(stdout, stderr, useColor: false, utf8: false, progress: false); + string toml = Path.Combine(dir.Path, "p", "game.toml"); + string[] args = ["ampr", "profile", "generate", Path.Combine(traces, "startup"), "--output", toml, "--report", Path.Combine(dir.Path, "p", "game.md"), + "--runtime-header", Path.Combine(dir.Path, "p", "game.h")]; + + Assert.Equal(0, MkPFSCli.Run(args, ctx)); + Assert.StartsWith($"generated {toml}: files=2 index=", stdout.ToString(), StringComparison.Ordinal); + Assert.StartsWith("# Generated by ampr_pack_profile.py 4.1 from APR traces.\n", File.ReadAllText(toml), StringComparison.Ordinal); + Assert.Contains("# AMPR trace-derived pack profile: startup", File.ReadAllText(Path.Combine(dir.Path, "p", "game.md")), StringComparison.Ordinal); + Assert.Contains("#define AMPR_EMU_PACK_WORKERS ", File.ReadAllText(Path.Combine(dir.Path, "p", "game.h")), StringComparison.Ordinal); + Assert.Equal("compress", AMPRPackConfig.Load(toml).SelectRule("data/level0.dat").Action); + + Assert.Equal(2, MkPFSCli.Run(args, ctx)); + Assert.EndsWith($"error: refusing to overwrite existing file: {toml}\n", stderr.ToString(), StringComparison.Ordinal); + Assert.Equal(0, MkPFSCli.Run([.. args, "--overwrite"], ctx)); + } + + [Fact] + public void Cli_profile_generate_writes_metrics_in_upstream_key_order() + { + using TempDir dir = new(); + (_, string traces) = TracedGame(dir); + CliContext ctx = new(new StringWriter(), new StringWriter(), useColor: false, utf8: false, progress: false); + string metrics = Path.Combine(dir.Path, "m.json"); + + Assert.Equal(0, MkPFSCli.Run(["ampr", "profile", "generate", Path.Combine(traces, "startup"), "--output", Path.Combine(dir.Path, "p.toml"), + "--metrics", metrics, "--full-metrics"], ctx)); + + string text = File.ReadAllText(metrics); + Assert.StartsWith("{\n \"tool_version\": \"4.1\",\n \"name\": \"startup\",\n \"traces\": [", text, StringComparison.Ordinal); + Assert.EndsWith("}\n", text, StringComparison.Ordinal); + using JsonDocument doc = JsonDocument.Parse(text); + JsonElement files = doc.RootElement.GetProperty("files"); + Assert.Equal(["data/level0.dat", "eboot.bin"], files.EnumerateArray().Select(f => f.GetProperty("relative").GetString())); + JsonElement first = files[0]; + Assert.Equal("compress", first.GetProperty("recommendation").GetProperty("action").GetString()); + Assert.Equal(JsonValueKind.Null, first.GetProperty("recommendation").GetProperty("sampled_ratio").ValueKind); + Assert.Equal(["16384", "32768", "65536", "131072", "262144", "524288", "1048576"], + first.GetProperty("candidates").EnumerateObject().Select(c => c.Name)); + } + + [Fact] + public void Cli_profile_batch_writes_one_profile_per_run_from_a_folder_or_zip() + { + using TempDir dir = new(); + (_, string traces) = TracedGame(dir); + foreach (string file in (string[])["ampr_commands.bin", "ampr_emu.index"]) + { + Directory.CreateDirectory(Path.Combine(traces, "level1")); + File.Copy(Path.Combine(traces, "startup", file), Path.Combine(traces, "level1", file)); + } + + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + CliContext ctx = new(stdout, stderr, useColor: false, utf8: false, progress: false); + string output = Path.Combine(dir.Path, "profiles"); + + Assert.Equal(0, MkPFSCli.Run(["ampr", "profile", "batch", traces, "--output-dir", output, "--batch-jobs", "2"], ctx)); + Assert.Equal("generated level1.toml\ngenerated startup.toml\n", stdout.ToString()); + foreach (string name in (string[])["level1", "startup"]) + { + foreach (string extension in (string[])[".toml", ".md", ".json", ".runtime.h"]) + { + Assert.True(File.Exists(Path.Combine(output, name + extension)), name + extension); + } + } + + using (JsonDocument summary = JsonDocument.Parse(File.ReadAllText(Path.Combine(output, "summary.json")))) + { + Assert.Equal(["level1", "startup"], summary.RootElement.EnumerateArray().Select(r => r.GetProperty("name").GetString())); + } + + // Batch defaults: hybrid patterns, no cache simulation. + Assert.Contains("| disabled |", File.ReadAllText(Path.Combine(output, "startup.md")), StringComparison.Ordinal); + + // A support bundle: only the trace files are extracted; an existing profile is not replaced. + string zip = Path.Combine(dir.Path, "bundle.zip"); + ZipFile.CreateFromDirectory(traces, zip); + Assert.Equal(2, MkPFSCli.Run(["ampr", "profile", "batch", zip, "--output-dir", output], ctx)); + Assert.EndsWith($"error: refusing to overwrite existing file: {Path.Combine(output, "level1.toml")}\n", stderr.ToString(), StringComparison.Ordinal); + Assert.Equal(0, MkPFSCli.Run(["ampr", "profile", "batch", zip, "--output-dir", Path.Combine(dir.Path, "fromzip")], ctx)); + Assert.True(File.Exists(Path.Combine(dir.Path, "fromzip", "startup.toml"))); + } + + [Fact] + public void Untraced_files_of_traced_types_are_packed_like_them_on_request() + { + using TempDir dir = new(); + (string game, string traces) = TracedGame(dir); + StringWriter stdout = new() { NewLine = "\n" }; + StringWriter stderr = new() { NewLine = "\n" }; + CliContext ctx = new(stdout, stderr, useColor: false, utf8: false, progress: false); + string output = Path.Combine(dir.Path, "out"); + + int exit = MkPFSCli.Run(["ampr", "game", "--root", game, "--output", output, "--traces", traces, "--pack-untraced-types"], ctx); + + Assert.True(exit == 0, stdout.ToString() + stderr.ToString()); + // level0.dat was traced; level1.dat has the same type, so it is packed too; eboot.bin stays loose. + Assert.Contains("1 file the game read through APR are packed (", stdout.ToString(), StringComparison.Ordinal); + Assert.Contains("), plus 1 untraced files of the same types (", stdout.ToString(), StringComparison.Ordinal); + Assert.False(File.Exists(Path.Combine(output, "data", "level1.dat"))); + Assert.True(File.Exists(Path.Combine(output, "eboot.bin"))); + + List runs = AMPRProfiler.DiscoverTracePairs(traces); + AMPRProfileOptions options = new() { Name = "game" }; + AMPRProfileResult result = AMPRProfiler.Build(runs, options); + string toml = AMPRProfiler.RenderToml(result, options, runs, null, "p", out AMPRProfiler.UntracedAddition added); + Assert.Equal(1, added.Files); + Assert.StartsWith(AMPRProfiler.RenderToml(result, options).Split("# Safety exclusions.")[0], toml, StringComparison.Ordinal); + Assert.Contains("# MkPFS: 1 untraced .dat files (", toml, StringComparison.Ordinal); + Assert.EndsWith(AMPRProfiler.RenderToml(result, options)[AMPRProfiler.RenderToml(result, options).IndexOf("# Safety exclusions.", StringComparison.Ordinal)..], toml, StringComparison.Ordinal); + + Assert.Equal(2, MkPFSCli.Run(["ampr", "game", "--root", game, "--output", Path.Combine(dir.Path, "o2"), "--pack-untraced-types"], ctx)); + Assert.EndsWith("error: --pack-untraced-types needs --traces\n", stderr.ToString(), StringComparison.Ordinal); + } +} diff --git a/tests/MkPFS.Tests/Build/AmprIndexTests.cs b/tests/MkPFS.Tests/Build/AmprIndexTests.cs index dfb1cb3..a2f7668 100644 --- a/tests/MkPFS.Tests/Build/AmprIndexTests.cs +++ b/tests/MkPFS.Tests/Build/AmprIndexTests.cs @@ -84,6 +84,87 @@ public void Hash_table_finds_every_row_by_probing() } } + // The ampr_emu resolver (compact_index_hash_key): FNV-1a 64 over UTF-8 bytes, '\' as '/', only ASCII A..Z folded. + private static ulong RuntimeHash(string path) + { + ulong hash = 1469598103934665603UL; + foreach (byte b in System.Text.Encoding.UTF8.GetBytes(path)) + { + byte c = b == (byte)'\\' ? (byte)'/' : b is >= 0x41 and <= 0x5A ? (byte)(b + 0x20) : b; + hash ^= c; + hash = unchecked(hash * 1099511628211UL); + } + + return hash == 0 ? 1 : hash; + } + + [Fact] + public void Non_ascii_paths_use_the_console_resolver_hash_and_folding() + { + using TempDir dir = new(); + string source = AmprTree(dir); + dir.File("src/Données/Écran.bin", "e"); + dir.File("src/zz/ünité.bin", "u"); + AmprIndex.Ensure(source, new ListLog()); + byte[] data = File.ReadAllBytes(Path.Combine(source, AmprIndex.IndexName)); + long hashOffset = BinaryPrimitives.ReadInt64LittleEndian(data.AsSpan(32)); + int slots = (int)BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(44)); + List rows = AmprIndex.ReadRows(data); + + Assert.Contains(rows, r => r.Path == "/app0/Données/Écran.bin"); + Assert.Contains(rows, r => r.Path == "/app0/zz/ünité.bin"); + Assert.NotEqual(AmprIndex.KeyFor("/app0/É"), AmprIndex.KeyFor("/app0/é")); // only ASCII folds + for (int i = 0; i < rows.Count; i++) + { + // The game asks with ASCII letters upper-cased; the resolver must still land on this row. + string asked = new([.. rows[i].Path.Select(c => c is >= 'a' and <= 'z' ? char.ToUpperInvariant(c) : c)]); + ulong hash = RuntimeHash(asked); + Assert.Equal(RuntimeHash(rows[i].Path), AmprIndex.PathHash(rows[i].Path)); + int pos = (int)(hash & (ulong)(slots - 1)); + while (BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan((int)hashOffset + (pos * 16) + 8)) != i + 1) + { + Assert.NotEqual(0u, BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan((int)hashOffset + (pos * 16) + 8))); + pos = (pos + 1) & (slots - 1); + } + } + + // Records are sorted by the ASCII-folded UTF-8 bytes, the order the resolver's binary search expects. + List keys = [.. rows.Select(r => System.Text.Encoding.UTF8.GetBytes(AmprIndex.KeyFor(r.Path)))]; + for (int i = 1; i < keys.Count; i++) + { + Assert.True(keys[i - 1].AsSpan().SequenceCompareTo(keys[i]) < 0, $"{rows[i - 1].Path} !< {rows[i].Path}"); + } + } + + [Fact] + public void Emulator_trace_and_log_files_are_not_indexed() + { + using TempDir dir = new(); + string source = AmprTree(dir); + dir.File("src/ampr_commands.bin", "trace"); + dir.File("src/APR_EMU.LOG", "log"); + dir.File("src/data2/ampr_commands.bin", "kept below the root"); + + AmprIndex.Ensure(source, new ListLog()); + + List rows = AmprIndex.ReadRows(File.ReadAllBytes(Path.Combine(source, AmprIndex.IndexName))); + Assert.Equal(["/app0/Data/Level1.pak", "/app0/data2/ampr_commands.bin", "/app0/data2/b.bin", "/app0/eboot.bin", "/app0/fakelib/libSceAmpr.sprx"], rows.Select(r => r.Path)); + } + + [Fact] + public void Fakelib2_also_marks_an_emulation_build() + { + using TempDir dir = new(); + string source = dir.Dir("src"); + dir.File("src/fakelib2/libSceAmpr.sprx", "sprx"); + dir.File("src/eboot.bin", "eboot"); + ListLog log = new(); + + Assert.NotNull(AmprIndex.Ensure(source, log)); + Assert.Equal("Info: Detected fakelib2/libSceAmpr.sprx; generating ampr_emu.index...", log.Lines[0]); + Assert.True(AmprLibs.HasEmulator(source)); + } + [Theory] [InlineData(0, 0)] [InlineData(1, 2)] @@ -158,6 +239,30 @@ public void Create_if_missing_keeps_a_valid_index_and_rebuilds_a_stale_one() Assert.StartsWith("Info: Detected", forced.Lines[0], StringComparison.Ordinal); } + [Fact] + public void An_ampr_pack_set_keeps_its_index_unless_regeneration_is_forced() + { + using TempDir dir = new(); + string source = AmprTree(dir); + AmprIndex.Ensure(source, new ListLog()); + string index = Path.Combine(source, AmprIndex.IndexName); + byte[] original = File.ReadAllBytes(index); + dir.File("src/ampr_assets.index", "manifest"); + dir.File("src/ampr_assets-000.pak", "volume"); + + // Default and create-if-missing both keep the index the packs were built against. + ListLog kept = new(); + Assert.Null(AmprIndex.Ensure(source, kept)); + Assert.Null(AmprIndex.Ensure(source, kept, createIfMissing: true)); + Assert.Equal(original, File.ReadAllBytes(index)); + Assert.All(kept.Lines, line => Assert.StartsWith("Warning: ampr_emu.index kept: the AMPR packs in this folder", line, StringComparison.Ordinal)); + + ListLog forced = new(); + Assert.NotNull(AmprIndex.Ensure(source, forced, forceRegen: true)); + Assert.StartsWith("Warning: Rebuilding ampr_emu.index although ampr_assets.index is present", forced.Lines[0], StringComparison.Ordinal); + Assert.Equal(6, AmprIndex.ReadRows(File.ReadAllBytes(index)).Count); + } + [Fact] public void Pack_folder_writes_the_index_into_the_image_unless_disabled() { diff --git a/tests/MkPFS.Tests/Compression/LZ4Tests.cs b/tests/MkPFS.Tests/Compression/LZ4Tests.cs new file mode 100644 index 0000000..5e5f41f --- /dev/null +++ b/tests/MkPFS.Tests/Compression/LZ4Tests.cs @@ -0,0 +1,108 @@ +using MkPFS.Core.Compression; + +namespace MkPFS.Tests.Compression; + +public sealed class LZ4Tests +{ + private static readonly string[] Words = ["pfs ", "block ", "inode ", "asset ", "texture ", "mesh ", "\n"]; + + private static byte[] Block(int seed) + { + // Random word sequence: LZ4 needs 4-byte repeats, so byte-level noise would not shrink. + Random random = new(seed); + System.Text.StringBuilder text = new(); + while (text.Length < 0x10000) + { + text.Append(Words[random.Next(Words.Length)]); + } + + return System.Text.Encoding.ASCII.GetBytes(text.ToString(0, 0x10000)); + } + + [Fact] + public void Native_library_reports_lz4_1_9_4() => Assert.Equal("1.9.4", LZ4Codec.NativeVersion); + + [Theory] + [InlineData(false, 1)] + [InlineData(false, 8)] + [InlineData(true, 1)] + [InlineData(true, 9)] + [InlineData(true, 12)] + public void Compress_then_decompress_round_trips(bool highCompression, int parameter) + { + byte[] block = Block(parameter); + using LZ4Encoder encoder = new(); + byte[] compressed = highCompression ? encoder.CompressHC(block, parameter) : encoder.CompressFast(block, parameter); + byte[] restored = new byte[block.Length]; + + Assert.True(compressed.Length < block.Length); + Assert.Equal(block.Length, LZ4Codec.Decompress(compressed, restored)); + Assert.Equal(block, restored); + } + + [Fact] + public void Reused_encoder_gives_the_same_bytes_as_a_fresh_one() + { + using LZ4Encoder encoder = new(); + byte[] fast = encoder.CompressFast(Block(1)); + byte[] hc = encoder.CompressHC(Block(1)); + encoder.CompressFast(Block(2)); + encoder.CompressHC(Block(2), 3); + + using LZ4Encoder fresh = new(); + Assert.Equal(fast, encoder.CompressFast(Block(1))); + Assert.Equal(hc, encoder.CompressHC(Block(1))); + Assert.Equal(fast, fresh.CompressFast(Block(1))); + Assert.Equal(hc, fresh.CompressHC(Block(1))); + } + + [Fact] + public void Out_of_range_parameters_are_clamped_like_ampr_pack() + { + using LZ4Encoder encoder = new(); + byte[] block = Block(3); + + Assert.Equal(encoder.CompressFast(block, 1), encoder.CompressFast(block, 0)); + Assert.Equal(encoder.CompressHC(block, 1), encoder.CompressHC(block, 0)); + Assert.Equal(encoder.CompressHC(block, 12), encoder.CompressHC(block, 13)); + } + + [Fact] + public void Empty_input_gives_an_empty_block_like_ampr_pack() + { + using LZ4Encoder encoder = new(); + Assert.Empty(encoder.CompressFast([])); + Assert.Empty(encoder.CompressHC([])); + } + + [Fact] + public void Decompress_rejects_corrupt_and_truncated_blocks() + { + using LZ4Encoder encoder = new(); + byte[] compressed = encoder.CompressHC(Block(4)); + byte[] output = new byte[0x10000]; + + Assert.Throws(() => LZ4Codec.Decompress(compressed.AsSpan(0, compressed.Length / 2), output)); + Assert.Throws(() => LZ4Codec.Decompress([0xF0, 0xFF, 0xFF, 0xFF], output)); + } + + [Fact] + public void Decompress_rejects_too_small_destination() + { + using LZ4Encoder encoder = new(); + byte[] compressed = encoder.CompressFast(Block(5)); + Assert.Throws(() => LZ4Codec.Decompress(compressed, new byte[100])); + } + + [Fact] + public void CompressBound_rejects_negative_length() => + Assert.Throws(() => LZ4Codec.CompressBound(-1)); + + [Fact] + public void Disposed_encoder_throws() + { + LZ4Encoder encoder = new(); + encoder.Dispose(); + Assert.Throws(() => encoder.CompressFast(new byte[16])); + } +} diff --git a/tests/MkPFS.Tests/Util/CpuTopologyTests.cs b/tests/MkPFS.Tests/Util/CpuTopologyTests.cs new file mode 100644 index 0000000..ac997de --- /dev/null +++ b/tests/MkPFS.Tests/Util/CpuTopologyTests.cs @@ -0,0 +1,54 @@ +using MkPFS.Core.Util; + +namespace MkPFS.Tests.Util; + +public sealed class CpuTopologyTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "mkpfs-cpu-" + Guid.NewGuid().ToString("N")); + + public void Dispose() + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + + private void Cpu(int index, int package, int core, bool online = true) + { + string topology = Directory.CreateDirectory(Path.Combine(_root, $"cpu{index}", "topology")).FullName; + File.WriteAllText(Path.Combine(topology, "core_id"), $"{core}\n"); + File.WriteAllText(Path.Combine(topology, "physical_package_id"), $"{package}\n"); + if (index > 0) + { + File.WriteAllText(Path.Combine(_root, $"cpu{index}", "online"), online ? "1\n" : "0\n"); + } + } + + [Fact] + public void Linux_counts_cores_not_hardware_threads() + { + // 16 cores with two threads each: 32 logical processors. + for (int thread = 0; thread < 32; thread++) + { + Cpu(thread, package: 0, core: thread % 16); + } + + Directory.CreateDirectory(Path.Combine(_root, "cpufreq")); + Assert.Equal(16, CpuTopology.LinuxCores(_root)); + } + + [Fact] + public void Linux_counts_each_package_and_skips_offline_cpus() + { + Cpu(0, package: 0, core: 0); + Cpu(1, package: 1, core: 0); // same core id on the second socket + Cpu(2, package: 1, core: 1, online: false); + + Assert.Equal(2, CpuTopology.LinuxCores(_root)); + } + + [Fact] + public void Physical_cores_stay_within_the_logical_count() => + Assert.InRange(CpuTopology.PhysicalCores, 1, Environment.ProcessorCount); +} diff --git a/tools/oracle/README.md b/tools/oracle/README.md index 3c0d107..4e0c3e5 100644 --- a/tools/oracle/README.md +++ b/tools/oracle/README.md @@ -1,7 +1,15 @@ -# Python oracle (Phase 0) +# Oracle corpus -Builds reference images with Python MkPFS so the C# port can be checked byte for byte. -Python MkPFS (`D:\TOOLS\PS5\MkPFS`, expected at `../MkPFS`) stays the oracle until parity sign-off. +The parity tests check MkPFS.CSharp **byte for byte** against the Python tools it ports. These scripts run +the Python tools on generated inputs and record what they produce. + +| Oracle | Checked out at | What it checks | +|---|---|---| +| [Python MkPFS](https://github.com/PSBrew/MkPFS) 1.0.0 | `../MkPFS` | Images, logs and game metadata (`build_goldens.py`). | +| [ampr_emu](https://github.com/drakmor/ampr_emu) `tools/ampr_pack.py` 4.0 | `../ampr_emu` | AMPR asset packs (`build_ampr_goldens.py`). | +| ampr_emu `tools/ampr_pack_profile.py` 4.1 | `../ampr_emu` | `mkpfs ampr profile` (`check_ampr_profile.py`). | + +The generated corpus lives in `tests/fixtures/generated/` (git-ignored). Without it, the parity tests skip. ## Files @@ -11,8 +19,10 @@ Python MkPFS (`D:\TOOLS\PS5\MkPFS`, expected at `../MkPFS`) stays the oracle unt | `make_fixtures.py` | Generates deterministic source trees (seeded data, mtimes pinned to 1600000000) | | `build_goldens.py` | Builds 33 cases, captures inspect/tree/verify output, writes `manifest.json` and zlib vectors | | `bench.py` | Python baseline timings on a ~330 MiB tree | +| `build_ampr_goldens.py` | AMPR asset-pack corpus from ampr_emu's `ampr_pack.py` (second oracle, see the last section) | +| `check_ampr_profile.py` | Runs ampr_emu's `ampr_pack_profile.py` and `mkpfs ampr profile` (`generate` and `batch`) on synthetic APR traces and ZIP bundles and compares every output byte for byte (45 cases) | -## Run +## Python MkPFS corpus Run from the repo root. `uv` uses the Python repo's environment. @@ -49,8 +59,7 @@ Both default to `tests/fixtures/generated/` (git-ignored, about 270 MB). | `batch` | folder + exFAT file: convert, rerun (skipped), dry run; output images hashed | | Game metadata | `metadata.json` / `metadata_src.json`: Python `read_game_metadata` for each image and its source | -Logs (`*.log`) are UTF-8 with ` -` line endings: Python runs with `PYTHONIOENCODING=utf-8` and +Logs (`*.log`) are UTF-8 with `\n` line endings: Python runs with `PYTHONIOENCODING=utf-8` and `MKPFS_NO_UTF8=1` (ASCII icons such as `WARN`). They contain Windows path separators; compare them only on Windows or normalize `\` first. Images are platform independent. @@ -65,6 +74,11 @@ Images are platform independent. ## Findings to carry into the port +Differences between Python MkPFS and the port. **Bug** marks a Python bug that the port fixes. + +
+Show the 21 findings + 1. **Bug:** `--skip-verification` alone always fails ("--verify-structure and --skip-verification cannot be used together") because `--verify-structure` defaults on (Python `cli.py:1053`). Port: skip wins over the default. The harness uses `--no-verify-structure`. @@ -123,3 +137,69 @@ Images are platform independent. 21. **Bug:** the directory walks recurse once per level, so a crafted image nested about 1000 levels deep stops with `RecursionError`. Port: walks PFS trees without recursion and reports directories nested deeper than 1024 levels (PFS and exFAT) as errors instead of ending the process. + +
+ +## AMPR asset-pack corpus (`build_ampr_goldens.py`) + +Second oracle, for the `ampr` commands: ampr_emu `tools/ampr_pack.py` (tool version 4.0) at commit +`cfa85df379f6eeeb165d7badf9b648e266fe77b7`, checked out clean at `../ampr_emu`. It needs python-lz4 +4.4.5 (bundles liblz4 1.9.4), which the `../MkPFS` environment lacks, so run it with a Python that has it: + +```bash +python tools/oracle/build_ampr_goldens.py --check +``` + +```bash +uv run --no-project --python 3.11 --with lz4==4.4.5 python tools/oracle/build_ampr_goldens.py --check +``` + +The script refuses another ampr_emu commit and fails on another python-lz4 unless `--allow-lz4-version`. + +Output (`tests/fixtures/generated/ampr/`, about 180 MB): +- `trees/ampr_assets/`: fixture `/app0` (edge sizes, incompressible data, duplicate files, 3 MiB archives, + a movie, 20 scripts, modules; mtimes 1600000000). +- `goldens//`: `config.toml` (+ `list.txt`, `runtime_alt.toml`), `ampr_emu.index` (oracle index, + input for the C# tests), `out/` (manifest, `.pak` volumes, `.crc`, `.runtime`), `config.canonical.json` + (`_canonical_config_bytes`, the build-id input), and logs for `index`, `pack`, `verify --root`, `list`, + `list --json`, `inspect`, `unpack`, `remove-packed-sources` (plan), `runtime-config`. Unpacked files are + checked against the source and hashed, then deleted with the `app0` copy. Rebuild `app0` from + `trees/ampr_assets` plus the case's `remove_before_pack` and `touch_after_index`. +- `goldens/manifest.json`: versions (ampr_emu commit, tool version, Python, python-lz4, liblz4), per-case + argv, exit codes, sha256 of every output. +- `vectors/lz4_vectors.bin`: `L4VEC001`, u32 count, then per entry u8 mode (0 fast, 1 hc), 3 zero bytes, + u32 param (acceleration or level), u32 raw_len, u32 comp_len, raw, comp (81 entries: fast 1/2/8 and HC + 1–12 on five small blocks, fast 1 and HC 9/12 on two 1 MiB blocks). + +Cases (34): default, no config, upstream example TOML, fast (accel 1, 8), HC 9, 16 KiB random hot, 1 MiB, +streaming, store, dedup off/group/streaming, lanes balanced/hash/round-robin, striping with 1 MiB volume +rollover, auto-loose, `--self-contained`, `[runtime]` + `runtime-config`, 4 KiB I/O page, mtime preserved +vs not, CLI `--include`/`--exclude`, `include_from`, `--allow-missing`, `--workers 1` vs `8` (must match), +an unsafe default without the system loose rule, and five error cases. + +Results (2026-10-04, Python 3.11.9, python-lz4 4.4.5, liblz4 1.9.4): 34/34 as expected, two builds +byte-identical, `--workers 1` and `8` produce identical packs. Build time about 2 minutes per pass. + +### Findings to carry into the port + +1. `--include`/`--exclude` only narrow the rule result; with no TOML `default_action` is `loose`, so + packing needs at least `[pack] default_action = "compress"`. +2. The packer has no hard exclusions: `default_action = "compress"` packs `eboot.bin`, PRX/SPRX and + `sce_sys`. Only `remove-packed-sources` refuses them ("manifest marks a protected game/runtime file as + packed"). Every golden config except `unsafe_default` ends with a `loose` rule for those paths. +3. `load_config` reads keys with `dict.get` and ignores unknown keys (only `[runtime]` checks its key + set). A misspelled key silently takes the default. +4. `validate_index_metadata` compares only sizes, not mtimes. `preserve_mtime = true` stores the disk + mtime, `false` the AMPRIDX3 mtime (`mtime_preserved` vs `no_preserve_mtime`). +5. MkPFS's `ampr_emu.index` and ampr_emu's `build_ampr_index.py` produce the same bytes for the same tree + (checked on the `ampr` tree; only mtimes differ when the copy does not keep them). + +## `ampr profile` check (`check_ampr_profile.py`) + +Generates synthetic APR traces and ZIP support bundles, runs ampr_emu's `ampr_pack_profile.py` and +`mkpfs ampr profile` (`generate` and `batch`) on each, and compares every output file byte for byte +(45 cases). Build MkPFS first; `--python`, `--mkpfs`, `--ampr-emu` and `--out` override the defaults. + +```bash +uv run --no-project --python 3.12 --with lz4==4.4.5 python tools/oracle/check_ampr_profile.py +``` diff --git a/tools/oracle/build_ampr_goldens.py b/tools/oracle/build_ampr_goldens.py new file mode 100644 index 0000000..90f14ca --- /dev/null +++ b/tools/oracle/build_ampr_goldens.py @@ -0,0 +1,423 @@ +"""Build the AMPR asset-pack golden corpus with ampr_emu's ``tools/ampr_pack.py`` (oracle for docs/AMPR_PACK_PLAN.md). + +Layout of ``--out`` (default ``tests/fixtures/generated/ampr``): + trees/ampr_assets/ fixture /app0 source + goldens// app0/ (copy + ampr_emu.index), config.toml, out/, unpacked/, *.log, + config.canonical.json (ampr_pack._canonical_config_bytes) + goldens/manifest.json case → argv, exit codes, sha256 of every output file, versions + vectors/lz4_vectors.bin raw blocks + oracle LZ4 output for codec byte tests + +Needs Python >= 3.11 with python-lz4 (the oracle's encoder), and ampr_emu checked out at +``ORACLE_COMMIT`` in ``../ampr_emu``. The ``../MkPFS`` uv environment has no lz4; use: + python tools/oracle/build_ampr_goldens.py --check + uv run --no-project --python 3.11 --with lz4==4.4.5 python tools/oracle/build_ampr_goldens.py --check +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import shutil +import struct +import subprocess +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +HERE: Path = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from build_goldens import ( + normalize_log, # ruff: ignore[module-import-not-at-top-of-file] (sibling script) +) +from make_fixtures import ( # ruff: ignore[module-import-not-at-top-of-file] + _param_json, + _semi, + _text, + _write, + pin_mtimes, +) + +ORACLE_COMMIT: str = "cfa85df379f6eeeb165d7badf9b648e266fe77b7" +EXPECTED_LZ4: str = "4.4.5" # python-lz4; bundles liblz4 1.9.4 (see AMPR_PACK_PLAN.md, hazard 1) +AMPR_EMU: Path = HERE.parent.parent.parent / "ampr_emu" +AMPR_TOOLS: Path = AMPR_EMU / "tools" +DEFAULT_OUT: Path = HERE.parent.parent / "tests" / "fixtures" / "generated" / "ampr" +MIB: int = 1024 * 1024 + +BASE: str = '[pack]\ndefault_action = "compress"\n' +# ampr_pack has no hard exclusions (only remove-packed-sources refuses these), so most configs end with +# a loose rule for modules and system files; ``cfg`` appends it (last match wins). +SYSTEM_LOOSE: str = ( + '\n[[rule]]\naction = "loose"\ninclude = ["eboot.bin", "**/*.prx", "**/*.sprx", "sce_sys/**", "sce_module/**", ' + '"fakelib/**"]\n' +) +GROUPS3: str = '\n[groups.default]\npack_count = 3\n' +RUNTIME: str = ( + '\n[runtime]\ndecoded_cache_bytes = "64MiB"\nphysical_cache_bytes = "16MiB"\n' + "workers = 4\nlatency_reserve_workers = 1\n" +) +RUNTIME_ALT: str = ( + '[runtime]\ndecoded_cache_bytes = "128MiB"\nphysical_cache_bytes = "32MiB"\n' + "workers = 8\nlatency_reserve_workers = 2\n" +) +TOUCHED: dict[str, int] = {"assets/text_05.dat": 1_600_000_500} +AUTO_LOOSE: str = ( + 'auto_loose_min_file_size = "2MiB"\nauto_loose_sample_blocks = 8\nauto_loose_sample_bytes = "1MiB"\n' +) + + +def tree_ampr_assets(root: Path) -> None: + """``/app0`` with every packer path: RAW fallback, dedup, auto-loose, store, hot scripts, edge sizes.""" + rng: random.Random = random.Random(6) + _write(root, "sce_sys/param.json", _param_json("PPSA00006")) + _write(root, "eboot.bin", _semi(rng, 90_000)) + _write(root, "sce_module/libfixture.prx", _semi(rng, 40_000)) + _write(root, "fakelib/libSceAmpr.sprx", _semi(rng, 20_000)) + for i, size in enumerate((0, 1, 16_384, 65_536, 65_537, 700_000)): + _write(root, f"assets/text_{i:02d}.dat", _text(rng, size)) + _write(root, "assets/semi.dat", _semi(rng, 300_000)) + _write(root, "assets/random.bin", rng.randbytes(200_000)) + dup: bytes = _text(rng, 200_000) + _write(root, "assets/dup_a.bin", dup) + _write(root, "assets/sub/dup_b.bin", dup) + _write(root, "assets/Sub2/Mixed.BIN", _text(rng, 30_000)) + _write(root, "assets/zeros.bin", bytes(MIB)) + _write(root, "archives/big.pak", rng.randbytes(3 * MIB)) + _write(root, "archives/big_text.pak", _text(rng, 3 * MIB)) + _write(root, "movies/intro.bik", rng.randbytes(3 * MIB // 2)) + for i in range(20): + _write(root, f"scripts/s_{i:03d}.lua", _text(rng, rng.randint(100, 4000))) + _write(root, "localization/en.txt", _text(rng, 50_000)) + + +@dataclass +class Case: + """One ``ampr_pack.py pack`` run plus its follow-up commands.""" + + name: str + toml: str | None = BASE + flags: list[str] = field(default_factory=list) + expect_ok: bool = True + expect_fail_steps: list[str] = field(default_factory=list) # follow-up steps that must fail + remove_before_pack: list[str] = field(default_factory=list) # deleted after the index is built + touch_after_index: dict[str, int] = field(default_factory=dict) # mtime changed after the index is built + extra_files: dict[str, str] = field(default_factory=dict) # written next to config.toml + note: str = "" + + +def cfg(text: str) -> str: + """Return ``text`` with the trailing system loose rule.""" + return text + SYSTEM_LOOSE + + +def cases() -> list[Case]: + """Return the case matrix.""" + example: str = (AMPR_TOOLS / "ampr_pack.example.toml").read_text(encoding="utf-8") + stripe: str = ( + BASE + '\n[groups.default]\npack_count = 2\nmax_pack_size = "1MiB"\nstripe_large_files = true\n' + 'stripe_threshold = "1MiB"\nstripe_group_blocks = 4\n' + ) + return [ + Case("default", cfg(BASE), note="HC 12, 64 KiB, auto layout"), + Case("unsafe_default", BASE, expect_fail_steps=["remove_plan"], + note="no system loose rule: eboot.bin/PRX/sce_sys get packed; remove-packed-sources refuses"), + Case("no_config", toml=None, note="default_action loose: nothing is packed"), + Case("example", toml=example, note="upstream ampr_pack.example.toml"), + Case("fast", cfg(BASE + 'compression_mode = "fast"\n')), + Case("fast_accel8", cfg(BASE + 'compression_mode = "fast"\nacceleration = 8\n')), + Case("hc9", cfg(BASE + "compression_level = 9\n")), + Case("block16k_random", cfg(BASE + '\n[[rule]]\nblock_size = "16KiB"\nlayout = "random"\nhot = true\n')), + Case("block1m", cfg(BASE + 'default_block_size = "1MiB"\n')), + Case("streaming", cfg(BASE + '\n[[rule]]\nlayout = "streaming"\nblock_size = "256KiB"\n')), + Case("store_movies", cfg(BASE + '\n[[rule]]\naction = "store"\ninclude = ["movies/**"]\nlayout = "streaming"\n')), + Case("dedup_off", cfg(BASE + "deduplicate = false\n")), + Case("dedup_group", cfg(BASE + 'deduplicate_scope = "group"\n' + GROUPS3)), + Case("dedup_streaming", cfg(BASE + 'deduplicate_streaming = true\n\n[[rule]]\nlayout = "streaming"\n')), + Case("lanes_balanced", cfg(BASE + GROUPS3)), + Case("lanes_hash", cfg(BASE + GROUPS3 + 'assignment = "hash"\n')), + Case("lanes_round_robin", cfg(BASE + GROUPS3 + 'assignment = "round_robin"\n')), + Case("stripe_rollover", cfg(stripe)), + Case("auto_loose", cfg(BASE + AUTO_LOOSE)), + Case("self_contained", cfg(BASE + AUTO_LOOSE), ["--self-contained"]), + Case("runtime", cfg(BASE + RUNTIME), extra_files={"runtime_alt.toml": RUNTIME_ALT}), + Case("io_page_4k", cfg(BASE + 'io_page_size = "4KiB"\n')), + Case("mtime_preserved", cfg(BASE), touch_after_index=TOUCHED, note="manifest takes the disk mtime"), + Case("no_preserve_mtime", cfg(BASE + "preserve_mtime = false\n"), touch_after_index=TOUCHED, + note="manifest takes the AMPRIDX3 mtime"), + Case("cli_include_exclude", cfg(BASE), ["--include", "assets/**", "--exclude", "**/*.bin"]), + Case( + "include_from", + BASE.replace('"compress"', '"loose"') + '\n[[rule]]\ninclude_from = ["list.txt"]\n', + extra_files={"list.txt": "# scripts and text\nscripts/**\n\nassets/text_*.dat\n"}, + ), + Case("allow_missing", cfg(BASE), ["--allow-missing"], remove_before_pack=["assets/semi.dat"]), + Case("workers1", cfg(BASE), ["--workers", "1"], note="must equal workers8"), + Case("workers8", cfg(BASE), ["--workers", "8"]), + Case("neg_missing", cfg(BASE), expect_ok=False, remove_before_pack=["assets/semi.dat"]), + Case("neg_bad_block", cfg(BASE + 'default_block_size = "48KiB"\n'), expect_ok=False), + Case("neg_unknown_group", cfg(BASE + '\n[[rule]]\ngroup = "nope"\n'), expect_ok=False), + Case("neg_require_packed", cfg(BASE + AUTO_LOOSE), ["--require-packed", "archives/big.pak"], expect_ok=False), + Case("neg_runtime_keys", cfg(BASE + '\n[runtime]\nworkers = 4\n'), expect_ok=False), + ] # fmt: skip + + +def run_tool(script: str, argv: list[str], cwd: Path, log_name: str) -> int: + """Run an ampr_emu tool in ``cwd`` and store normalized combined output in ``log_name``.""" + env: dict[str, str] = {**os.environ, "PYTHONIOENCODING": "utf-8"} + proc: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, str(AMPR_TOOLS / script), *argv], + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + text: str = f"$ {script} {' '.join(argv)}\nexit={proc.returncode}\n--- stdout\n{proc.stdout}\n--- stderr\n{proc.stderr}" + (cwd / log_name).write_text(normalize_log(text, cwd), encoding="utf-8", newline="\n") + return proc.returncode + + +def canonical_config(config_path: Path | None) -> bytes: + """Return ``ampr_pack._canonical_config_bytes`` for a TOML (the build-id input).""" + import ampr_pack # ../ampr_emu/tools is on sys.path (see main) + + return ampr_pack._canonical_config_bytes(ampr_pack.load_config(config_path)) + + +def hash_tree(root: Path, prefix: str) -> dict[str, str]: + """Return ``prefix/relative`` → sha256 for every file under ``root``.""" + out: dict[str, str] = {} + if root.is_dir(): + for path in sorted(root.rglob("*")): + if path.is_file(): + out[f"{prefix}/{path.relative_to(root).as_posix()}"] = hashlib.sha256(path.read_bytes()).hexdigest() + return out + + +def build_case(case: Case, tree: Path, goldens: Path) -> dict[str, object]: + """Build one case and its follow-up outputs; return its manifest entry.""" + work: Path = goldens / case.name + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True) + shutil.copytree(tree, work / "app0") + pin_mtimes(work / "app0") + for name, text in case.extra_files.items(): + (work / name).write_text(text, encoding="utf-8", newline="\n") + config_args: list[str] = [] + if case.toml is not None: + (work / "config.toml").write_text(case.toml, encoding="utf-8", newline="\n") + config_args = ["--config", "config.toml"] + + exit_codes: dict[str, int] = {"index": run_tool("build_ampr_index.py", ["app0"], work, "index.log")} + for rel in case.remove_before_pack: + (work / "app0" / rel).unlink() + for rel, mtime in case.touch_after_index.items(): + os.utime(work / "app0" / rel, (mtime, mtime)) + pack_argv: list[str] = ["pack", "--root", "app0", "--ampr-index", "app0/ampr_emu.index", "--output", "out", + *config_args, "--no-progress", *case.flags] # fmt: skip + steps: dict[str, list[str]] = {"pack": pack_argv} + exit_codes["pack"] = run_tool("ampr_pack.py", pack_argv, work, "pack.log") + + built: bool = exit_codes["pack"] == 0 + index: str = "out/ampr_assets.index" + if built: + follow: dict[str, list[str]] = { + "verify": ["verify", "--index", index, "--root", "app0"], + "list": ["list", "--index", index, "--json"], + "list_text": ["list", "--index", index], + "inspect": ["inspect", "--index", index], + "unpack": ["unpack", "--index", index, "--output", "unpacked"], + "remove_plan": ["remove-packed-sources", "--index", index, "--root", "app0"], + } + if "runtime_alt.toml" in case.extra_files: + follow["runtime_config"] = ["runtime-config", "--index", index, "--config", "runtime_alt.toml"] + for step, argv in follow.items(): + steps[step] = argv + exit_codes[step] = run_tool("ampr_pack.py", argv, work, f"{step}.log") + if case.toml is not None: + (work / "config.canonical.json").write_bytes(canonical_config(work / "config.toml")) + + # Every unpacked file must equal its source. + unpack_ok: bool = True + for path in sorted((work / "unpacked").rglob("*")) if (work / "unpacked").is_dir() else []: + if path.is_file(): + source: Path = work / "app0" / path.relative_to(work / "unpacked") + unpack_ok = unpack_ok and source.read_bytes() == path.read_bytes() + + hashes: dict[str, str] = {} + for path in sorted(work.iterdir()): + if path.is_file(): + hashes[path.name] = hashlib.sha256(path.read_bytes()).hexdigest() + hashes |= hash_tree(work / "out", "out") + hashes |= hash_tree(work / "unpacked", "unpacked") + # Keep only the oracle index; tests rebuild app0 from trees/ampr_assets plus the recorded edits. + if (work / "app0" / "ampr_emu.index").is_file(): + shutil.copy2(work / "app0" / "ampr_emu.index", work / "ampr_emu.index") + hashes["ampr_emu.index"] = hashlib.sha256((work / "ampr_emu.index").read_bytes()).hexdigest() + shutil.rmtree(work / "app0") + shutil.rmtree(work / "unpacked", ignore_errors=True) + follow_ok: bool = ( + all((code != 0) == (step in case.expect_fail_steps) for step, code in exit_codes.items()) if built else True + ) + return { + "toml": case.toml is not None, + "flags": case.flags, + "remove_before_pack": case.remove_before_pack, + "touch_after_index": case.touch_after_index, + "steps": steps, + "exit_codes": exit_codes, + "built": built, + "expect_ok": case.expect_ok, + "unpack_matches_source": unpack_ok, + "status_ok": built == case.expect_ok and follow_ok and unpack_ok, + "note": case.note, + "sha256": hashes, + } + + +def write_lz4_vectors(tree: Path, out_path: Path) -> int: + """Write ``L4VEC001`` vectors with the oracle codec (``ampr_pack_format.Lz4Codec``). + + Format (little endian): magic[8], u32 count, then per entry u8 mode (0 fast, 1 hc), u8[3] zero, + u32 param (acceleration or level), u32 raw_len, u32 comp_len, raw bytes, comp bytes. + """ + from ampr_pack_format import Lz4Codec + + codec = Lz4Codec() + small: list[bytes] = [ + (tree / "assets/text_05.dat").read_bytes()[:0x10000], + (tree / "assets/semi.dat").read_bytes()[:0x10000], + (tree / "assets/random.bin").read_bytes()[:0x4000], + (tree / "scripts/s_000.lua").read_bytes(), + b"\x00", + ] + large: list[bytes] = [(tree / "archives/big_text.pak").read_bytes()[:MIB], bytes(MIB)] + params: list[tuple[int, int]] = [(0, 1), (0, 2), (0, 8), *((1, level) for level in range(1, 13))] + entries: list[tuple[int, int, bytes, bytes]] = [] + for raw in small: + for mode, param in params: + entries.append((mode, param, raw, codec_compress(codec, raw, mode, param))) + for raw in large: + for mode, param in ((0, 1), (1, 9), (1, 12)): + entries.append((mode, param, raw, codec_compress(codec, raw, mode, param))) + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("wb") as fh: + fh.write(b"L4VEC001" + struct.pack(" bytes: + """Compress like ``ampr_pack._compress_block`` does (raw LZ4 block, no size prefix).""" + if mode == 0: + return codec.compress(raw, mode="fast", acceleration=param) # type: ignore[attr-defined] + return codec.compress(raw, mode="hc", level=param) # type: ignore[attr-defined] + + +def oracle_versions() -> dict[str, str]: + """Check the pinned ampr_emu commit and the python-lz4 encoder; return the recorded versions.""" + head: str = subprocess.run( + ["git", "-C", str(AMPR_EMU), "rev-parse", "HEAD"], capture_output=True, text=True, check=True + ).stdout.strip() + dirty: str = subprocess.run( + ["git", "-C", str(AMPR_EMU), "status", "--porcelain", "--", "tools"], capture_output=True, text=True, check=True + ).stdout.strip() + if head != ORACLE_COMMIT or dirty: + raise SystemExit(f"error: {AMPR_EMU} must be a clean checkout of {ORACLE_COMMIT} (HEAD {head})") + try: + import lz4 + import lz4.block + except ImportError as exc: + raise SystemExit("error: python-lz4 is required (pip install lz4==4.4.5)") from exc + import ampr_pack + + return { + "ampr_emu_commit": head, + "ampr_pack_tool_version": ampr_pack.TOOL_VERSION, + "python": sys.version, + "python_lz4": lz4.__version__, + "liblz4": lz4.library_version_string(), + } + + +def build_all(out: Path) -> dict[str, object]: + """Recreate the tree, build every case, write manifest and vectors.""" + tree: Path = out / "trees" / "ampr_assets" + goldens: Path = out / "goldens" + if tree.exists(): + shutil.rmtree(tree) + tree.mkdir(parents=True) + tree_ampr_assets(tree) + pin_mtimes(tree) + goldens.mkdir(parents=True, exist_ok=True) + manifest: dict[str, object] = {"versions": oracle_versions(), "fixed_mtime": 1_600_000_000, "cases": {}} + for case in cases(): + entry: dict[str, object] = build_case(case, tree, goldens) + manifest["cases"][case.name] = entry # type: ignore[index] + print(f"{'OK ' if entry['status_ok'] else 'FAIL'} {case.name} {entry['exit_codes']}") + manifest["lz4_vectors"] = write_lz4_vectors(tree, out / "vectors" / "lz4_vectors.bin") + (goldens / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + return manifest + + +def compare(a: dict[str, object], b: dict[str, object]) -> list[str]: + """Return per-file differences between two manifests.""" + diffs: list[str] = [] + for name, ea in a["cases"].items(): # type: ignore[union-attr] + eb = b["cases"][name] # type: ignore[index] + for fname, h in ea["sha256"].items(): + if eb["sha256"].get(fname) != h: + diffs.append(f"{name}/{fname}") + return diffs + + +def main() -> int: + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT, help="Default: tests/fixtures/generated/ampr") + parser.add_argument("--check", action="store_true", help="Build twice and report nondeterministic files") + parser.add_argument("--allow-lz4-version", action="store_true", help=f"Accept python-lz4 other than {EXPECTED_LZ4}") + args: argparse.Namespace = parser.parse_args() + sys.path.insert(0, str(AMPR_TOOLS)) + + first: dict[str, object] = build_all(args.out) + versions: dict[str, str] = first["versions"] # type: ignore[assignment] + print(f"python-lz4 {versions['python_lz4']} (liblz4 {versions['liblz4']}), ampr_pack {versions['ampr_pack_tool_version']}") + failed: list[str] = [n for n, e in first["cases"].items() if not e["status_ok"]] # type: ignore[union-attr] + if versions["python_lz4"] != EXPECTED_LZ4 and not args.allow_lz4_version: + failed.append(f"python-lz4 {versions['python_lz4']} != {EXPECTED_LZ4}") + + cases_: dict[str, dict[str, object]] = first["cases"] # type: ignore[assignment] + w1: dict[str, str] = cases_["workers1"]["sha256"] # type: ignore[assignment] + w8: dict[str, str] = cases_["workers8"]["sha256"] # type: ignore[assignment] + same_workers: bool = {k: v for k, v in w1.items() if k.startswith("out/")} == { + k: v for k, v in w8.items() if k.startswith("out/") + } + print(f"workers1 == workers8 packs: {same_workers}") + if not same_workers: + failed.append("workers1 != workers8") + + if args.check: + second_out: Path = Path(tempfile.mkdtemp(prefix="mkpfs_ampr_check_")) + second: dict[str, object] = build_all(second_out) + diffs: list[str] = compare(first, second) + shutil.rmtree(second_out, ignore_errors=True) + (args.out / "determinism.json").write_text(json.dumps(diffs, indent=2), encoding="utf-8") + print(f"nondeterministic files: {len(diffs)}") + for d in diffs: + print(f" {d}") + if diffs: + failed.append("nondeterministic output") + print(f"failed: {failed}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/oracle/check_ampr_profile.py b/tools/oracle/check_ampr_profile.py new file mode 100644 index 0000000..555e646 --- /dev/null +++ b/tools/oracle/check_ampr_profile.py @@ -0,0 +1,505 @@ +"""Check ``mkpfs ampr profile`` against ampr_emu's ``tools/ampr_pack_profile.py`` (the oracle). + +Writes synthetic APR traces (``AMPRCMD1`` journals and their ``AMPRIDX3`` indexes: mixed read patterns, several +runs, gather/scatter state, corrupt records, an unsafe path, content for LZ4 sampling), runs both tools on the same +inputs with option sets that reach every branch (budget enforcement, cache-sampling windows, pattern modes, include +sidecars, sampling, option errors; ``batch`` on folders and ZIP bundles) and compares exit codes, stdout, the last +stderr line and every output file (TOML, sidecars, report, metrics JSON, runtime header, batch summary) byte for +byte. Only the random temporary folder of an extracted ZIP is normalized. + +Needs ampr_emu at ``ORACLE_COMMIT`` in ``../ampr_emu`` and a Python with python-lz4 4.4.5 for the oracle; build +MkPFS first. From the repo root: + dotnet build src/MkPFS.Cli + uv run --no-project --python 3.12 --with lz4==4.4.5 python tools/oracle/check_ampr_profile.py +""" + +from __future__ import annotations + +import argparse +import random +import re +import shutil +import struct +import subprocess +import sys +import zipfile +from pathlib import Path + +HERE: Path = Path(__file__).resolve().parent +REPO: Path = HERE.parent.parent +AMPR_EMU: Path = REPO.parent / "ampr_emu" +DEFAULT_OUT: Path = REPO / "tests" / "fixtures" / "generated" / "ampr_profile" +DEFAULT_MKPFS: Path = REPO / "src" / "MkPFS.Cli" / "bin" / "Debug" / "net10.0" / "mkpfs" + +HEADER = struct.Struct("<8sHHIQQQQQQIIIIIIII") + + +def fnv(data: bytes) -> int: + h = 1469598103934665603 + for b in data: + h ^= b + h = (h * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return h or 1 + + +def write_index(path: Path, entries: list[tuple[str, int]]) -> None: + blob = b"" + records = b"" + for p, size in entries: + raw = p.encode("utf-8", "surrogateescape") + records += struct.pack(" bytes: + return struct.pack(" bytes: + dwords = 6 if (wide or offset >= 1 << 32) else 5 + w0 = 40 | ((dwords - 1) << 8) | ((offset & 0x3FFFF) << 12) + w4 = (offset & 0xFFFC0000) | 0x1234 + out = w(w0) + w(length - 1) + w(file_id) + w(0xDEAD0000) + w(w4) + if dwords == 6: + out += w((offset >> 32) & 0xFF) + return out + + +def gather(offset: int, length: int) -> bytes: + dwords = 3 if offset >= 1 << 18 else 2 + w0 = 41 | ((dwords - 1) << 8) | ((offset & 0x3FFFF) << 12) + out = w(w0) + w(length - 1) + if dwords == 3: + out += w((offset >> 18) & 0x3FFFFF) + return out + + +def scatter(length: int) -> bytes: + return w(0x22A | (0x12 << 12)) + w(length - 1) + w(0x1000) + + +def gather_scatter(offset: int, length: int, wide: bool = False) -> bytes: + dwords = 5 if (wide or offset >= 1 << 32) else 4 + w0 = 43 | ((dwords - 1) << 8) | ((offset & 0x3FFFF) << 12) + out = w(w0) + w(length - 1) + w(0x2000) + w((offset & 0xFFFC0000) | 7) + if dwords == 5: + out += w((offset >> 32) & 0xFF) + return out + + +RESET = w(47) +MAP_END = w(46) + + +def noise(rng: random.Random) -> bytes: + choice = rng.randrange(9) + if choice == 0: # WaitOnAddress 3 dwords + return w(1 | (2 << 8)) + w(0x1000) + w(5) + if choice == 1: # WaitOnCounter 2 dwords with extra + return w(2 | (1 << 8)) + w(0x31000005) + if choice == 2: # WriteAddress 3 dwords + return w(5 | (2 << 8)) + w(0x2000 | 2) + w(9) + if choice == 3: # WriteCounter 3 dwords + return w(118 | (2 << 8)) + w(0x00100001) + w(3) + if choice == 4: # WriteKernelEventQueue + return w(1032) + w(1) + w(2) + w(3) + w(4) + if choice == 5: # marker set "hi" + return w(0x5452000F | (1 << 12) | (1 << 8)) + b"hi\0\0" + if choice == 6: # MarkerPop + return w(0x5452300F) + if choice == 7: # AprMapBegin + end + return w(557) + w(1) + w(2) + MAP_END + return w(0x325) + w(1) + w(2) + w(3) # AmmMapDirect + + +class Trace: + def __init__(self, path: Path, rng: random.Random) -> None: + self.path = path + self.rng = rng + self.data = bytearray() + self.seq = 0 + self.time = 1_000_000_000 + + def record(self, commands: list[bytes], priority: int = 0, domain: int = 1, *, seq: int | None = None, + bad_hash: bool = False, count: int | None = None, extra: bytes = b"", noise_rate: float = 0.2) -> None: + parts: list[bytes] = [] + for c in commands: + if self.rng.random() < noise_rate: + parts.append(noise(self.rng)) + parts.append(c) + payload = b"".join(parts) + self.seq = self.seq + 1 if seq is None else seq + self.time += self.rng.randrange(1000, 5_000_000) + n = count if count is not None else 0 + h = fnv(payload) ^ (1 if bad_hash else 0) + header_bytes = HEADER.size + len(extra) + head = HEADER.pack(b"AMPRCMD1", 1, header_bytes, header_bytes + len(payload), self.seq, 0, self.time, 0, 0, h, + len(payload), 4096, n, priority, domain, 0, 0, 0) + self.data += head + extra + payload + + def raw(self, data: bytes) -> None: + self.data += data + + def save(self) -> None: + self.path.write_bytes(bytes(self.data)) + + +def make_files(rng: random.Random, count: int, prefix: str = "") -> list[tuple[str, int]]: + exts = [".dat", ".bin", ".pak", ".bik", ".wem", ".prx", ".assets", ".bank", ".arc", ""] + out = [] + for i in range(count): + d = rng.choice(["data", "data/levels", "audio", "video", "bin", "data/levels/l1", "ui"]) + name = f"/app0/{prefix}{d}/file_{i:04d}{rng.choice(exts)}" + out.append((name, rng.choice([300, 4096, 70_000, 1 << 20, 5 << 20, 40 << 20, 200 << 20, 3 << 30]))) + return out + + +def workload(t: Trace, rng: random.Random, entries: list[tuple[str, int]], reads: int) -> None: + """Mix of random, streaming, gather/scatter and tiny-read patterns.""" + styles = {fid: rng.choice(["random", "stream", "tiny", "mixed", "gather"]) for fid in range(1, len(entries) + 1)} + cursor = {fid: 0 for fid in styles} + batch: list[bytes] = [] + priority = rng.randrange(3) + for _ in range(reads): + fid = rng.randrange(1, len(entries) + 1) + size = entries[fid - 1][1] + style = styles[fid] + if style == "stream": + length = rng.choice([256 << 10, 512 << 10, 1 << 20]) + off = cursor[fid] % max(1, size) + cursor[fid] = off + length + batch.append(read_file(fid, off, length)) + elif style == "tiny": + off = rng.randrange(0, max(1, min(size, 1 << 16))) + batch.append(read_file(fid, off, rng.choice([16, 64, 128, 200]))) + elif style == "random": + off = rng.randrange(0, max(1, size)) + batch.append(read_file(fid, off, rng.choice([4096, 8192, 16384, 32768]))) + elif style == "gather": + off = rng.randrange(0, max(1, size)) + batch.append(read_file(fid, off, 4096)) + for _ in range(rng.randrange(1, 4)): + if rng.random() < 0.5: + batch.append(scatter(rng.choice([4096, 65536]))) + else: + batch.append(gather(rng.randrange(0, max(1, min(size, 1 << 40))), rng.choice([2048, 65536]))) + if rng.random() < 0.3: + batch.append(gather_scatter(rng.randrange(0, max(1, size)), 8192, wide=rng.random() < 0.3)) + if rng.random() < 0.2: + batch.append(RESET) + else: + off = rng.randrange(0, max(1, size)) + batch.append(read_file(fid, off, rng.choice([65536, 131072, 300_000]), wide=rng.random() < 0.1)) + if len(batch) >= rng.randrange(1, 12): + t.record(batch, priority=priority, count=len(batch) if rng.random() < 0.3 else None, noise_rate=0.0) + batch = [] + priority = rng.randrange(3) + if batch: + t.record(batch, priority=priority) + + +def case_dir(root: Path, name: str) -> Path: + d = root / name + d.mkdir(parents=True, exist_ok=True) + return d + + +def build(root: Path) -> None: + rng = random.Random(1234) + + # basic: one run, mixed patterns, unicode + '|' names, prx / eboot (known loose), streaming suffixes + d = case_dir(root, "basic") + entries = make_files(rng, 40) + entries += [("/app0/eboot.bin", 5 << 20), ("/APP0/Data/ünïcode|name.dat", 3 << 20), ("/app0/sce_module/libc.prx", 1 << 20), + ("/app0/./odd//double/slash.dat", 2 << 20), ("/app0", 10)] + write_index(d / "ampr_emu.index", entries) + t = Trace(d / "ampr_commands.bin", rng) + workload(t, rng, entries, 3000) + t.record([read_file(len(entries) - 3, 0, 4096), read_file(len(entries) - 2, 100, 8192), read_file(len(entries) - 1, 0, 4096)]) + t.save() + + # multi: three runs merged with --trace, sizes changing between runs, unseen ids, warnings + d = case_dir(root, "multi") + base = make_files(rng, 60, "m") + for run in range(3): + rd = case_dir(d, f"run{run}") + ents = [(p, s + (4096 * run if i % 7 == 0 else 0)) for i, (p, s) in enumerate(base)] + write_index(rd / "ampr_emu.index", ents) + t = Trace(rd / "ampr_commands.bin", rng) + workload(t, rng, ents, 1500) + t.record([read_file(9999, 0, 10)]) # absent fileId + t.record([read_file(1, ents[0][1] + 10, 10)]) # beyond size + t.record([read_file(2, max(0, ents[1][1] - 5), 100)]) # clamped + t.record([gather(0, 10)], priority=7) # no state (fresh priority) + t.record([scatter(10)], priority=8) + t.record([gather_scatter(0, 10)], priority=9) + t.save() + + # many: 400 small files read randomly -> externalized / chunked rules + d = case_dir(root, "many") + ents = [(f"/app0/pak/{i // 50:02d}/chunk_{i:05d}.bin", 256 << 10) for i in range(400)] + write_index(d / "ampr_emu.index", ents) + t = Trace(d / "ampr_commands.bin", rng) + for _ in range(4000): + fid = rng.randrange(1, 401) + t.record([read_file(fid, rng.randrange(0, 256 << 10), rng.choice([4096, 16384]))], noise_rate=0.05) + t.save() + + # big: many touches for windowed cache sampling, plus lots of files (1000+ / switches -> 8 workers) + d = case_dir(root, "big") + ents = [(f"/app0/big/{i // 100:03d}/b{i:05d}.dat", rng.choice([1 << 20, 4 << 20])) for i in range(1200)] + write_index(d / "ampr_emu.index", ents) + t = Trace(d / "ampr_commands.bin", rng) + batch = [] + for _ in range(40000): + fid = rng.randrange(1, 1201) + batch.append(read_file(fid, rng.randrange(0, ents[fid - 1][1]), rng.choice([4096, 16384, 70000]))) + if len(batch) == 8: + t.record(batch, noise_rate=0) + batch = [] + t.save() + + # dirs: directory generalization (hybrid / directory) + d = case_dir(root, "dirs") + ents = [] + for dn in ["tex/a", "tex/b", "snd", "geo/x/y", "misc"]: + for i in range(rng.randrange(5, 30)): + ents.append((f"/app0/{dn}/f{i}.bin", rng.choice([1 << 20, 8 << 20]))) + write_index(d / "ampr_emu.index", ents) + t = Trace(d / "ampr_commands.bin", rng) + for _ in range(5000): + fid = rng.randrange(1, len(ents) + 1) + if ents[fid - 1][0].startswith("/app0/snd"): + t.record([read_file(fid, rng.randrange(0, 4) << 18, 1 << 18)], noise_rate=0) + elif rng.random() < 0.85: + t.record([read_file(fid, rng.randrange(0, ents[fid - 1][1]), 8192)], noise_rate=0) + t.save() + + # broken: corrupt journal pieces (decode errors, sequence gap, bad hash, AMM domain, truncation) + d = case_dir(root, "broken") + ents = make_files(rng, 10, "b") + [("/app0/../escape.dat", 4096)] + write_index(d / "ampr_emu.index", ents) + t = Trace(d / "ampr_commands.bin", rng) + t.record([read_file(1, 0, 4096)], seq=5) + t.record([read_file(2, 0, 4096)], seq=9, bad_hash=True) + t.record([read_file(3, 0, 4096)], domain=2) + t.record([w(40 | (2 << 8)) + w(0) + w(0)], count=3) # invalid AprReadFile length -> decode error + t.record([read_file(4, 0, 4096), read_file(5, 0, 4096)], count=5) # count mismatch + t.record([read_file(11, 0, 100)]) # unsafe path + t.record([read_file(6, 0, 4096)], extra=b"\x01" * 8) # extended header + t.record([w(1 | (7 << 8))]) # WaitOnAddress invalid + t.record([w(0xABC | (3 << 8))]) # Unknown, guessed 4 dwords but only 1 present -> 1 + t.record([read_file(7, 0, 4096)]) + for i in range(250): + t.record([read_file(9999, i, 1)], noise_rate=0) # >200 warnings -> omitted marker + t.save() + data = (d / "ampr_commands.bin").read_bytes() + (d / "ampr_commands.bin").write_bytes(data + data[:HEADER.size + 10]) # truncated payload at the end + + # trunchead: truncated header + d = case_dir(root, "trunchead") + write_index(d / "ampr_emu.index", ents[:5]) + t = Trace(d / "ampr_commands.bin", rng) + t.record([read_file(1, 0, 4096)]) + t.raw(b"AMPRCMD1" + b"\0" * 20) + t.save() + + # badmagic + d = case_dir(root, "badmagic") + write_index(d / "ampr_emu.index", ents[:5]) + (d / "ampr_commands.bin").write_bytes(b"NOTMAGIC" + b"\0" * 88) + + # empty journal + d = case_dir(root, "empty") + write_index(d / "ampr_emu.index", ents[:5]) + (d / "ampr_commands.bin").write_bytes(b"") + + # sample: content root with compressible / incompressible files + d = case_dir(root, "sample") + content = d / "app0" + ents = [] + for i in range(14): + name = f"data/s{i}.bin" + size = rng.choice([200 << 10, 1 << 20, 3 << 20]) if i < 12 else (65 << 20) + i + p = content / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(rng.randbytes(size) if i % 2 else (b"abcdefgh" * (size // 8 + 1))[:size]) + ents.append((f"/app0/{name}", size)) + write_index(d / "ampr_emu.index", ents) + t = Trace(d / "ampr_commands.bin", rng) + workload(t, rng, ents, 800) + t.save() + + + +def build_batch_inputs(root: Path) -> None: + """Folders and ZIP bundles for ``batch``: duplicate names, junk members, unsafe and empty archives.""" + dups = root / "dups" + for run in ("x/y", "x-y", "z"): + target = dups / run + target.mkdir(parents=True, exist_ok=True) + for f in ("ampr_commands.bin", "ampr_emu.index"): + shutil.copy2(root / "multi" / "run1" / f, target / f) + (dups / "orphan").mkdir() + (dups / "orphan" / "ampr_commands.bin").write_bytes(b"") + with zipfile.ZipFile(root / "bundle.zip", "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("support/", b"") + z.writestr("support/logs/decoded.txt", b"x" * 1000) + for run in ("run0", "run2"): + for f in ("ampr_commands.bin", "ampr_emu.index", "ampr_emu.log"): + source = root / "multi" / run / f + z.writestr(f"support/{run}/{f}", source.read_bytes() if source.exists() else b"log") + z.writestr("./support/./many/ampr_commands.bin", (root / "many" / "ampr_commands.bin").read_bytes()) + z.writestr("support/many/ampr_emu.index", (root / "many" / "ampr_emu.index").read_bytes()) + with zipfile.ZipFile(root / "unsafe.zip", "w") as z: + z.writestr("ok/readme.txt", b"x") + z.writestr("../evil/ampr_commands.bin", b"x") + with zipfile.ZipFile(root / "notrace.zip", "w") as z: + z.writestr("logs/a.txt", b"x") + (root / "notzip.bin").write_bytes(b"PK\x03\x04 not really a zip" * 10) + (root / "nothing").mkdir() + +GENERATE: list[tuple[str, list[str]]] = [ + ("basic", ["data/basic"]), + ("basic-nocache", ["data/basic", "--no-cache-sim", "--name", "x y", "--full-metrics"]), + ("basic-budget", ["data/basic", "--pack-index-budget", "1MiB", "--strategy", "conservative"]), + ("basic-aggr", ["data/basic", "--pack-index-budget", "1MiB", "--strategy", "aggressive", "--lanes", "3", "--runtime-workers", "5"]), + ("basic-touches", ["data/basic", "--cache-max-touches", "5000", "--cache-candidates", "1MiB,4MiB,64MiB,64MiB,300MiB"]), + ("basic-touches2", ["data/basic", "--cache-max-touches", "100000"]), + ("basic-minreads", ["data/basic", "--min-reads", "5", "--min-requested-bytes", "64KiB", "--io-page-size", "16KiB"]), + ("multi", ["--trace", "data/multi/run0/ampr_commands.bin", "data/multi/run0/ampr_emu.index", + "--trace", "data/multi/run1/ampr_commands.bin", "data/multi/run1/ampr_emu.index", + "--trace", "data/multi/run2/ampr_commands.bin", "data/multi/run2/ampr_emu.index"]), + ("multi-hybrid", ["--trace", "data/multi/run0/ampr_commands.bin", "data/multi/run0/ampr_emu.index", + "--trace", "data/multi/run2/ampr_commands.bin", "data/multi/run2/ampr_emu.index", "--pattern-mode", "hybrid"]), + ("multi-windows", ["--trace", "data/multi/run0/ampr_commands.bin", "data/multi/run0/ampr_emu.index", + "--trace", "data/multi/run1/ampr_commands.bin", "data/multi/run1/ampr_emu.index", + "--trace", "data/multi/run2/ampr_commands.bin", "data/multi/run2/ampr_emu.index", "--cache-max-touches", "8192"]), + ("big-windows", ["data/big", "--cache-max-touches", "16384"]), + ("big", ["data/big", "--pattern-mode", "hybrid"]), + ("many-windows", ["data/many", "--cache-max-touches", "4100"]), + ("many", ["data/many", "--full-metrics"]), + ("many-inline", ["data/many", "--externalize-paths", "0", "--max-rule-files", "70"]), + ("many-ext10", ["data/many", "--externalize-paths", "10"]), + ("many-hybrid", ["data/many", "--pattern-mode", "hybrid"]), + ("dirs-hybrid", ["data/dirs", "--pattern-mode", "hybrid"]), + ("dirs-directory", ["data/dirs", "--pattern-mode", "directory", "--generalize-min-files", "3", "--generalize-coverage", "0.9"]), + ("dirs-exact", ["data/dirs"]), + ("broken", ["data/broken"]), + ("trunchead", ["data/trunchead"]), + ("empty", ["data/empty"]), + ("badmagic", ["data/badmagic"]), + ("sample", ["data/sample", "--content-root", "data/sample/app0", "--sample-budget", "8MiB", "--full-metrics"]), + ("sample-fast", ["data/sample", "--content-root", "data/sample/app0", "--sample-budget", "300KiB", "--sample-mode", "fast", + "--sample-blocks-per-file", "1"]), + ("err-none", []), + ("err-dir", ["data/multi"]), + ("err-missing", ["--trace", "data/nope.bin", "data/nope.index"]), + ("err-io", ["data/basic", "--io-page-size", "3000"]), + ("err-lanes", ["data/basic", "--lanes", "65"]), + ("err-cov", ["data/basic", "--generalize-coverage", "1.5"]), + ("err-file", ["data/basic/ampr_emu.index"]), +] + + +BATCH: list[tuple[str, list[str]]] = [ + ("batch-multi", ["data/multi", "--batch-jobs", "1"]), + ("batch-multi-par", ["data/multi", "--batch-jobs", "3", "--full-metrics"]), + ("batch-exact-cache", ["data/multi", "--pattern-mode", "exact", "--cache-sim", "--batch-jobs", "1"]), + ("batch-dups", ["data/dups", "--batch-jobs", "1"]), + ("batch-single", ["data/many", "--batch-jobs", "1", "--summary", "SUMMARY"]), + ("batch-zip", ["data/bundle.zip", "--batch-jobs", "2", "--content-root", "data/sample/app0", "--sample-budget", "1MiB"]), + ("batch-err-unsafe", ["data/unsafe.zip"]), + ("batch-err-notrace", ["data/notrace.zip"]), + ("batch-err-notzip", ["data/notzip.bin"]), + ("batch-err-empty", ["data/nothing"]), + ("batch-err-missing", ["data/missing-folder"]), + ("batch-err-lanes", ["data/many", "--lanes", "99"]), +] + + +def run(cmd: list[str], cwd: Path) -> tuple[int, str, str]: + p = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + return p.returncode, p.stdout, p.stderr + + +def normalized(path: Path) -> bytes: + # Batch ZIP inputs are extracted to a random temporary folder that the report and metrics name. + return re.sub(rb"ampr-profile-[A-Za-z0-9_]+", b"ampr-profile-TEMP", path.read_bytes()) + + +def compare_trees(pdir: Path, cdir: Path) -> list[str]: + problems = [] + pfiles = sorted(str(p.relative_to(pdir)) for p in pdir.rglob("*") if p.is_file()) + cfiles = sorted(str(p.relative_to(cdir)) for p in cdir.rglob("*") if p.is_file()) + if pfiles != cfiles: + problems.append(f"files {pfiles} vs {cfiles}") + for f in sorted(set(pfiles) & set(cfiles)): + if normalized(pdir / f) != normalized(cdir / f): + diff = subprocess.run(["diff", str(pdir / f), str(cdir / f)], capture_output=True, text=True).stdout + problems.append(f"{f} differs:\n{diff[:3000]}") + return problems + + +def check(work: Path, python: str, mkpfs: Path, ampr_emu: Path) -> int: + tool = ampr_emu / "tools" / "ampr_pack_profile.py" + cases = [("generate", n, a) for n, a in GENERATE] + [("batch", n, a) for n, a in BATCH] + failures = 0 + for kind, name, args in cases: + outs = {} + for side in ("py", "cs"): + out = work / "out" / side / name + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + o = f"out/{side}/{name}" + if kind == "generate": + extra = ["--output", f"{o}/profile.toml", "--report", f"{o}/report.md", "--metrics", f"{o}/metrics.json", + "--runtime-header", f"{o}/runtime.h"] + case_args = args + else: + extra = ["--output-dir", f"{o}/profiles"] + case_args = [f"{o}/summary-custom.json" if a == "SUMMARY" else a for a in args] + cmd = [python, str(tool), kind, *case_args, *extra] if side == "py" else [str(mkpfs), "ampr", "profile", kind, *case_args, *extra] + code, so, se = run(cmd, work) + outs[side] = (code, so.replace(f"out/{side}/", "out/X/"), se.strip().splitlines()[-1:] if se.strip() else []) + (pc, pso, pse), (cc, cso, cse) = outs["py"], outs["cs"] + problems = [] + if pc != cc: + problems.append(f"exit {pc} vs {cc}") + if pso != cso: + problems.append(f"stdout\n py: {pso!r}\n cs: {cso!r}") + if pse != cse: + problems.append(f"stderr\n py: {pse!r}\n cs: {cse!r}") + problems += compare_trees(work / "out" / "py" / name, work / "out" / "cs" / name) + print(f"{'OK ' if not problems else 'FAIL'} {kind} {name} (exit {pc})") + for p in problems: + print(" ", p) + failures += bool(problems) + print(f"{failures} failing case(s) of {len(cases)}") + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT, help="work folder (default: tests/fixtures/generated/ampr_profile)") + parser.add_argument("--python", default=sys.executable, help="interpreter for ampr_pack_profile.py (needs python-lz4)") + parser.add_argument("--mkpfs", type=Path, default=DEFAULT_MKPFS, help="mkpfs executable") + parser.add_argument("--ampr-emu", type=Path, default=AMPR_EMU, help="ampr_emu checkout (default: ../ampr_emu)") + args = parser.parse_args() + work: Path = args.out.resolve() + if work.exists(): + shutil.rmtree(work) + build(work / "data") + build_batch_inputs(work / "data") + return check(work, args.python, args.mkpfs.resolve(), args.ampr_emu.resolve()) + + +if __name__ == "__main__": + raise SystemExit(main())