diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4723d071..6ae505da1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,7 @@ through one file: the connector, `consoles/__init__.py`. | `consoles/ps1/` | PlayStation | ROM | `exe.py` the game's program (PS-X EXE) and the disc's BOOT line; its discs and overlays through the PS2's readers · `guides/` | | `consoles/ps2/` | PlayStation 2 | ROM | `elf.py` the game's program · `disc.py`, `chd.py` disc images · `overlays.py` code in the disc's other files · `syscalls.py` kernel calls · `symtab.py`, `mdebug.py`, `stabs.py`, `cheader.py`, `debuginfo.py` the game's own debug symbols · `guides/` | | `consoles/gc_wii/` | GameCube, Wii | PROJECT | `scratch.py` a dtk-template project for the game · `guides/` | +| `consoles/ps4/` | PlayStation 4 | PROJECT | `scratch.py` reads decrypted Orbis ELF segments and function maps; `project_template/` builds strict objdiff targets with public clang; `guides/` | | `consoles/pc/` | Windows PC | PROJECT | `scratch.py` Ghidra finds the functions, decompdecomp cuts them out of the .exe · `project_template/` · `guides/` | | `consoles/nds/` | Nintendo DS | PROJECT | `scratch.py` a ds-decomp project for the ROM, a CodeWarrior version per source file · `project_template/` · `guides/` | diff --git a/README.md b/README.md index 898bdcef8..30a28645e 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,9 @@ brings back a window closed while helpers work. Free tools use Windows builds wh > GameCube, Wii, DS and Windows PC games, and N64 games built with IDO, don't need WSL: their compilers run on > Windows itself. +PS4 projects currently need the Linux app on x86-64 (including inside WSL 2), a decrypted `eboot.elf`, +and the pinned public Clang 5.0.2 available in Compilers & tools. No Sony SDK or game files are included. + ### macOS (Intel and Apple Silicon) Install the current source branch, or clone the repository and install the checkout: @@ -215,6 +218,7 @@ game's code is its rights holder's: keep it to yourself. | Nintendo 64 | Recommended | | PlayStation (PS1) | Beta | | PlayStation 2 | Beta | +| PlayStation 4 (decrypted ELF, Linux x86-64) | Beta | | PSP | Beta | | GameCube, Wii | Beta | | Windows PC games | Beta | diff --git a/ROADMAP.md b/ROADMAP.md index d881cc743..51b5f13bb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -15,43 +15,44 @@ Each feature and the platforms it covers today. A — is a gap: pick one up (the start). Keep this table true when a feature ships or reaches another platform. Each column's code is its console's folder, `src/decompnet/consoles//`; a new console adds a column (CONTRIBUTING.md, "Consoles"). -| Feature | N64 | PS1 | PS2 | PSP | GameCube · Wii | Windows PC | DS | Public decomps | -| :-- | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | -| Status | recommended | beta | beta | beta | beta | beta | beta | the project's own | -| Start from your own copy | ✓ | ✓ | ✓ | ✓ unencrypted or decrypted program and modules (ISO, CSO, CHD) | ✓ | ✓ | ✓ | fork by address | -| Judge | link · compare | link · compare | link · compare | link · compare | objdiff | objdiff | objdiff | objdiff | -| Machine drafts (m2c, Ghidra for x86) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ Ghidra | — | by architecture | -| Autopilot: free matches from drafts | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ exact drafts of tiny shapes; Ghidra's made compilable | — | — | -| Drafts follow the header, made in the background; `redraft` | ✓ | ✓ | ✓ | ✓ | — | ✓ Ghidra's | — | — | -| C++ methods drafted as `Class::method` | ✓ | ✓ | ✓ | ✓ | — | ✓ (RTTI, and classes from the vtables constructors store) | — | — | -| Twins: free matches from identical code | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Permuter on near misses | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Hand-written assembly set aside | ✓ | ✓ | ✓ | ✓ | ✓ | Visual C++'s exception-handling stubs | BIOS call stubs, pieces of functions | MIPS, PowerPC, ARM's BIOS stubs and pieces of functions | -| Pieces of functions joined to their function | — | — | — | — | — | ✓ (`scratch.merge_pieces`) | ✓ (`recover.merge_pieces`) | — | -| Agents' verdict "no C makes this" (3 agents of 2 models: set aside, red in the map) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Check again (and before helpers start, once after each update) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Kernel stubs set aside and named | — | set aside (BIOS calls) | ✓ | set aside, named by library and NID | — | ✓ import thunks | — | — | -| Function-end reports (`report_boundary`) | ✓ | ✓ | ✓ | ✓ | — | ✓ (agents agree, the rebuild checks) | — | — | -| Compiler ranges and suggestions | ✓ | ✓ | ✓ | ✓ | from scratch: per source file (`try_compiler`, Settings); else the project's build | per source file (`try_compiler`, Settings) | per source file (`try_compiler`, Settings) | the project's build | -| Flag search (`search_flags`) | ✓ | ✓ | ✓ | ✓ | the project's build | per source file (/Oy-, /O1, /Ob1) | the project's build | the project's build | -| Compiler map and the judge's survey | ✓ | ✓ | ✓ | ✓ | — | ✓ per source file, Rich header | — | — | -| Range checks: a range applies after its neighbours confirm it | ✓ | ✓ | ✓ | ✓ | — | ✓ over source files | — | — | -| C++ classes from the code (`get_class`) | ✓ | ✓ | ✓ | ✓ | — | ✓ (named by RTTI where the program has it) | — | — | -| Struct layouts from loads and stores (`guess_struct`) | ✓ | ✓ | ✓ | ✓ | — | ✓ | — | — | -| The game's own debug symbols: names, types, prototypes | — | — | ✓ symbol table, STABS, DWARF 1; the compiler of each object file | ✓ symbol table, DWARF 1 | — | ✓ PDB (Visual C++ 6.0's PDB 2.0, MSF 7.00), linker map, COFF symbols | — | — | -| Twins across your games | ✓ | ✓ | ✓ | ✓ | — | ✓ (between PC projects) | — | — | -| Shared knowledge (two models agree) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Tidy each source file once it's polished (shared types in one header, names, dead declarations) | — | — | — | — | ✓ | ✓ | ✓ | ✓ | -| Polish matched code (still matching) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Real names for the game's symbols (`rename_symbol`) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | dtk and ds-decomp projects | -| Save and restore (`progress/`, Open a saved project) | ✓ | ✓ | ✓ | ✓ | the project's git; best attempts saved | the project's git; best attempts saved | the project's git; best attempts saved | the project's git; best attempts saved | -| Saved README preserves notes, with completion bar and function map | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Reimport: a new split, matched work kept (Settings) | ✓ | ✓ | ✓ | ✓ | — | ✓ (Ghidra again, the newest rules) | — | — | -| Rebuildable export | ✓ | ✓ | ✓ | ✓ | the project | the project (`ninja rebuild`) | the project | the project | -| Progress bars for foreground jobs | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Rooms: decompile together | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Feature | N64 | PS1 | PS2 | PSP | PS4 | GameCube · Wii | Windows PC | DS | Public decomps | +| :-- | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | +| Status | recommended | beta | beta | beta | beta | beta | beta | beta | the project's own | +| Start from your own copy | ✓ | ✓ | ✓ | ✓ unencrypted or decrypted program and modules (ISO, CSO, CHD) | ✓ decrypted ELF | ✓ | ✓ | ✓ | fork by address | +| Judge | link · compare | link · compare | link · compare | link · compare | objdiff | objdiff | objdiff | objdiff | objdiff | +| Machine drafts (m2c, Ghidra for x86) | ✓ | ✓ | ✓ | ✓ | ✓ Ghidra | ✓ | ✓ Ghidra | — | by architecture | +| Autopilot: free matches from drafts | ✓ | ✓ | ✓ | ✓ | — | ✓ | ✓ exact drafts of tiny shapes; Ghidra's made compilable | — | — | +| Drafts follow the header, made in the background; `redraft` | ✓ | ✓ | ✓ | ✓ | — | — | ✓ Ghidra's | — | — | +| C++ methods drafted as `Class::method` | ✓ | ✓ | ✓ | ✓ | — | — | ✓ (RTTI, and classes from the vtables constructors store) | — | — | +| Twins: free matches from identical code | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Permuter on near misses | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Hand-written assembly set aside | ✓ | ✓ | ✓ | ✓ | — | ✓ | Visual C++'s exception-handling stubs | BIOS call stubs, pieces of functions | MIPS, PowerPC, ARM's BIOS stubs and pieces of functions | +| Pieces of functions joined to their function | — | — | — | — | — | — | ✓ (`scratch.merge_pieces`) | ✓ (`recover.merge_pieces`) | — | +| Agents' verdict "no C makes this" (3 agents of 2 models: set aside, red in the map) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Check again (and before helpers start, once after each update) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Kernel stubs set aside and named | — | set aside (BIOS calls) | ✓ | set aside, named by library and NID | — | — | ✓ import thunks | — | — | +| Function-end reports (`report_boundary`) | ✓ | ✓ | ✓ | ✓ | — | — | ✓ (agents agree, the rebuild checks) | — | — | +| Compiler ranges and suggestions | ✓ | ✓ | ✓ | ✓ | the project's build | from scratch: per source file (`try_compiler`, Settings); else the project's build | per source file (`try_compiler`, Settings) | per source file (`try_compiler`, Settings) | the project's build | +| Flag search (`search_flags`) | ✓ | ✓ | ✓ | ✓ | the project's build | the project's build | per source file (/Oy-, /O1, /Ob1) | the project's build | the project's build | +| Compiler map and the judge's survey | ✓ | ✓ | ✓ | ✓ | — | — | ✓ per source file, Rich header | — | — | +| Range checks: a range applies after its neighbours confirm it | ✓ | ✓ | ✓ | ✓ | — | — | ✓ over source files | — | — | +| C++ classes from the code (`get_class`) | ✓ | ✓ | ✓ | ✓ | — | — | ✓ (named by RTTI where the program has it) | — | — | +| Struct layouts from loads and stores (`guess_struct`) | ✓ | ✓ | ✓ | ✓ | — | — | ✓ | — | — | +| The game's own debug symbols: names, types, prototypes | — | — | ✓ symbol table, STABS, DWARF 1; the compiler of each object file | ✓ symbol table, DWARF 1 | — | — | ✓ PDB (Visual C++ 6.0's PDB 2.0, MSF 7.00), linker map, COFF symbols | — | — | +| Twins across your games | ✓ | ✓ | ✓ | ✓ | — | — | ✓ (between PC projects) | — | — | +| Shared knowledge (two models agree) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Tidy each source file once it's polished (shared types in one header, names, dead declarations) | — | — | — | — | — | ✓ | ✓ | ✓ | ✓ | +| Polish matched code (still matching) | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Real names for the game's symbols (`rename_symbol`) | ✓ | ✓ | ✓ | ✓ | ✓ functions | ✓ | ✓ | ✓ | dtk and ds-decomp projects | +| Save and restore (`progress/`, Open a saved project) | ✓ | ✓ | ✓ | ✓ | the project's git; best attempts saved | the project's git; best attempts saved | the project's git; best attempts saved | the project's git; best attempts saved | the project's git; best attempts saved | +| Saved README preserves notes, with completion bar and function map | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Reimport: a new split, matched work kept (Settings) | ✓ | ✓ | ✓ | ✓ | — | — | ✓ (Ghidra again, the newest rules) | — | — | +| Rebuildable export | ✓ | ✓ | ✓ | ✓ | — | the project | the project (`ninja rebuild`) | the project | the project | +| Progress bars for foreground jobs | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Rooms: decompile together | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | Where to start on a gap: +- **PS4**: initial Linux x86-64 support imports decrypted ELF programs and judges functions with strict objdiff. Encrypted SELF/PKG, PRX modules, whole-game relinking/export, compiler switching, source splitting and function-piece joining remain open. The public compiler needs no Sony SDK; fixed imported-global bindings must be configured per game. - **PowerPC (GameCube, Wii) and x86 (PC)**: class recovery and the compiler map work on a project's objects in fork mode, not on a ROM; the readers are the part to write (CodeWarrior's vtable pointer is often not at offset 0). x86 has its reader (`arch/x86_fields.py`): PC classes and struct layouts come from it (`Console.classes`, diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 605b1442b..26e1e8134 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -184,6 +184,7 @@ the game. | **Nintendo 64** | ★ **Recommended** | ✓ ROM | KMC GCC 2.7.2 · IDO 5.3 / 7.1 | ✓ export | ✓ | | **PlayStation** | Beta | ✓ disc (BIN, ISO, CHD) or PS-X EXE | GCC 2.6.3 · 2.7.2 · 2.8.1 (PSY-Q), with maspsx | ✓ export | ✓ | | **PlayStation 2** | Beta | ✓ disc or ELF | EE-GCC 2.9 · 2.95.2/2.95.3 (SN) · 2.96 · 3.2 · CodeWarrior 2.3.3 / 3.0.1 | ✓ export | ✓ | +| **PlayStation 4** | Beta | ✓ decrypted x86-64 Orbis ELF (Linux x86-64, including WSL 2) | Public Clang 5.0.2 (verify per game) | — | ✓ | | **PSP** | Beta | ✓ UMD image (ISO, CSO, CHD) with its other modules, or ELF/PRX, unencrypted or decrypted | CodeWarrior for PSP 3.0.1 · PSP SDK GCC 3.3.3 · SN pspsnc | ✓ export | ✓ | | **GameCube · Wii** | Beta | ✓ disc (ISO, RVZ, WBFS…) or DOL | CodeWarrior GC 1.0 – Wii 1.7 | ✓ dtk-template | ✓ | | **Windows PC** | Beta | ✓ program (32-bit x86 .exe) | Visual C++ 6.0 · 7.0 · 7.1 · 2005 · 2008 · 2010 | — | ✓ | @@ -230,6 +231,7 @@ of its own project. | :-- | :-- | | N64, PS1, PS2, PSP | splat and spimdisasm. They also find code that the game loads later, and overlays. | | GameCube, Wii | decomp-toolkit, in a [dtk-template](https://github.com/encounter/dtk-template) project (`consoles/gc_wii/scratch.py`) | +| PlayStation 4 | Ghidra, or an explicit function map selected with the decrypted executable. Strict objdiff projects; encrypted SELF/PKG, PRX modules and full-game relinking remain planned. | | Windows PC | [Ghidra](https://github.com/NationalSecurityAgency/ghidra). decompdecomp cuts each function out of the .exe and the game's own DLLs into a project for Visual C++ and objdiff (`consoles/pc/scratch.py`). | **C and C++.** GameCube, Wii, PS2 and Windows PC games are often C++. On these platforms, agents can write diff --git a/projects/demo-ps4/README.md b/projects/demo-ps4/README.md new file mode 100644 index 000000000..0e5966262 --- /dev/null +++ b/projects/demo-ps4/README.md @@ -0,0 +1,15 @@ +# PS4 importer demo + +This builds the repository's CC0 Demo Quest source with the pinned public clang +5.0.2 and lld. It needs no Sony SDK or game files. The output is a synthetic Orbis +ELF for importer and matching tests, not a bootable PS4 application. + +Install Clang 5.0.2 in Compilers & tools, then run: + +```sh +uv run python projects/demo-ps4/build.py +``` + +Open `build/demoquest.elf` in decompdecomp. Keep its adjacent +`demoquest.functions.jsonl` file: it supplies exact function extents without +requiring Ghidra. Generated files stay in the ignored `build/` directory. diff --git a/projects/demo-ps4/build.py b/projects/demo-ps4/build.py new file mode 100644 index 000000000..3938c74ed --- /dev/null +++ b/projects/demo-ps4/build.py @@ -0,0 +1,57 @@ +"""Build a shareable PS4-shaped ELF from Demo Quest's CC0 source. + +The pinned public clang and lld compile/link the demo. Its section table is removed +and its e_type changed to Orbis EXEC so the importer exercises the loader-header +path, not a game's symbols. The adjacent map supplies the known function extents. +This is an importer fixture, not a bootable PS4 application; no SDK is included. +""" +from __future__ import annotations + +import json +import struct +import subprocess +from pathlib import Path + +from elftools.elf.elffile import ELFFile + +from decompnet import toolchains +from decompnet.consoles.ps4.project_template.tools.compiler_flags import CL_FLAGS +from decompnet.limits import capped + +HERE = Path(__file__).resolve().parent +BUILD = HERE / "build" + + +def main() -> None: + """Write build/demo.json and the ELF/map used by console conformance tests.""" + compiler = toolchains.binary("clang-5.0.2") + linker = compiler.with_name("ld.lld") + BUILD.mkdir(exist_ok=True) + obj, game = BUILD / "game.o", BUILD / "demoquest.elf" + source = HERE.parent / "demo-n64" / "original" / "game.c" + for cmd in ([str(compiler), *CL_FLAGS, "-c", str(source), "-o", str(obj)], + [str(compiler), *CL_FLAGS, "-c", str(HERE / "extras.c"), "-o", str(BUILD / "extras.o")], + [str(linker), "-e", "abs_s32", "-Ttext=0x1000", str(obj), str(BUILD / "extras.o"), "-o", str(game)]): + subprocess.run(capped(cmd), check=True, timeout=120) + with game.open("rb") as file: + elf = ELFFile(file) + code = next(s for s in elf.iter_segments() if s["p_type"] == "PT_LOAD" and s["p_flags"] & 1) + symbols = {s.name: (s["st_value"], s["st_size"]) + for s in elf.get_section_by_name(".symtab").iter_symbols() + if s["st_info"]["type"] == "STT_FUNC" and s["st_size"]} + rows = [{"a": f"{addr-code['p_vaddr']:x}", "end": f"{addr+size-code['p_vaddr']:x}", "n": name} + for name, (addr, size) in sorted(symbols.items(), key=lambda item: item[1][0])] + data = bytearray(game.read_bytes()) + struct.pack_into(" str: _REGVAR = re.compile(r"\b(?:asm|__asm__|__asm)\s*\(\s*\"\$?[a-z]{1,2}\d{0,2}\"\s*\)", re.I) +# A typed extern with an assembler *name* changes linkage only; it emits no assembly. +# Keep this grammar narrow so instruction strings/extended asm still fail the check. +_LINKAGE_DECL = re.compile( + r'\bextern\s+__typeof__\s*\(\s*[A-Za-z_]\w*\s*\)\s+[A-Za-z_]\w*\s+' + r'__asm__\s*\(\s*"[A-Za-z_]\w*"\s*\)\s*' + r'__attribute__\s*\(\(\s*visibility\s*\(\s*"hidden"\s*\)\s*\)\)\s*;') + + def _strip_comments(code: str) -> str: - return _REGVAR.sub(" ", re.sub(r"//[^\n]*|/\*.*?\*/", " ", code, flags=re.S)) + return _REGVAR.sub(" ", _LINKAGE_DECL.sub(" ", re.sub(r"//[^\n]*|/\*.*?\*/", " ", code, flags=re.S))) def _rewrite(path: Path, text: str) -> None: @@ -2301,7 +2309,7 @@ def _draft(self, proj, fp: ForkProject, cfg: dict, fn, unit) -> str | None: found = ghidra.decompile_object(unit.target, fp.root / STATE / "ghidra") names = [meta.get("symbol") or fn["name"], *self._names(meta, fn)] body = next((found[n] for n in names if n in found), None) - text = body and "/* Ghidra */\n" + body.strip() + text = body and "/* Ghidra */\n" + consoles.get(cfg.get("platform") or "").adapt_draft(body.strip()) except OSError: text = None with self.net.store.tx() as db: diff --git a/src/decompnet/app/routes/state.py b/src/decompnet/app/routes/state.py index 7f94d6809..d2187d2c8 100644 --- a/src/decompnet/app/routes/state.py +++ b/src/decompnet/app/routes/state.py @@ -22,7 +22,7 @@ from ..support import MAX_DISC, job_error, static_file from . import App -DEBUG_SUFFIXES = (".pdb", ".map") # files sent with a game: debug information its import reads (consoles/pc/debuginfo.py) +DEBUG_SUFFIXES = (".pdb", ".map", ".functions.jsonl") # sidecars read by the PC and PS4 importers def system_name() -> str: @@ -128,16 +128,17 @@ async def api_events(request: Request) -> Response: def split_extras(game: Path, size: int, extras: str | None) -> None: """The files dropped with a game, sent after it in one upload (`extras`: [[name, bytes], ...]): a PC game's - program database or linker map, cut off the end and put beside it, where its import looks for them.""" + program database or linker map, or PS4 function map. Put them beside the uploaded program.""" if not extras: return try: wanted = [(Path(str(name)).name, int(n)) for name, n in json.loads(extras)] except (ValueError, TypeError): raise ValueError("the files sent with the game aren't listed right") from None - if any(not name.lower().endswith(DEBUG_SUFFIXES) or name == game.name or n < 0 for name, n in wanted) or \ + if any(not (name.lower().endswith(DEBUG_SUFFIXES) or name.lower() == "functions.jsonl") + or name == game.name or n < 0 for name, n in wanted) or \ sum(n for _, n in wanted) >= size: - raise ValueError("only a PC game's .pdb or .map can be sent with it") + raise ValueError("only .pdb, .map or functions.jsonl sidecars can be sent with a game") with open(game, "r+b") as f: f.seek(size - sum(n for _, n in wanted)) for name, n in wanted: diff --git a/src/decompnet/app/static/index.html b/src/decompnet/app/static/index.html index 00c16a076..e339311d8 100644 --- a/src/decompnet/app/static/index.html +++ b/src/decompnet/app/static/index.html @@ -81,7 +81,7 @@

Pick your game.
Your agents turn it into C and C++.

Drop a game here

-

ROM, disc image or program file (a PC game's .pdb or .map with it) · or choose a file

+

ROM, disc image or program file (PS4 function maps or PC .pdb / .map files can go with it) · or choose a file

N64PS1PS2PSPGameCubeWiiPCDS
diff --git a/src/decompnet/app/static/js/home.js b/src/decompnet/app/static/js/home.js index 493924760..9336f06f9 100644 --- a/src/decompnet/app/static/js/home.js +++ b/src/decompnet/app/static/js/home.js @@ -75,8 +75,8 @@ drop.addEventListener("drop", (e) => { pickFiles(e.dataTransfer.files); }); $("file").addEventListener("change", () => pickFiles($("file").files)); -/* A game, and the debug information that may come with it (a PC game's .pdb or linker .map: its import reads them). */ -const DEBUG_FILE = /\.(pdb|map)$/i; +/* A game with its PC debug symbols or PS4 function map. */ +const DEBUG_FILE = /\.(pdb|map)$|(?:^|\.)functions\.jsonl$/i; function pickFiles(list) { const files = [...list], debug = files.filter((f) => DEBUG_FILE.test(f.name)); const game = files.find((f) => !debug.includes(f)) || files[0]; @@ -202,6 +202,9 @@ function identifyHead(head, file) { if (isNds(head, dv)) return { ...base, platform: "nds", order: "DS ROM", code: ascii(12, 16), title: ascii(0, 12) || stem, entry: dv.getUint32(0x24, true) }; if (pe && pe + 0x1a <= head.length && text(pe, pe + 4) === "PE\0\0" && dv.getUint16(pe + 4, true) === 0x14c && dv.getUint16(pe + 24, true) === 0x10b) return { ...base, platform: "win32", order: "Windows program (.exe)", entry: dv.getUint32(pe + 52, true) + dv.getUint32(pe + 40, true), pack: pcPack(head, pe) }; + if (head.length >= 64 && text(0, 4) === "\x7fELF" && head[4] === 2 && head[5] === 1 + && [0xfe10, 0xfe04].includes(dv.getUint16(16, true)) && dv.getUint16(18, true) === 62) + return { ...base, platform: "ps4", order: "decrypted Orbis ELF", entry: Number(dv.getBigUint64(24, true)) }; if (text(0, 8) === "PS-X EXE") return { ...base, platform: "ps", order: "PS-X EXE program", entry: dv.getUint32(0x10, true) }; // a PSP's UMD image names its system in the volume descriptor ("PSP GAME"); which other PlayStation a disc is // for, its SYSTEM.CNF says (the server reads it, and a CHD's too, which the browser can't look into) @@ -272,8 +275,9 @@ async function pickFile(file, extras = []) { return toast(esc(NOT_A_GAME), "err"); } } - // only a PC game's import reads them (debuginfo.py): with another game they'd be uploaded for nothing - picked = { buf, info, name: file.name, extras: info.platform === "win32" ? extras : [] }; + // Send only the sidecars this console reads beside its uploaded program. + picked = { buf, info, name: file.name, extras: extras.filter((f) => info.platform === "win32" ? /\.(pdb|map)$/i.test(f.name) + : info.platform === "ps4" && /(?:^|\.)functions\.jsonl$/i.test(f.name)) }; if (!S.st) await loadState().catch(() => {}); // dropped before the page finished loading: the compiler lists $("romFile").textContent = file.name; $("romTitle").textContent = info.title; @@ -283,7 +287,7 @@ async function pickFile(file, extras = []) { ["Entry", hex8(info.entry)], ["Looks like", info.compiler === "IDO" ? "IDO" : "GCC (KMC)"]] : [["Platform", platformName(p)], ["Format", info.order], ["Size", MB(info.size)], info.code ? ["Game ID", info.code] : ["Entry", info.entry != null ? hex8(info.entry) : "from the disc"], - ["Project", consoleOf(p)?.layout || ""], ...(picked.extras.length ? [["Debug symbols", picked.extras.map((f) => f.name).join(", ")]] : [])]; + ["Project", consoleOf(p)?.layout || ""], ...(picked.extras.length ? [["Function maps / debug symbols", picked.extras.map((f) => f.name).join(", ")]] : [])]; $("romFacts").innerHTML = facts.map(([k, v]) => `
${k}
${esc(v)}
`).join(""); const level = platformStatus(p); $("romStatus").hidden = !level; diff --git a/src/decompnet/app/twins.py b/src/decompnet/app/twins.py index 54bee63bc..89181dad8 100644 --- a/src/decompnet/app/twins.py +++ b/src/decompnet/app/twins.py @@ -33,7 +33,8 @@ log = logging.getLogger("decompdecomp") IDENT = re.compile(r"^[A-Za-z_]\w*$") -_MACHINE = {"EM_PPC": ("ppc", "big"), "EM_MIPS": ("mips", None), "EM_ARM": ("arm", "little")} +_MACHINE = {"EM_PPC": ("ppc", "big"), "EM_MIPS": ("mips", None), "EM_ARM": ("arm", "little"), + "EM_X86_64": ("x86_64", "little")} _CACHE_VERSION = 5 # of twins.json: 2 added why code is hand-written assembly, 3 ARM's BIOS call stubs, 4 ARM's # system instructions, 5 COFF objects (PC) and the C names of their symbols # A twin's code that didn't match isn't tried again for this long while neither function's code changes (the diff --git a/src/decompnet/arch/x86.py b/src/decompnet/arch/x86.py index bf18dd992..73be0e2ea 100644 --- a/src/decompnet/arch/x86.py +++ b/src/decompnet/arch/x86.py @@ -17,7 +17,7 @@ RELOC_MASKS: dict[int, tuple[int, int, int]] = {} # every x86 relocation fills the 4 bytes at r_offset (the default) CAPSTONE = {False: (capstone.CS_ARCH_X86, capstone.CS_MODE_32)} -RELOC_MASKS_64 = {1: (0, 8, 0xFFFFFFFFFFFFFFFF)} # R_X86_64_64; the rest fill 4 bytes +RELOC_MASKS_64 = {1: (0, 8, 0xFFFFFFFFFFFFFFFF), 15: (0, 1, 0xFF)} # R_X86_64_64; the rest fill 4 bytes CAPSTONE_64 = {False: (capstone.CS_ARCH_X86, capstone.CS_MODE_64)} _READER = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32) diff --git a/src/decompnet/consoles/__init__.py b/src/decompnet/consoles/__init__.py index ebf95c78d..7f7f54a36 100644 --- a/src/decompnet/consoles/__init__.py +++ b/src/decompnet/consoles/__init__.py @@ -211,6 +211,10 @@ def tool_args(self) -> list[str]: """The arguments that point the project's configure step at this computer's pinned tools.""" return [] + def adapt_draft(self, text: str) -> str: + """Translate machine-draft type names into this console's project headers.""" + return text + def identity(self, path: Path) -> str | None: """Which game a file is (a sha1), when that isn't the file's own bytes: rooms check members have the same.""" return None @@ -407,12 +411,13 @@ def _registry() -> list[Console]: from .pc import PC from .ps1 import PS1 from .ps2 import PS2 + from .ps4 import PS4_CONSOLE from .psp import PSP from .public import PUBLIC_ONLY # the order sniff is asked in: a file is the first console's that says it's its own (the PSP and the PS1 before # the PS2, whose sniff takes any ISO 9660 disc and any MIPS ELF) - return [N64, PSP, PS1, PS2, GAMECUBE, WII, PC, NDS, *PUBLIC_ONLY] + return [N64, PSP, PS1, PS2, PS4_CONSOLE, GAMECUBE, WII, PC, NDS, *PUBLIC_ONLY] CONSOLES: list[Console] = _registry() diff --git a/src/decompnet/consoles/ps4/__init__.py b/src/decompnet/consoles/ps4/__init__.py new file mode 100644 index 000000000..52929a8f8 --- /dev/null +++ b/src/decompnet/consoles/ps4/__init__.py @@ -0,0 +1,97 @@ +"""PS4 decrypted x86-64 ELF programs, imported as strict objdiff projects. + +`scratch.py` reads loader segments, maps functions, relocations and data, and creates +projects from `project_template/`. `guides/ps4.md` describes the public compiler and +matching workflow. Encrypted SELF files and PS4 packages are not accepted. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from ...forks.project import ForkError +from .. import BETA, PROJECT, Compiler, Console + +# the EBoot's e_type: Sony's own ELF types, outside the standard ones +ET_SCE_EXEC, ET_SCE_DYNEXEC = 0xFE10, 0xFE04 +EM_X86_64 = 62 + + +class PS4(Console): + id = "ps4" + name = "PlayStation 4" + short = "PS4" + aliases = ("orbis",) + status = BETA + arch = "x86_64" + big_endian = False + mode = PROJECT + layout = "Ghidra + objdiff" + compilers = (Compiler("clang-5.0.2", "Clang 5.0.2 (Orbis era)", + "PS4-era x86-64 programs; verify the compiler for each game"),) + packs = (("clang-5.0.2", "public compiler for PS4-era ELF projects (Linux x86-64)"), + ("ghidra", "finds functions and provides machine drafts (needs Java 21+)")) + guides = ("ps4",) + compiler_guide = "ps4" + m2c = "" # m2c reads MIPS only + ram = (0x0, 0x40000000) # the image is linked at 0 and loaded wherever; words this low read as addresses + demo = "demo-ps4" + demo_tools = ("clang-5.0.2", "objdiff-cli") + compiler_family = "" # decomp-permuter has no clang weights; use its supported base set + + def sniff(self, head: bytes, file) -> bool: + """An x86-64 ELF, of one of Sony's executable types (an eboot.elf, not a library).""" + return (len(head) >= 64 and head[:4] == b"\x7fELF" and head[4] == 2 and head[5] == 1 + and int.from_bytes(head[16:18], "little") in (ET_SCE_EXEC, ET_SCE_DYNEXEC) + and int.from_bytes(head[18:20], "little") == EM_X86_64) + + def upload_name(self, data: bytes) -> str: + return "eboot.elf" + + def create_project(self, root: Path, game: Path, *, compiler: str | None = None, log=print) -> dict: + from . import scratch + + return scratch.create(root, game, compiler=compiler, log=log) + + def game_info(self, game: Path) -> dict: + from . import scratch + + return scratch.game_info(game) + + def tool_args(self) -> list[str]: + from . import scratch + + clang = scratch.clang_path() + if clang is None: + raise ForkError("no clang 5.0.2 for PS4 matching: install the pinned " + "build, then try again") + return ["--clang", str(clang)] + + def adapt_draft(self, text: str) -> str: + """Ghidra's C in this project's type names. It writes `undefined8` and `undefined4` where common.h has `u64` and + `u32`, and an agent handed those spends a turn translating instead of matching; its overlap warnings go too.""" + for theirs, ours in (("undefined8", "u64"), ("undefined4", "u32"), ("undefined2", "u16"), + ("undefined1", "u8"), ("undefined", "u8")): + text = re.sub(rf"\b{theirs}\b", ours, text) + return "\n".join(line for line in text.splitlines() if "WARNING: Globals starting with" not in line + and "WARNING: Removing unreachable" not in line) + + def rename_symbol(self, fp, version: str, old: str, new: str) -> dict: + """A function's real name, in the config the delinker names the target object's symbols from.""" + from . import scratch + + return scratch.rename(fp, version, old, new) + + def read_data(self, fp, name: str, length: int) -> str | None: + from . import scratch + + return scratch.read_data(fp.root, fp.manifest or {}, name, length) + + def data_bytes(self, fp, addr: int, length: int) -> bytes | None: + from . import scratch + + return scratch.data_bytes(fp.root, fp.manifest or {}, addr, length) + + +PS4_CONSOLE = PS4() diff --git a/src/decompnet/consoles/ps4/guides/ps4.md b/src/decompnet/consoles/ps4/guides/ps4.md new file mode 100644 index 000000000..b23229ed9 --- /dev/null +++ b/src/decompnet/consoles/ps4/guides/ps4.md @@ -0,0 +1,76 @@ +# PlayStation 4: initial ELF support + +Open a decrypted x86-64 Orbis executable (`eboot.elf`). Encrypted SELF files, +PKG packages and PRX libraries are not supported. Supply your own executable; +no game files or Sony SDK are downloaded. + +Install the pinned public Clang 5.0.2 in Compilers & tools. The initial compiler +is a starting point, not proof that every game used that version. This build +currently runs on Linux x86-64; Windows users can run the Linux app in WSL 2. + +## Import and build + +An adjacent `eboot.functions.jsonl` or `functions.jsonl` can supply functions. +Each line has `a` and `end`: hexadecimal offsets relative to the first executable +segment, with `end` exclusive. Optional `ranges` lists body intervals as numeric +offsets relative to that segment. `n` is an optional C identifier. Invalid or duplicate names use `func_
` instead. Body address +counts are not sizes: old maps with only `s` fall back to Ghidra. + +Drop or select the executable and its map together. When opening by path, keep the map +beside the executable. Uploaded `eboot.elf` files use the title "PS4 game"; rename +the project after importing. Overlapping bodies are trimmed to the next function +and reported in the import log. + +Without an explicit map, install Ghidra and its required JDK in Setup. Analysis +supplies function extents and body ranges. Check suspicious boundaries before +spending turns matching them. Shared tails remain external references. + +The generated project contains `orig/`, `config//functions.json`, +`include/common.h`, `src/auto/`, `tools/`, `build.ninja` and `objdiff.json`. +Run its configure command with the pinned compiler path, then `ninja prepare`. +The app regenerates tool paths when preparing a copied project. Paths with spaces +are supported. Build temporary experiments outside `src/`, where ninja will +otherwise compile them as part of the project. + +## Compiler and linker behavior + +The public compiler uses `-target x86_64-scei-ps4 -ffreestanding`. This selects the +PS4 ABI and global stack guard without a locally patched compiler. The project +keeps its starting flags in `tools/compiler_flags.py`. Headers for proprietary +platform APIs must be reconstructed from the user's code; none are supplied. + +GOT address loads for internal symbols are relaxed by the build to their linked +LEA form. Default-visibility extern declarations can preserve a separate address +load when the target has LEA followed by a load at an offset. Hidden visibility +can fold a reference directly into a RIP-relative memory operand. Choose by the +target instructions, not a blanket visibility rule. + +Imported globals stay unresolved. If analysis independently identifies a fixed +stack-guard GOT slot, the project owner can set `stack_guard_got` in +`config//functions.json` to its address (integer or `0x` string). The +build binds only that explicitly supplied address. Never borrow a slot address +from another game. Without it, canary-bearing functions may remain unmatched. + +## Matching + +Write C/C++ with the `func_
` and `D_
` names in the target. +Keep relocation identities: a different callee or global is a real mismatch. +Mapped BSS addresses are valid references but have no bytes to read from the +executable. Data inspection reports only file-backed contents. + +For scheduling-only differences, a per-function target attribute can help, but +it can also affect features, inlining and register allocation. Compile and compare +before keeping it. A branch-local `__atomic_signal_fence(__ATOMIC_ACQ_REL)` after +a pointer read can preserve separate branch reads without a hardware fence; +placing it before the read does not impose the same constraint. Neither is a +global default. Intrinsics can reconstruct vector helpers without the original +inline source. A local `target("no-sse")` can change large integer-only by-value +copies; it affects the whole function and is unsuitable for floating-point code. + +## Current limits + +This is beta, with strict per-function objdiff matching. It does not relink a +complete runnable game or export a verified rebuild. Compiler switching, source +splits, joining function pieces, automatic compiler surveys and class/structure +recovery need further work. Symbol renaming, data inspection, drafts, twins, +permuting and shared project workflows use the normal console interfaces. diff --git a/src/decompnet/consoles/ps4/project_template/tools/bind.py b/src/decompnet/consoles/ps4/project_template/tools/bind.py new file mode 100644 index 000000000..47d5b213e --- /dev/null +++ b/src/decompnet/consoles/ps4/project_template/tools/bind.py @@ -0,0 +1,201 @@ +"""Apply only explicit link-time bindings to compiled PS4 objects. + +GOT loads for internal symbols can be relaxed to LEA. An imported stack guard is +left unresolved unless the project's config explicitly supplies its GOT address. +No address is assumed from another game. The unbound object remains judgeable. +""" + +import argparse +import struct +import sys +from pathlib import Path + +GUARD_SYMBOL = "__stack_chk_guard" +GOTPCREL, GOTPCRELX, REX_GOTPCRELX, PC32 = 9, 41, 42, 2 + +SHDR = struct.Struct(" dict: + shoff, = struct.unpack_from(" list[tuple[str, int, int, int, int]]: + """(name, type, section index, value, size).""" + data, secs = tab["data"], tab["secs"] + symtab = next((s for s in secs if s["type"] == 2), None) + if symtab is None: + return [] + strtab = secs[symtab["link"]] + out = [] + for i in range(symtab["size"] // SYM.size): + name_off, info, _other, shndx, value, size = SYM.unpack_from(data, symtab["off"] + i * SYM.size) + start = strtab["off"] + name_off + end = data.index(b"\0", start) + out.append((data[start:end].decode("utf-8", "replace"), info & 0xF, shndx, value, size)) + return out + + +def unit_vaddr(obj: Path) -> int | None: + """A unit object is named after the game address it starts at: text_005C08A0 -> 0x5c08a0.""" + try: + return int(obj.stem.rsplit("_", 1)[1], 16) + except (IndexError, ValueError): + return None + + +def game_addresses(target: Path) -> dict[str, int]: + """name -> the address the game has that symbol at (from the delinked object, laid out at real offsets).""" + base = unit_vaddr(target) + if base is None: + return {} + tab = section_table(target.read_bytes()) + return {name: base + value for name, _t, shndx, value, _s in symbol_table(tab) if shndx != 0 and name} + + +def bind(obj: Path, target: Path | None, guard: int | None) -> int: + """Bind an explicitly configured guard slot; absent means preserve its relocation.""" + if guard is None: + return 0 + data = bytearray(obj.read_bytes()) + if data[:4] != b"\x7fELF": + return 0 + tab = section_table(bytes(data)) + secs, shoff, shentsize = tab["secs"], tab["shoff"], tab["shentsize"] + syms = symbol_table(tab) + addr_of = game_addresses(target) if target is not None else {} + if not addr_of: + return 0 + slot_name = f"D_{guard:08X}" + target_symbols = symbol_table(section_table(target.read_bytes())) if target is not None else [] + named_slot = any(name == slot_name for name, *_ in target_symbols) + if named_slot: + # The target delinker names mapped slots. Keep the operand relocated to + # that exact slot instead of turning it into an unlabelled literal. + symtab = next(s for s in secs if s["type"] == 2) + strtab = secs[symtab["link"]] + strings = bytes(data[strtab["off"]:strtab["off"] + strtab["size"]]) + name_offset = len(strings) + strings += slot_name.encode("utf-8") + b"\0" + for index, (name, *_rest) in enumerate(syms): + if name == GUARD_SYMBOL: + struct.pack_into("> 32, r_info & 0xFFFFFFFF + raw = bytes(data[sec["off"] + i * RELA.size: sec["off"] + (i + 1) * RELA.size]) + name = syms[sym_index][0] if sym_index < len(syms) else "" + if r_type in (GOTPCREL, GOTPCRELX, REX_GOTPCRELX) and name == GUARD_SYMBOL: + if named_slot: + kept.append(RELA.pack(r_offset, (sym_index << 32) | PC32, _addend)) + patched += 1 + continue + field = secs[sec["info"]]["off"] + r_offset + insn_end = func_addr + (r_offset - owner[3]) + 4 + struct.pack_into(" int: + """The linker's GOTPCREL relaxation, done here because nothing links these objects. + + `mov reg,[rip+GOT]` against a symbol the binary defines is relaxed by the linker into `lea reg,[rip+sym]` + (GOTPCRELX), which is why the game's code reads `lea rax,[D]; load [rax+off]` while our object holds the + pre-link `mov rax,[rip+GOT]`. Those are the same 7 bytes with 8B where the lea has 8D, so this rewrites the + opcode and turns the relocation into R_X86_64_PC32 with the addend kept (-4) - exactly what `ld` leaves behind. + + Only symbols this binary defines: `D_`/`func_` names the delinked target names, i.e. the game's own, which are + non-preemptible and therefore relaxable. An imported symbol (`__stack_chk_guard`, libc data) keeps its GOT load. + """ + data = bytearray(obj.read_bytes()) + if data[:4] != b"\x7fELF" or target is None or not target.exists(): + return 0 + tab = section_table(bytes(data)) + secs, syms = tab["secs"], symbol_table(tab) + known = {name for name, _t, shndx, _v, _s in symbol_table(section_table(target.read_bytes())) + if name and (shndx != 0 or name.startswith(("D_", "func_")))} + if not known: + return 0 + relaxed = 0 + for sec in secs: + if sec["type"] != 4: # SHT_RELA + continue + base = secs[sec["info"]]["off"] # sh_info: the section these relocations apply to + for i in range(sec["size"] // RELA.size): + at = sec["off"] + i * RELA.size + r_offset, r_info, r_addend = RELA.unpack_from(data, at) + sym_index, r_type = r_info >> 32, r_info & 0xFFFFFFFF + if r_type not in (GOTPCREL, GOTPCRELX, REX_GOTPCRELX): + continue + name = syms[sym_index][0] if sym_index < len(syms) else "" + if name not in known: + continue + field = base + r_offset # the disp32 + start = field - 3 # REX 8B modrm disp32, or 8B modrm disp32 + if start >= base and data[start] in (0x48, 0x49, 0x4C, 0x4D) and data[start + 1] == 0x8B \ + and (data[start + 2] & 0xC7) == 0x05: + data[start + 1] = 0x8D + elif field - 2 >= base and data[field - 2] == 0x8B and (data[field - 1] & 0xC7) == 0x05: + data[field - 2] = 0x8D + else: + continue # not the load-of-address shape: leave the GOT access alone + struct.pack_into(" int: + parser = argparse.ArgumentParser() + parser.add_argument("object") + parser.add_argument("--target", help="the delinked object the addresses come from (default: beside build/)") + parser.add_argument("--guard", type=lambda s: int(s, 0), default=None) + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args() + obj = Path(args.object) + target = Path(args.target) if args.target else obj.parent.parent.parent / "target" / obj.name + exists = target.exists() + n = bind(obj, target if exists else None, args.guard) + relaxed = relax_got(obj, target if exists else None) + if not args.quiet: + print(f"{obj}: bound {n} guard load(s), relaxed {relaxed} GOT load(s)" + + ("" if exists else f" (no target object at {target})")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/decompnet/consoles/ps4/project_template/tools/compile.py b/src/decompnet/consoles/ps4/project_template/tools/compile.py new file mode 100644 index 000000000..9c7de9e63 --- /dev/null +++ b/src/decompnet/consoles/ps4/project_template/tools/compile.py @@ -0,0 +1,36 @@ +"""Compile one PS4 unit with the pinned clang, without a shell parsing paths or flags.""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +COMPILE_SECONDS = 300 +NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--compiler", required=True) + parser.add_argument("--include", required=True) + parser.add_argument("--config", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--source", required=True) + parser.add_argument("flags", nargs=argparse.REMAINDER) + args = parser.parse_args() + flags = args.flags[1:] if args.flags[:1] == ["--"] else args.flags + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + command = [args.compiler, *flags, f"-I{args.include}", "-I.", "-c", args.source, "-o", args.output] + code = subprocess.run(command, timeout=COMPILE_SECONDS, creationflags=NO_WINDOW).returncode + if code == 0: + config = json.loads(Path(args.config).read_text(encoding="utf-8")) + command = [sys.executable, str(Path(__file__).with_name("bind.py")), args.output, "--quiet"] + if (guard := config.get("stack_guard_got")) is not None: + command += ["--guard", str(int(guard, 0) if isinstance(guard, str) else int(guard))] + code = subprocess.run(command, timeout=COMPILE_SECONDS, creationflags=NO_WINDOW).returncode + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/decompnet/consoles/ps4/project_template/tools/compiler_flags.py b/src/decompnet/consoles/ps4/project_template/tools/compiler_flags.py new file mode 100644 index 000000000..a61edc0b2 --- /dev/null +++ b/src/decompnet/consoles/ps4/project_template/tools/compiler_flags.py @@ -0,0 +1,6 @@ +"""Starting flags for public clang 5.0.2, using its PS4 ABI and global stack guard.""" + +# Settings (decompdecomp wrote these for this game; change them if the game was built differently) +CL_FLAGS = ["-target", "x86_64-scei-ps4", "-ffreestanding", "-O2", "-mavx", "-fno-omit-frame-pointer", "-fPIE", "-mllvm", "-x86-use-vzeroupper=0", + "-fstack-protector-strong", "-fno-asynchronous-unwind-tables", "-ffunction-sections", + "-mllvm", "-no-x86-call-frame-opt", "-mbmi", "-mllvm", "-unroll-runtime=false", "-w"] diff --git a/src/decompnet/consoles/ps4/project_template/tools/configure.py b/src/decompnet/consoles/ps4/project_template/tools/configure.py new file mode 100644 index 000000000..757004112 --- /dev/null +++ b/src/decompnet/consoles/ps4/project_template/tools/configure.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Write build.ninja and objdiff.json for this PlayStation 4 decompilation. + + python tools/configure.py VERSION --clang PATH_TO_CLANG_5_0_2 + +The build cuts every function out of your own copy of the eboot (orig/, see config/VERSION/functions.json) into ELF64 +objects, one per source file (tools/delink.py), and compiles src/ with the pinned clang. objdiff.json (for objdiff and +decompdecomp) compares every source file's functions with the original's. Made by decompdecomp. +""" + +import argparse +import io +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path + +try: + import fcntl +except ImportError: # Windows has no flock + fcntl = None + +import ninja_syntax +from compiler_flags import CL_FLAGS + +ROOT = Path(__file__).resolve().parent.parent +os.chdir(ROOT) +parser = argparse.ArgumentParser(description="Writes build.ninja and objdiff.json") +parser.add_argument("version", help="game version (a directory in config/)") +parser.add_argument("--clang", required=True, help="the pinned clang 5.0.2 to build src/ with") +args = parser.parse_args() + +VERSION = args.version +CONFIG = Path("config") / VERSION / "functions.json" +BUILD = Path("build") / VERSION +CLANG = Path(args.clang) + + +def q(argv: list[str]) -> str: + return subprocess.list2cmdline(argv) if sys.platform == "win32" else shlex.join(argv) + + +# the Python the build's steps run with: on Windows python.exe, not the console-less pythonw.exe +_W = Path(sys.executable) +PYTHON = str(_W.with_name("python.exe")) if _W.name.lower() == "pythonw.exe" and _W.with_name("python.exe").is_file() \ + else sys.executable +PY = q([PYTHON]).replace("$", "$$") + + +def write_atomic(path: Path, text: str) -> None: + """Write beside the target and rename: parallel builds each run configure, and a file written in place gets + interleaved into a corrupt one (ninja then refuses the whole project).""" + tmp = path.with_name(f"{path.name}.tmp{os.getpid()}") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + +def files(top: str, suffixes: tuple[str, ...]) -> list[Path]: + return sorted(p for p in Path(top).rglob("*") if p.suffix.lower() in suffixes) + + +def source_of(unit: str) -> str: + """A unit's source file.""" + c = Path("src") / "auto" / f"{unit}.c" + return c.as_posix() + + +def main() -> None: + # one generation at a time: every agent's build can trigger this rule, and two runs writing at once either + # interleave (a corrupt file ninja cannot rebuild, because it cannot parse it to find the rule) or finish out + # of order (a stale build.ninja replacing a fresh one) + lock = open("build.ninja.lock", "w") + if fcntl is not None: + fcntl.flock(lock, fcntl.LOCK_EX) + units = json.loads(CONFIG.read_text(encoding="utf-8"))["units"] + headers = [str(p) for p in files("include", (".h", ".hpp"))] + ninja_text = io.StringIO() # the writer goes to memory, then one atomic rename + n = ninja_syntax.Writer(ninja_text) + n.rule("delink", f"{PY} tools/delink.py {CONFIG} {BUILD / 'target'}", restat=True) + compiler = [PYTHON, "tools/compile.py", "--compiler", str(CLANG), "--include", "include", + "--config", str(CONFIG)] + n.rule("cc", q(compiler).replace("$", "$$") + " --output $out --source $in -- " + + q(CL_FLAGS).replace("$", "$$") + " $cflags") + n.rule("configure", f"{PY} tools/configure.py " + q(sys.argv[1:]).replace("$", "$$"), generator=True) + + targets = [str(BUILD / "target" / f"{u}.o") for u in units] + n.build(targets, "delink", [str(CONFIG), "tools/delink.py"]) + objs = [] + for src in files("src", (".c", ".cpp")): + obj = BUILD / src.with_suffix(".o") + # Binding uses target symbols: its object must exist before the + # compiler's post-processing, including the first parallel build. + target = BUILD / "target" / f"{src.stem}.o" + binding = [str(target)] if src.parent.as_posix() == "src/auto" and src.stem in units else [] + n.build(str(obj), "cc", str(src), implicit=[*headers, str(CONFIG), "tools/compile.py", "tools/bind.py", *binding]) + objs.append(str(obj)) + n.build("all_source", "phony", objs) + n.build("prepare", "phony", [*targets, *objs]) + n.build("build.ninja", "configure", ["tools/configure.py", "tools/compiler_flags.py", str(CONFIG), *headers, + *[str(s) for s in files("src", (".c", ".cpp"))]]) + n.default(["prepare"]) + write_atomic(Path("build.ninja"), ninja_text.getvalue()) + + objdiff = { + "$schema": "https://raw.githubusercontent.com/encounter/objdiff/main/config.schema.json", + "custom_make": "ninja", "custom_args": [], "build_target": False, "build_base": True, + "watch_patterns": ["*.c", "*.h", "*.hpp"], + "options": {"functionRelocDiffs": "all", "combineDataSections": True, "combineTextSections": True}, + "units": [{"name": u, "target_path": str(BUILD / "target" / f"{u}.o"), + "base_path": str(BUILD / "src" / "auto" / f"{u}.o"), + "metadata": {"source_path": source_of(u), "complete": False}} for u in units], + } + write_atomic(Path("objdiff.json"), json.dumps(objdiff, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/decompnet/consoles/ps4/project_template/tools/delink.py b/src/decompnet/consoles/ps4/project_template/tools/delink.py new file mode 100644 index 000000000..0534c854d --- /dev/null +++ b/src/decompnet/consoles/ps4/project_template/tools/delink.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Cut the game's functions out of your copy of the eboot into one ELF64 object file per source file, for objdiff. + + python tools/delink.py config//functions.json OUT_DIR + +A unit's object holds the game's bytes for that range of code unchanged (so a reference from one function to another +of the unit keeps its distance), each function as a global FUNC symbol at its own offset, and a relocation for every +reference that leaves its function: calls and jumps, rip-relative data addresses, absolute immediates. objdiff masks +the bytes a relocation fills, so the original's absolute address and the C's symbol compare as the same thing. +Standard library only. Made by decompdecomp. +""" + +import hashlib +import json +import struct +import sys +from pathlib import Path + +R_X86_64_64, R_X86_64_PC32, R_X86_64_PLT32, R_X86_64_32 = 1, 2, 4, 10 +RELOC_WIDTH = {1: 8, 2: 4, 4: 4, 10: 4, 11: 4, 15: 1} +ET_REL, EM_X86_64, SHT_RELA = 1, 62, 4 +SHF_ALLOC, SHF_EXECINSTR, SHF_INFO_LINK = 0x2, 0x4, 0x40 +STB_LOCAL, STB_GLOBAL, STT_NOTYPE, STT_FUNC = 0, 1, 0, 2 +EMPTY = 0 + + +def _align(data: bytearray, n: int) -> None: + while len(data) % n: + data.append(0) + + +def _section(name: int, stype: int, flags: int, addr: int, offset: int, size: int, link: int = 0, info: int = 0, + addralign: int = 1, entsize: int = 0) -> bytes: + return struct.pack(" bytes: + """One unit's object: .text (the game's bytes), .rela.text, .symtab, .strtab, .shstrtab.""" + text = bytearray(blob) + # the symbols: the unit's functions first (global), then whatever their code refers to (undefined) + symbols: list[tuple[str, int, int, int, int, int]] = [] # (name, info, shndx, value, size, section of name) + externs: dict[str, int] = {} + for fn in functions: + at = fn["addr"] - start + externs[fn["name"]] = len(symbols) + 1 # index zero is the ELF null symbol + symbols.append((fn["name"], (STB_GLOBAL << 4) | STT_FUNC, 1, at, fn["size"], 0)) + relocs = bytearray() + for index, fn in enumerate(functions): + for off, rtype, name, addend in fn.get("relocs") or []: + if not name: # a label inside the function: nothing to relocate across objects + continue + at = fn["addr"] - start + off + size = RELOC_WIDTH.get(rtype) + if size is None or off < 0 or off + size > fn["size"] or at < 0 or at + size > len(text): + raise ValueError(f"invalid relocation field in {fn['name']} at {off}: type {rtype}") + text[at:at + size] = bytes(size) # the addend lives in the relocation; objdiff masks the field + if name not in externs: + externs[name] = len(symbols) + 1 + symbols.append((name, (STB_GLOBAL << 4) | STT_NOTYPE, 0, 0, 0, 0)) + relocs += struct.pack(" None: + config, out = Path(sys.argv[1]), Path(sys.argv[2]) + info = json.loads(config.read_text(encoding="utf-8")) + data = Path(info["elf"]).read_bytes() + if hashlib.sha1(data).hexdigest() != info["sha1"]: + sys.exit(f"{info['elf']} isn't the copy this project was made from (its SHA-1 should be {info['sha1']})") + out.mkdir(parents=True, exist_ok=True) + for unit, functions in info["units"].items(): + start = functions[0]["addr"] + end = max(fn["addr"] + fn["size"] for fn in functions) + blob = data[functions[0]["offset"]:functions[0]["offset"] + (end - start)] + obj = elf64(unit, functions, blob, start) + path = out / f"{unit}.o" + if not path.exists() or path.read_bytes() != obj: # unchanged objects keep their time: nothing rebuilds + path.write_bytes(obj) + + +if __name__ == "__main__": + main() diff --git a/src/decompnet/consoles/ps4/project_template/tools/ninja_syntax.py b/src/decompnet/consoles/ps4/project_template/tools/ninja_syntax.py new file mode 100644 index 000000000..e0716e8aa --- /dev/null +++ b/src/decompnet/consoles/ps4/project_template/tools/ninja_syntax.py @@ -0,0 +1,229 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python module for generating .ninja files. + +Note that this is emphatically not a required piece of Ninja; it's +just a helpful utility for build-file-generation systems that already +use Python. +""" + +import re +import textwrap +from io import TextIOWrapper +from typing import Dict, List, Match, Optional, Tuple, Union + +def escape_path(word: str) -> str: + return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') + +class Writer(object): + def __init__(self, output: TextIOWrapper, width: int = 78) -> None: + self.output = output + self.width = width + + def newline(self) -> None: + self.output.write('\n') + + def comment(self, text: str) -> None: + for line in textwrap.wrap(text, self.width - 2, break_long_words=False, + break_on_hyphens=False): + self.output.write('# ' + line + '\n') + + def variable( + self, + key: str, + value: Optional[Union[bool, int, float, str, List[str]]], + indent: int = 0, + ) -> None: + if value is None: + return + if isinstance(value, list): + value = ' '.join(filter(None, value)) # Filter out empty strings. + self._line('%s = %s' % (key, value), indent) + + def pool(self, name: str, depth: int) -> None: + self._line('pool %s' % name) + self.variable('depth', depth, indent=1) + + def rule( + self, + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[str] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[str] = None, + rspfile_content: Optional[str] = None, + deps: Optional[Union[str, List[str]]] = None, + ) -> None: + self._line('rule %s' % name) + self.variable('command', command, indent=1) + if description: + self.variable('description', description, indent=1) + if depfile: + self.variable('depfile', depfile, indent=1) + if generator: + self.variable('generator', '1', indent=1) + if pool: + self.variable('pool', pool, indent=1) + if restat: + self.variable('restat', '1', indent=1) + if rspfile: + self.variable('rspfile', rspfile, indent=1) + if rspfile_content: + self.variable('rspfile_content', rspfile_content, indent=1) + if deps: + self.variable('deps', deps, indent=1) + + def build( + self, + outputs: Union[str, List[str]], + rule: str, + inputs: Optional[Union[str, List[str]]] = None, + implicit: Optional[Union[str, List[str]]] = None, + order_only: Optional[Union[str, List[str]]] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[Union[str, List[str]]]]], + Dict[str, Optional[Union[str, List[str]]]], + ] + ] = None, + implicit_outputs: Optional[Union[str, List[str]]] = None, + pool: Optional[str] = None, + dyndep: Optional[str] = None, + ) -> List[str]: + outputs = as_list(outputs) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in as_list(inputs)] + + if implicit: + implicit = [escape_path(x) for x in as_list(implicit)] + all_inputs.append('|') + all_inputs.extend(implicit) + if order_only: + order_only = [escape_path(x) for x in as_list(order_only)] + all_inputs.append('||') + all_inputs.extend(order_only) + if implicit_outputs: + implicit_outputs = [escape_path(x) + for x in as_list(implicit_outputs)] + out_outputs.append('|') + out_outputs.extend(implicit_outputs) + + self._line('build %s: %s' % (' '.join(out_outputs), + ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) + if dyndep is not None: + self._line(' dyndep = %s' % dyndep) + + if variables: + if isinstance(variables, dict): + iterator = iter(variables.items()) + else: + iterator = iter(variables) + + for key, val in iterator: + self.variable(key, val, indent=1) + + return outputs + + def include(self, path: str) -> None: + self._line('include %s' % path) + + def subninja(self, path: str) -> None: + self._line('subninja %s' % path) + + def default(self, paths: Union[str, List[str]]) -> None: + self._line('default %s' % ' '.join(as_list(paths))) + + def _count_dollars_before_index(self, s: str, i: int) -> int: + """Returns the number of '$' characters right in front of s[i].""" + dollar_count = 0 + dollar_index = i - 1 + while dollar_index > 0 and s[dollar_index] == '$': + dollar_count += 1 + dollar_index -= 1 + return dollar_count + + def _line(self, text: str, indent: int = 0) -> None: + """Write 'text' word-wrapped at self.width characters.""" + leading_space = ' ' * indent + while len(leading_space) + len(text) > self.width: + # The text is too wide; wrap if possible. + + # Find the rightmost space that would obey our width constraint and + # that's not an escaped space. + available_space = self.width - len(leading_space) - len(' $') + space = available_space + while True: + space = text.rfind(' ', 0, space) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + + if space < 0: + # No such space; just use the first unescaped space we can find. + space = available_space - 1 + while True: + space = text.find(' ', space + 1) + if (space < 0 or + self._count_dollars_before_index(text, space) % 2 == 0): + break + if space < 0: + # Give up on breaking. + break + + self.output.write(leading_space + text[0:space] + ' $\n') + text = text[space+1:] + + # Subsequent lines are continuations, so indent them. + leading_space = ' ' * (indent+2) + + self.output.write(leading_space + text + '\n') + + def close(self) -> None: + self.output.close() + + +def as_list(input: Optional[Union[str, List[str]]]) -> List[str]: + if input is None: + return [] + if isinstance(input, list): + return input + return [input] + + +def escape(string: str) -> str: + """Escape a string such that it can be embedded into a Ninja file without + further interpretation.""" + assert '\n' not in string, 'Ninja syntax does not allow newlines' + # We only have one special metacharacter: '$'. + return string.replace('$', '$$') + + +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + def exp(m: Match[str]) -> str: + var = m.group(1) + if var == '$': + return '$' + return local_vars.get(var, vars.get(var, '')) + return re.sub(r'\$(\$|\w*)', exp, string) diff --git a/src/decompnet/consoles/ps4/scratch.py b/src/decompnet/consoles/ps4/scratch.py new file mode 100644 index 000000000..f1df56614 --- /dev/null +++ b/src/decompnet/consoles/ps4/scratch.py @@ -0,0 +1,569 @@ +"""Start a PlayStation 4 decompilation from scratch: a project made for the user's own eboot.elf, judged by objdiff. + +An eboot is an ELF64 with Sony's own e_type. Section headers can be absent, so code and data are read through +the loader segments. Delinked objects retain each function's original byte span and reference identities. + +- `read_elf` / `ELF`: the file's PT_LOAD segments, entry point, address→offset. +- `map_functions`: the functions of the game, from a `functions.jsonl` map next to the file (what the reverse + engineering map produced: {"a": text offset hex, "end": exclusive end hex, "n": name}), + else from Ghidra's analysis. +- `analyse`: each function's extent, symbol (`func_00000320`, data `D_00002000`) and relocations, read with + capstone: calls and jumps to other functions, rip-relative data addresses, absolute immediates in the image. +- `units`: source files, runs of about 4 KiB of neighbouring functions. +- `create`: the project itself (project_template/), built once with the pinned clang. +""" + +from __future__ import annotations + +import bisect +import hashlib +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +import capstone +from capstone import x86 + +from decompnet.hostenv import NO_WINDOW + +from ...forks.project import ForkError + +PT_LOAD = 1 +PT_SCE_RELRO = 0x61000010 # loader-mapped, file-backed read-only data (including GOT slots) +PF_X = 0x1 +EM_X86_64 = 62 +HEAD_BYTES = 64 + +# ELF64 relocation types this writes into the target objects (objdiff masks the bytes they fill, so a function's +# absolute address in the game and the C's symbol both compare as "relocated") +R_X86_64_64, R_X86_64_PC32, R_X86_64_PLT32, R_X86_64_32, R_X86_64_32S = 1, 2, 4, 10, 11 +R_X86_64_PC8 = 15 + +UNIT_BYTES = 0x1000 +TEMPLATE = Path(__file__).with_name("project_template") +PAD = (0x90, 0xCC, 0x00) +READ_DEFAULT_BYTES, READ_MIN_BYTES, READ_MAX_BYTES = 64, 4, 512 +HEX_ROW_BYTES = 16 +POINTERS_SHOWN, FLOATS_SHOWN = 24, 8 +PLAUSIBLE_FLOAT = (1e-6, 1e7) +MIN_STRING_CHARS = 3 +PRINTABLE = range(32, 127) +TOOL_SECONDS = 3600 +ERROR_TAIL = 1500 + +TYPES_H = """/* Basic types for this decompilation (x86-64, clang 5.0.2, written in C). Agents add structs and prototypes here + * or in new headers next to it. The game's functions are func_
and its data D_
: write them with + * those names, as C, and the build gives them the original's symbols. Data: `extern int D_00002000;`. */ +#ifndef COMMON_H +#define COMMON_H + +typedef signed char s8; +typedef unsigned char u8; +typedef signed short s16; +typedef unsigned short u16; +typedef signed int s32; +typedef unsigned int u32; +typedef signed long long s64; +typedef unsigned long long u64; +typedef float f32; +typedef double f64; +typedef int BOOL; +#define TRUE 1 +#define FALSE 0 +#ifndef NULL +#define NULL 0 +#endif + +#endif +""" + +_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + + +@dataclass +class Segment: + va: int + offset: int + filesz: int + memsz: int + flags: int + + @property + def end(self) -> int: + return self.va + self.memsz + + +@dataclass +class ELF: + data: bytes + entry: int + e_type: int + segments: list[Segment] + + @property + def text(self) -> Segment: + """The executable segment: the game's code, and the constants the code reads in place.""" + for s in self.segments: + if s.flags & PF_X: + return s + raise ForkError("this ELF has no executable segment") + + def segment_at(self, addr: int) -> Segment | None: + return next((s for s in self.segments if s.va <= addr < s.va + max(s.filesz, s.memsz)), None) + + def offset(self, addr: int) -> int | None: + s = self.segment_at(addr) + return None if s is None or addr - s.va >= s.filesz else s.offset + addr - s.va + + def read(self, addr: int, n: int) -> bytes: + off = self.offset(addr) + segment = self.segment_at(addr) + return b"" if off is None or n < 0 else self.data[off:off + min(n, segment.filesz - (addr - segment.va))] + + def contains(self, addr: int) -> bool: + s = self.segment_at(addr) + return s is not None and addr - s.va < s.filesz + + +def is_elf64(head: bytes) -> bool: + return len(head) >= HEAD_BYTES and head[:4] == b"\x7fELF" and head[4] == 2 and head[5] == 1 \ + and int.from_bytes(head[18:20], "little") == EM_X86_64 + + +def read_elf(data: bytes) -> ELF: + if not is_elf64(data[:HEAD_BYTES]): + raise ForkError("this isn't an x86-64 ELF: PS4 games start from their eboot.elf") + e_type = struct.unpack_from(" len(data): + raise ForkError("this ELF's program headers are missing or past the end of the file") + segments = [] + for i in range(phnum): + at = phoff + phentsize * i + p_type, p_flags, p_offset, p_vaddr, _, p_filesz, p_memsz, _ = struct.unpack_from(" p_memsz or (p_filesz and p_offset + p_filesz > len(data)): + raise ForkError("this ELF has a truncated or invalid load segment") + segments.append(Segment(p_vaddr, p_offset, p_filesz, p_memsz, p_flags)) + if not segments: + raise ForkError("this ELF has no PT_LOAD segments") + return ELF(data, entry, e_type, segments) + + +def clang_path() -> Path | None: + """Only the registered, checksum-verified compiler; never a host clang from PATH.""" + from ... import toolchains + + if toolchains.installed("clang-5.0.2") is None: + return None + if sys.platform != "linux": + raise ForkError("PS4 builds currently need the Linux app on x86-64; on Windows run it inside WSL 2") + return toolchains.binary("clang-5.0.2") + + +def write_atomic(path: Path, text: str) -> None: + """Write beside the file and rename: the app and agents can act on the same project at once.""" + tmp = path.with_name(f"{path.name}.tmp{os.getpid()}") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + +def game_info(path: Path) -> dict: + data = path.read_bytes() + elf = read_elf(data) + stem = re.sub(r"[^a-z0-9]+", "", path.stem.lower()) or "game" + title = path.stem if path.stem.lower() != "eboot" else "PS4 game" + return {"platform": "ps4", "id": stem, "title": title, "region": "PS4", + "sha1": hashlib.sha1(data).hexdigest(), "entry": elf.entry, "e_type": elf.e_type, + "segments": len(elf.segments), + "compiler": "clang-5.0.2", "compiler_why": "starting compiler; verify its version and flags against the game"} + + +# ---------------------------------------------------------------------------------------------------- functions + + +@dataclass +class Function: + addr: int + size: int + name: str + relocs: list[tuple[int, int, str | None, int]] = field(default_factory=list) # (offset, type, symbol, addend) + + + +def is_padding(blob: bytes) -> bool: + """A run that is nothing but alignment nops. Function maps list some of these as functions because a scan saw + a call or a boundary there; no C can be them, since even an empty function is a `ret`.""" + if not blob or len(blob) > 64: + return False + instructions = list(capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64).disasm(blob, 0)) + return (sum(ins.size for ins in instructions) == len(blob) + and all(ins.mnemonic in ("nop", "int3") for ins in instructions)) + + +def map_functions(game: Path, text: Segment, data: bytes | None = None, *, log=print) -> list[dict] | None: + """The game's functions from a functions.jsonl beside it (offsets in the executable segment), else None. + With `data`, entries that are only alignment padding are dropped: they are not functions and no C is them.""" + candidates = [game.with_suffix(".functions.jsonl"), game.parent / "functions.jsonl", + game.parent / "functions" / "functions.jsonl"] + for candidate in candidates: + if not candidate.is_file(): + continue + out = [] + padding = 0 + for line in candidate.read_text(encoding="utf-8", errors="replace").splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + start = int(row["a"], 16) + # The legacy s field counts body addresses, not its span when blocks have gaps. + # Fall back to Ghidra rather than manufacturing a truncated target. + if "end" not in row: + log(f"{candidate}: legacy function map has no end addresses; using Ghidra") + return None + end_value = row["end"] + end = int(end_value, 16) if isinstance(end_value, str) else int(end_value) + size = end - start + except (ValueError, KeyError, TypeError): + continue + if start < 0 or size <= 0 or start + size > text.filesz: + raise ForkError(f"{candidate}: function extent is outside the executable segment") + if data is not None and is_padding(data[text.offset + start: text.offset + start + size]): + padding += 1 + continue + out.append({"entry": text.va + start, "end": text.va + start + size, + "name": row.get("n") or "", "source": "map", + "ranges": [[text.va + (int(a, 16) if isinstance(a, str) else int(a)), + text.va + (int(b, 16) if isinstance(b, str) else int(b))] + for a, b in row.get("ranges", [])]}) + if padding: + log(f"the map lists {padding} entries that are only alignment padding: dropped, they are not functions") + if out: + return out + return None + + +def _symbol_for(addr: int, names: dict[int, str]) -> str: + return names.get(addr) or f"D_{addr:08X}" + + +def analyse(elf: ELF, found: list[dict], *, log=print) -> list[Function]: + """Every function's extent, name and relocations, from the map's (or Ghidra's) function list.""" + md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) + md.detail = True + rows: dict[int, dict] = {} + for r in found: + start = int(r["entry"]) + if not elf.contains(start) or not elf.segment_at(start).flags & PF_X: + continue + rows[start] = r + starts = sorted(rows) + # Alignment-only map entries do not delimit another function's body. + boundaries = [s for s in starts if not is_padding(elf.read(s, int(rows[s].get("end") or s + 1) - s))] + names = {s: f"func_{s:08X}" for s in starts} + used = set(names.values()) + for start in starts: + name = rows[start].get("name", "") if rows[start].get("source") == "map" else "" + if isinstance(name, str) and name and _IDENT.fullmatch(name) and name not in used: + used.discard(names[start]) + names[start] = name + used.add(name) + out = [] + for start in starts: + next_index = bisect.bisect_right(boundaries, start) + segment = elf.segment_at(start) + file_end = segment.va + segment.filesz + limit = min(boundaries[next_index], file_end) if next_index < len(boundaries) else file_end + end = int(rows[start].get("end") or limit) + body_ranges = rows[start].get("ranges") + if body_ranges: + # Ghidra includes shared tails belonging to earlier/later functions. Keep the + # entry's local extent; references to those other bodies remain relocations. + local = [[max(lo, start), min(hi, limit)] for lo, hi in body_ranges if hi > start and lo < limit] + if not local: + log(f"skipping function 0x{start:X}: no body at its entry") + continue + if any(hi > limit for lo, hi in body_ranges if hi > start and lo < limit): + log(f"trimming overlapping body at 0x{start:X} to 0x{limit:X}") + end = max(hi for lo, hi in local) + elif end > limit: + log(f"trimming overlapping function at 0x{start:X} to 0x{limit:X}") + end = limit + if end <= start or end > file_end: + raise ForkError(f"invalid function extent at 0x{start:X}: end 0x{end:X}") + fn = Function(start, end - start, names[start]) + + def symbol(addr: int, start: int = start, end: int = end) -> tuple[str, int]: + if start <= addr < end: + return "", addr - start # inside this function: a label, not a relocation + return _symbol_for(addr, names), 0 + + ranges = local if body_ranges else [[start, end]] + instructions = [] + for lo, hi in ranges: + if hi > end or hi <= lo: + raise ForkError(f"invalid body range for function 0x{start:X}") + # Ghidra can include a shared return belonging to an earlier function. + # This symbol starts at entry; its outgoing branch remains a relocation. + if hi <= start: + continue + lo = max(lo, start) + instructions.extend(md.disasm(elf.read(lo, hi - lo), lo)) + for ins in instructions: + at = ins.address - start + ops = ins.operands + if (x86.X86_GRP_CALL in ins.groups or x86.X86_GRP_JUMP in ins.groups) and ops \ + and ops[0].type == x86.X86_OP_IMM: + target = ops[0].imm + if not start <= target < end and elf.contains(target): + name, _ = symbol(target) + # PC32, not PLT32: objdiff's x86 reader rejects PLT32 as an inline relocation + if ins.imm_size not in (1, 4): + raise ForkError(f"unsupported branch displacement at 0x{ins.address:X}") + kind = R_X86_64_PC8 if ins.imm_size == 1 else R_X86_64_PC32 + fn.relocs.append((at + ins.imm_offset, kind, name, -ins.imm_size)) + continue + for op in ops: + if op.type == x86.X86_OP_MEM and op.mem.base == x86.X86_REG_RIP and ins.disp_size == 4: + target = ins.address + ins.size + op.mem.disp + # BSS has runtime addresses but no file bytes. Relocate references to + # mapped memory without making those addresses readable from the ELF. + if elf.segment_at(target) is not None: + name, addend = symbol(target) + fn.relocs.append((at + ins.disp_offset, R_X86_64_PC32, name, addend - (ins.size - ins.disp_offset))) + elif op.type == x86.X86_OP_IMM: + # a 64-bit immediate (movabs) that names a function or lands in the data segment is an address: + # relocate it (its field is 2 bytes into the instruction, never at a boundary, which is what + # objdiff refuses). 32-bit immediates stay literal: their field can coincide with a boundary the + # way objdiff decodes a unit, and one such relocation fails the whole report. + seg = elf.segment_at(op.imm) + if ins.imm_size == 8 and op.imm >= 0x1000 and \ + (op.imm in names or (seg is not None and not seg.flags & PF_X)): + name, addend = symbol(op.imm) + fn.relocs.append((at + ins.imm_offset, R_X86_64_64, name, addend)) + fn.relocs.sort() + out.append(fn) + return out + + + +GAP_BYTES = 0x400 # a hole this big between functions means another region of code, not padding + + +def units(functions: list[Function], elf: ELF | None = None) -> dict[str, list[Function]]: + """Source files: runs of neighbouring functions of about UNIT_BYTES each.""" + out: dict[str, list[Function]] = {} + cur: list[Function] = [] + for fn in functions: + if cur and (fn.addr + fn.size - cur[0].addr > UNIT_BYTES or fn.addr - (cur[-1].addr + cur[-1].size) > GAP_BYTES + or (elf is not None and elf.offset(fn.addr) - elf.offset(cur[-1].addr) != fn.addr - cur[-1].addr)): + out[f"text_{cur[0].addr:08X}"] = cur + cur = [] + cur.append(fn) + if cur: + out[f"text_{cur[0].addr:08X}"] = cur + return out + + +# ---------------------------------------------------------------------------------------------------- data + + +def _program(root: Path, manifest: dict) -> ELF: + path = root / manifest["game"]["path"] + return read_elf(path.read_bytes()) + + +def data_bytes(root: Path, manifest: dict, addr: int, length: int) -> bytes | None: + try: + elf = _program(root, manifest) + except Exception: # no copy, or one that doesn't parse: the diff goes without values + return None + if not elf.contains(addr): + return None + data = elf.read(addr, length) + return data if data and len(data) == length else None + + +def read_data(root: Path, manifest: dict, name: str, length: int = READ_DEFAULT_BYTES) -> str: + """The bytes at a D_
of the user's copy, the addresses its words point to, and a string or floats.""" + m = re.fullmatch(r"_?D_([0-9A-Fa-f]{6,8})|(?:0x)?([0-9A-Fa-f]{6,8})", name) + if m is None: + raise ForkError(f"`{name}` isn't data by address: PS4 data is named D_
(D_00002000)") + addr = int(m.group(1) or m.group(2), 16) + elf = _program(root, manifest) + seg = elf.segment_at(addr) + if seg is None: + raise ForkError(f"0x{addr:08X} isn't in the game's program") + n = max(READ_MIN_BYTES, min(int(length or READ_DEFAULT_BYTES), READ_MAX_BYTES)) + data = elf.read(addr, n) + out = [f"D_{addr:08X} (+0x{addr - seg.va:X} of the segment at 0x{seg.va:08X}, {n} bytes)"] + if not data: + return out[0] + ": uninitialized data, zeros at start-up." + for i in range(0, len(data), HEX_ROW_BYTES): + chunk = data[i:i + HEX_ROW_BYTES] + out.append(f" +0x{i:03X}: {chunk.hex(' '):<47} {''.join(chr(c) if c in PRINTABLE else '.' for c in chunk)}") + words = [w for (w,) in struct.iter_unpack(" 0x{w:08X}" for at, w in ptrs[:POINTERS_SHOWN])) + floats = [struct.unpack("= MIN_STRING_CHARS and all(c in PRINTABLE or c in b"\t\n" for c in text): + out.append(f"as a string: {json.dumps(text.decode('latin-1'))}") + return "\n".join(out) + + +def rename(fp, version: str, old: str, new: str) -> dict: + """Rename one of the game's functions in the project's config, where the delinker takes every target symbol from + (`func_` -> a real name). The engine renames the C and checks that every match still matches. Returns the + files changed, with the text they had (ForkEngine puts them back if the rename doesn't hold).""" + path = Path(fp.root) / "config" / version / "functions.json" + text = path.read_text(encoding="utf-8") + cfg = json.loads(text) + hit = False + for functions in cfg.get("units", {}).values(): + for fn in functions: + if fn.get("name") == old: + fn["name"] = new + hit = True + for reloc in fn.get("relocs", []): + if reloc[2] == old: + reloc[2] = new + if not hit: + raise ForkError(f"{old} isn't a function in config/{version}/functions.json: only the game's functions " + f"(func_
) can be renamed for now, not its data") + write_atomic(path, json.dumps(cfg, indent=1)) + return {path: text} + + +# ---------------------------------------------------------------------------------------------------- the project + + +def _run(cmd: list[str], root: Path, what: str, log) -> None: + from ...forks.project import ForkProject + + ok, tail = ForkProject(root).run(cmd, log=log, timeout=TOOL_SECONDS) + if not ok: + raise ForkError(f"{what} failed:\n{tail[-ERROR_TAIL:]}") + + +def ghidra_functions(game: Path, elf: ELF) -> list[dict]: + """Stock Ghidra needs a standard ELF type; only its disposable input is changed (#92).""" + from ...forks import ghidra + from . import ET_SCE_DYNEXEC, ET_SCE_EXEC + + if elf.e_type not in (ET_SCE_EXEC, ET_SCE_DYNEXEC): + return ghidra.functions(game) + with tempfile.TemporaryDirectory(prefix="decomp-ps4-") as tmp: + target = Path(tmp) / game.name + data = bytearray(elf.data) + struct.pack_into(" dict: + """Make the project in `root` (which must not exist yet) and build it once. Returns facts about the game.""" + from ...forks import ghidra + + clang = clang_path() + if clang is None: + raise ForkError("PS4 projects need the pinned Clang 5.0.2: install it in Compilers & tools, " + "then try again") + if compiler not in (None, "clang-5.0.2"): + raise ForkError(f"unsupported PS4 compiler {compiler!r}; choose clang-5.0.2") + info = game_info(game) + version = info["id"] + data = game.read_bytes() + elf = read_elf(data) + text = elf.text + log(f"reading {game.name}: {len(elf.segments)} segments, code at 0x{text.va:X} ({text.filesz} bytes)") + + found = map_functions(game, text, data, log=log) + if found: + log(f"using the function map beside the game: {len(found)} functions") + else: + if not ghidra.available(): + raise ForkError("no functions.jsonl map beside the game and no Ghidra to find them: put one there " + "({\"a\": offset hex, \"end\": exclusive end hex, \"n\": name} per line) or install Ghidra in Setup") + log(f"analysing {game.name} with Ghidra (finds every function; a big game takes a few minutes)") + found = ghidra_functions(game, elf) + if not found: + raise ForkError("neither a function map nor Ghidra found functions in this program") + functions = analyse(elf, found, log=log) + if not functions: + raise ForkError("the function map gave no function in this program's code") + groups = units(functions, elf) + log(f"{len(functions)} functions in {len(groups)} source files") + + shutil.copytree(TEMPLATE, root) + (root / "orig").mkdir() + eboot = root / "orig" / f"{version}.elf" + try: + eboot.hardlink_to(game) + except OSError: + shutil.copyfile(game, eboot) + (root / "include").mkdir() + (root / "include" / "common.h").write_text(TYPES_H, encoding="utf-8") + conf = root / "config" / version + conf.mkdir(parents=True) + (conf / "functions.json").write_text(json.dumps({ + "target_format_version": 3, "elf": f"orig/{version}.elf", "sha1": info["sha1"], "image_base": min(s.va for s in elf.segments), "entry": elf.entry, + "text": {"va": text.va, "offset": text.offset, "size": text.filesz}, + "units": {u: [{"name": f.name, "addr": f.addr, "offset": elf.offset(f.addr), "size": f.size, + "relocs": [list(r) for r in f.relocs]} for f in fns] for u, fns in groups.items()}, + }, indent=1), encoding="utf-8") + for u, fns in groups.items(): + src = root / "src" / "auto" / f"{u}.c" + src.parent.mkdir(parents=True, exist_ok=True) + src.write_text(f"/* {len(fns)} function{'s' if len(fns) != 1 else ''} from 0x{fns[0].addr:08X} to " + f"0x{fns[-1].addr + fns[-1].size:08X}, from the game's function map. */\n" + "#include \"common.h\"\n", encoding="utf-8") + log(f"{len(groups)} source files for agents to write into") + + args = ["--clang", str(clang)] + for cmd in ([sys.executable, "tools/configure.py", version, *args], [str(Path(sys.executable).with_name("ninja")) if Path(sys.executable).with_name("ninja").is_file() else "ninja", "prepare"]): + _run(cmd, root, "the first build", log) + if not (root / "objdiff.json").exists(): + raise ForkError("the first build finished without writing objdiff.json") + + (root / ".decomp.wtf.json").write_text(json.dumps({ + "created": "from scratch", "platform": "ps4", "version": version, "compiler": "clang-5.0.2", + "game": {"what": "your own copy of the game's eboot.elf", "path": f"orig/{version}.elf", "sha1": info["sha1"]}, + "setup": [["python", "tools/configure.py", version, *args], ["ninja", "prepare"]], + }, indent=2), encoding="utf-8") + (root / ".gitignore").write_text("# made from your copy of the game, or by the build\norig/\nbuild/\nbuild.ninja\n" + ".ninja_*\nbuild.ninja.lock\nobjdiff.json\n__pycache__/\n.decomp-wtf/\n", encoding="utf-8") + (root / "README.md").write_text(f"""# {info['title']} decompilation + +Started from scratch with [decompdecomp](https://github.com/n-delic/decompdecomp): the game's functions are cut out of +your own `eboot.elf` into ELF64 objects and compared, function by function, with what `src/` compiles to, with objdiff. + +1. Your copy of the game's program is here as `orig/{version}.elf` (never commit it). +2. `python tools/configure.py {version} --clang `, then `ninja`. +3. Decompiled code goes in `src/`; `objdiff.json` shows each function's progress. + +Compiler: the pinned public clang 5.0.2 with the PS4 target and freestanding headers. +See `tools/compiler_flags.py` for the starting flags; verify them against this game. `config/{version}/functions.json` holds every function's address, size, symbol and relocations. +""", encoding="utf-8") + if shutil.which("git"): + for cmd in (["git", "init", "-q", "-b", "main"], ["git", "add", "-A"], + ["git", "-c", "user.name=decompdecomp", "-c", "user.email=noreply@decompdecomp.invalid", "commit", + "-q", "-m", f"Start the {info['title']} decompilation (decompdecomp, PS4)"]): + subprocess.run(cmd, cwd=root, capture_output=True, creationflags=NO_WINDOW) + return {"platform": "ps4", "id": version, "title": info["title"], "compiler": "clang-5.0.2", + "sources": len(groups), "sha1": info["sha1"]} diff --git a/src/decompnet/forks/ghidra_scripts/DecompWtfFunctions.java b/src/decompnet/forks/ghidra_scripts/DecompWtfFunctions.java index 4a76ce9b9..422920d26 100644 --- a/src/decompnet/forks/ghidra_scripts/DecompWtfFunctions.java +++ b/src/decompnet/forks/ghidra_scripts/DecompWtfFunctions.java @@ -33,7 +33,14 @@ public void run() throws Exception { if (f.isExternal()) continue; long entry = f.getEntryPoint().getOffset(); long end = f.getBody().getMaxAddress().getOffset() + 1; + StringBuilder ranges = new StringBuilder(); + for (ghidra.program.model.address.AddressRange range : f.getBody().getAddressRanges()) { + if (ranges.length() > 0) ranges.append(','); + ranges.append('[').append(range.getMinAddress().getOffset()).append(',') + .append(range.getMaxAddress().getOffset() + 1).append(']'); + } w.println("{\"entry\": " + entry + ", \"end\": " + end + ", \"name\": " + quote(f.getName()) + + ", \"ranges\": [" + ranges + "]" + ", \"source\": " + quote(f.getSymbol().getSource().toString()) + ", \"purge\": " + f.getStackPurgeSize() + ", \"convention\": " + quote(String.valueOf(f.getCallingConventionName())) diff --git a/src/decompnet/toolchains.toml b/src/decompnet/toolchains.toml index c49542896..2659113c3 100644 --- a/src/decompnet/toolchains.toml +++ b/src/decompnet/toolchains.toml @@ -908,3 +908,18 @@ url = "https://github.com/encounter/dtk-template/archive/95a941f755919ebe50c1725 sha256 = "c49f889a4e092388beef4867340887ec6f0410bd1055d79a8f8850bea89ada61" subdir = "dtk-template-95a941f755919ebe50c1725a4ce73524470e7a02" + +["clang-5.0.2"] +name = "Clang 5.0.2 (PS4-era x86-64)" +license = "NCSA" +source = "LLVM 5.0.2 official release, Ubuntu 16.04 x86-64" +description = "Compiles decrypted PS4 ELF projects with the public PS4 target; no Sony SDK is supplied." +platform = "ps4" +arch = "x86_64" +binary_name = "clang" +cc = ["{tc}/bin/clang", "-target", "x86_64-scei-ps4", "-ffreestanding", "-c"] +[["clang-5.0.2".parts]] +url = "https://releases.llvm.org/5.0.2/clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-16.04.tar.xz" +sha256 = "15a97aa654b529793da7d115bcfdd8b93281d94f86ec98f0850aff7cf82cbb3f" +subdir = "clang+llvm-5.0.2-x86_64-linux-gnu-ubuntu-16.04" +platforms = ["linux-x86_64"] diff --git a/tests/gui_polls.js b/tests/gui_polls.js index ed06d22ad..69d1e0122 100644 --- a/tests/gui_polls.js +++ b/tests/gui_polls.js @@ -134,6 +134,16 @@ const settle = async () => { for (let i = 0; i < SETTLE_TICKS; i++) await tick() S.ov = before; return { guest, host, alone }; })(), + ps4: (() => { + const h = new Uint8Array(64), dv = new DataView(h.buffer); + h.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); + dv.setUint16(18, 62, true); + const file = { name: "eboot.elf", size: 256 }; + return [0xfe10, 0xfe04, 2, 0xfe18].map((type) => { + dv.setUint16(16, type, true); + return identifyHead(h, file)?.platform ?? null; + }); + })(), dropped: (() => { const head = (at, text) => { const h = new Uint8Array(0x10000); [...text].forEach((c, i) => (h[at + i] = c.charCodeAt(0))); return h; }; const file = { name: "SLUS_000.01", size: 0x10000 }; diff --git a/tests/test_consoles.py b/tests/test_consoles.py index 661318068..a96a0344b 100644 --- a/tests/test_consoles.py +++ b/tests/test_consoles.py @@ -137,7 +137,12 @@ def there(tool: str) -> bool: return toolchains.installed(tool) is not None except toolchains.ToolchainError: # a program on PATH, not a toolchain return shutil.which(tool) is not None - return [t for t in console.demo_tools if not there(t)] + ([] if ZLIB.exists() else ["zlib (projects/zlib-n64/prepare.py)"]) + missing = [t for t in console.demo_tools if not there(t)] + # Read the demo's own prerequisites, including consoles added after this test. + build = ROOT / "projects" / getattr(console, "demo", "") / "build.py" + if build.is_file() and "zlib-n64" in build.read_text() and not ZLIB.exists(): + missing.append("zlib (projects/zlib-n64/prepare.py)") + return missing @pytest.fixture(scope="module", params=DEMOS, ids=[c.id for c in DEMOS]) @@ -169,7 +174,8 @@ def _function(net, slug: str, symbols: dict, name: str) -> tuple[str, str]: address = next(a for s, a in symbols.items() if re.sub(r"^[_@]|@\d+$", "", s) == name) pid = net.project_row(slug)["id"] row = net.store.one("SELECT name FROM functions WHERE project_id=? AND vram=?", pid, address) or next( - r for r in net.store.all("SELECT name FROM functions WHERE project_id=?", pid) if f"{address:08x}" in r["name"].lower()) + r for r in net.store.all("SELECT name FROM functions WHERE project_id=?", pid) if f"{address:08x}" in r["name"].lower() + or re.sub(r"^[_@]|@\d+$", "", r["name"]) == name) plain = re.sub(r"^[_@]|@\d+$", "", row["name"]) # (a PC function's name says its calling convention) return row["name"], csource.get(DEMO_GAME.read_text(), name).replace(f"{name}(", f"{plain}(", 1) diff --git a/tests/test_gui.py b/tests/test_gui.py index b78140839..646b8e732 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -176,3 +176,7 @@ def test_the_drop_zone_knows_the_visual_cpp_ids_the_import_knows(): assert js == {v: {*compilers, *glued} for v, compilers, glued in scratch.RICH_COMPILERS} linkers = re.search(r"const RICH_LINKERS = \[(.*?)\];", text, re.S).group(1) assert {int(x, 16) for x in re.findall(r"0x[0-9a-f]+", linkers)} == scratch.RICH_LINKERS + + +def test_ps4_decrypted_executables_are_accepted_but_other_elfs_are_not(gui): + assert gui["checks"]["ps4"] == ["ps4", "ps4", None, None] diff --git a/tests/test_pc_debug.py b/tests/test_pc_debug.py index c750056d5..689664016 100644 --- a/tests/test_pc_debug.py +++ b/tests/test_pc_debug.py @@ -369,8 +369,15 @@ def imported(self, game, **kw): if (state := c.get(f"/api/job?id={j['job']}").json())["state"] != "running": break assert state["state"] == "done" and seen == {"game.exe": b"MZ-game", "game.pdb": b"pdb", "game.map": b"mp"} + seen.clear() + extras = json.dumps([["eboot.functions.jsonl", 3]]) + j = c.post(f"/api/import?confirmed=1&name=eboot.elf&extras={extras}", content=b"elf-game" + b"map").json() + for _ in range(300): + if (state := c.get(f"/api/job?id={j['job']}").json())["state"] != "running": + break + assert state["state"] == "done" and seen == {"eboot.elf": b"elf-game", "eboot.functions.jsonl": b"map"} bad = c.post(f"/api/import?confirmed=1&name=game.exe&extras={json.dumps([['x.dll', 3]])}", content=b"MZ-gamex.d") - assert bad.status_code == 400 and ".pdb or .map" in bad.json()["error"] + assert bad.status_code == 400 and ".pdb, .map" in bad.json()["error"] @pytest.mark.skipif(toolchains.installed("msvc") is None or toolchains.installed("objdiff-cli") is None diff --git a/tests/test_ps4_targets.py b/tests/test_ps4_targets.py new file mode 100644 index 000000000..2c2df012c --- /dev/null +++ b/tests/test_ps4_targets.py @@ -0,0 +1,444 @@ +"""Target objects must preserve instructions and the destinations the original code uses.""" +import importlib.util +import io +import json +import shutil +import subprocess +import sys + +import pytest +from elftools.elf.elffile import ELFFile + +from decompnet import toolchains +from decompnet.consoles.ps4 import scratch + + +def delinker(): + spec = importlib.util.spec_from_file_location('ps4_delink_test', scratch.TEMPLATE / 'tools/delink.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def image(code, base=0x1000): + return scratch.ELF(code, base, 2, [scratch.Segment(base, 0, len(code), len(code), 1)]) + + +def object_for(code, relocs, size=None): + f = {'name': 'demo', 'addr': 0, 'size': size or len(code), 'relocs': relocs} + return delinker().elf64('demo', [f], code, 0) + + +def refs(blob): + elf = ELFFile(io.BytesIO(blob)) + syms = elf.get_section_by_name('.symtab') + return [(r['r_offset'], r['r_info_type'], syms.get_symbol(r['r_info_sym']).name, r['r_addend']) + for r in elf.get_section_by_name('.rela.text').iter_relocations()] + + +def test_symbol_index_accounts_for_null_and_reuses_defined_symbols(): + code = bytes.fromhex('e800000000e900000000') + obj = object_for(code, [(1, 2, 'external', -4), (6, 2, 'demo', -4)]) + assert refs(obj) == [(1, 2, 'external', -4), (6, 2, 'demo', -4)] + names = [s.name for s in ELFFile(io.BytesIO(obj)).get_section_by_name('.symtab').iter_symbols()] + assert names.count('demo') == 1 + + +def test_short_branch_masks_only_its_one_byte_operand(): + code = bytes.fromhex('4885ff7405c390909090c3') + fn = scratch.analyse(image(code), [{'entry': 0x1000, 'end': 0x1005}])[0] + assert fn.relocs == [(4, 15, 'D_0000100A', -1)] + obj = object_for(code[:5], fn.relocs) + assert ELFFile(io.BytesIO(obj)).get_section_by_name('.text').data() == bytes.fromhex('4885ff7400') + + +def test_internal_short_branch_is_not_relocated_when_true_end_is_known(): + code = bytes.fromhex('4885ff7405c390909090c3') + fn = scratch.analyse(image(code), [{'entry': 0x1000, 'end': 0x100b}])[0] + assert fn.relocs == [] + + +def test_rip_displacement_addend_accounts_for_trailing_immediate(): + code = bytes.fromhex('c7050100000007000000c3') + b'\0' * 16 + fn = scratch.analyse(image(code), [{'entry': 0x1000, 'end': 0x100b}])[0] + assert fn.relocs == [(2, 2, 'D_0000100B', -8)] + + +def test_discontinuous_body_keeps_span_and_reads_later_block(): + code = bytes.fromhex('c3909090e801000000c3c3') + fn = scratch.analyse(image(code), [{'entry': 0x1000, 'end': 0x100a, + 'ranges': [[0x1000, 0x1001], [0x1004, 0x100a]]}])[0] + assert fn.size == 10 + assert fn.relocs == [(5, 2, 'D_0000100A', -4)] + + + +@pytest.mark.parametrize('reloc', [(-1, 2, 'external', -4), (3, 2, 'external', -4), (1, 99, 'external', 0)]) +def test_invalid_relocation_bounds_fail_instead_of_corrupting_neighbors(reloc): + with pytest.raises(ValueError, match='invalid relocation'): + object_for(bytes.fromhex('e800000000c3'), [reloc], size=5) + + +def test_legacy_address_count_is_not_used_as_contiguous_size(tmp_path): + (tmp_path / 'functions.jsonl').write_text(json.dumps({'a': '0', 's': 5}) + '\n') + assert scratch.map_functions(tmp_path / 'game.elf', image(b'\xc3'*20, 0).text) is None + + +def test_explicit_end_wins_over_body_byte_count(tmp_path): + (tmp_path / 'functions.jsonl').write_text(json.dumps({'a': '0', 's': 5, 'end': 'b', + 'ranges': [[0, 1], [4, 11]]}) + '\n') + row = scratch.map_functions(tmp_path / 'game.elf', image(b'\xc3'*20, 0).text)[0] + assert row['end'] == 11 + assert row['ranges'] == [[0, 1], [4, 11]] + + +def test_generated_project_requires_relocation_identity(tmp_path): + root = tmp_path / 'project' + shutil.copytree(scratch.TEMPLATE, root) + (root / 'config/eboot').mkdir(parents=True) + (root / 'config/eboot/functions.json').write_text(json.dumps({'units': {}})) + subprocess.run([sys.executable, str(root / 'tools/configure.py'), 'eboot', '--clang', 'unused-clang'], + check=True, capture_output=True) + cfg = json.loads((root / 'objdiff.json').read_text()) + assert cfg['options']['functionRelocDiffs'] == 'all' + + +def test_objdiff_rejects_wrong_call_and_accepts_same_call_with_strict_options(tmp_path): + if toolchains.installed('objdiff-cli') is None: + pytest.skip('pinned objdiff is not installed here') + exe = toolchains.binary('objdiff-cli') + code = bytes.fromhex('e800000000c3') + target, base = tmp_path / 'target.o', tmp_path / 'base.o' + target.write_bytes(object_for(code, [(1, 2, 'correct', -4)])) + for callee, matches in [('wrong', False), ('correct', True)]: + base.write_bytes(object_for(code, [(1, 2, callee, -4)])) + result = subprocess.run([str(exe), 'diff', '-1', str(target), '-2', str(base), '-o', '-', + '-c', 'functionRelocDiffs=all', 'demo'], capture_output=True, text=True, check=True) + symbols = json.loads(result.stdout)['left']['symbols'] + percent = next(s['match_percent'] for s in symbols if s['name'] == 'demo') + assert (percent == 100) == matches + + +def test_shared_tail_in_another_function_does_not_swallow_intervening_functions(): + code = bytes.fromhex('eb02c390c3') + rows = [{'entry': 0x1000, 'end': 0x1005, 'ranges': [[0x1000, 0x1002], [0x1004, 0x1005]]}, + {'entry': 0x1002, 'end': 0x1003, 'ranges': [[0x1002, 0x1003]]}, + {'entry': 0x1004, 'end': 0x1005, 'ranges': [[0x1004, 0x1005]]}] + fn = scratch.analyse(image(code), rows)[0] + assert fn.size == 2 + assert fn.relocs == [(1, 15, 'func_00001004', -1)] + + +def test_shared_tail_before_entry_is_external_not_a_negative_relocation(): + code = bytes.fromhex('c39090ebfb') + rows = [{'entry': 0x1000, 'end': 0x1001, 'ranges': [[0x1000, 0x1001]]}, + {'entry': 0x1003, 'end': 0x1005, 'ranges': [[0x1000, 0x1001], [0x1003, 0x1005]]}] + fn = scratch.analyse(image(code), rows)[1] + assert fn.size == 2 + assert fn.relocs == [(1, 15, 'func_00001000', -1)] + + +def test_typed_linkage_declaration_is_allowed_but_assembly_is_not(): + from decompnet.app.forkengine import _CHEAT, _strip_comments + + declaration = 'extern __typeof__(old) recovered __asm__("correct") __attribute__((visibility("hidden")));' + assert not _CHEAT.search(_strip_comments(declaration)) + for text in ['__asm__("correct");', 'asm volatile("nop");', + declaration.replace('"correct"', '"correct\\n nop"'), + declaration + ' __asm__("nop");']: + assert _CHEAT.search(_strip_comments(text)) + + +def test_rip_reference_to_bss_is_relocated_without_file_backing(): + code = bytes.fromhex('488d05f90f0000c3') + elf = image(code) + elf.segments.append(scratch.Segment(0x2000, len(code), 0, 0x100, 6)) + fn = scratch.analyse(elf, [{'entry': 0x1000, 'end': 0x1008}])[0] + assert fn.relocs == [(3, 2, 'D_00002000', -4)] + assert elf.offset(0x2000) is None + assert elf.read(0x2000, 8) == b'' + assert not elf.contains(0x2000) + + +def test_rip_reference_past_mapped_memory_stays_literal(): + code = bytes.fromhex('488d05f9100000c3') + elf = image(code) + elf.segments.append(scratch.Segment(0x2000, len(code), 0, 0x100, 6)) + fn = scratch.analyse(elf, [{'entry': 0x1000, 'end': 0x1008}])[0] + assert fn.relocs == [] + + +def test_bss_reference_with_trailing_immediate_keeps_pc_addend(): + code = bytes.fromhex('c705f60f000007000000c3') + elf = image(code) + elf.segments.append(scratch.Segment(0x2000, len(code), 0, 0x100, 6)) + fn = scratch.analyse(elf, [{'entry': 0x1000, 'end': 0x100b}])[0] + assert fn.relocs == [(2, 2, 'D_00002000', -8)] + + +def make_elf(code=b'\xc3', *, phentsize=56, filesz=None, memsz=None, kind=1): + """A synthetic header-only Orbis ELF; no bytes come from a game.""" + import struct + + header = bytearray(128) + header[:7] = b'\x7fELF\x02\x01\x01' + struct.pack_into('