From d1eb99279089f01cfd06c146aa18106155aae4f6 Mon Sep 17 00:00:00 2001 From: nolffan Date: Sat, 24 Oct 2026 00:06:08 +0000 Subject: [PATCH 01/19] PS2: read the PS2 release's archive and convert its worlds to PC worlds The PlayStation 2 release of No One Lives Forever has levels the PC game never had (the three "Nine years ago" flashbacks; a secret basement in "A Man of Influence"). These tools, in build/ps2/, are the start of adding them to the modern build from the player's own PS2 disc; nothing from the disc is committed. - ps2rez.py reads LITH_PS2.REZ: its directory tables and the hash its file names are looked up by. - ps2world.py reads a PS2 world (.ltb, version 66): the PC's BSPs with the PS2's own polygon, lightmap-page and texture data, its static lights and objects. spec-ps2-world.md is the format. - ps2dtx.py turns the PS2's 8-bit palettised textures (BPP_32P) into PC 32-bit textures with mipmaps. - pcworld.py reads and checks a PC world (.dat, version 66), every byte accounted for; spec-pc-world.md is the format, from our engine's loader. - ps2topc.py converts a PS2 world into a PC world: one surface per polygon, texture mapping solved from the PS2 texture coordinates and the PC texture's size (the PC's own, higher-resolution texture wherever one has the name), the PS2's baked lightmaps resampled into PC lightmaps, the lights as Light objects. Co-Authored-By: Claude Opus 5.5 --- build/ps2/pcworld.py | 431 +++++++++++++++++++ build/ps2/ps2dtx.py | 160 ++++++++ build/ps2/ps2rez.py | 108 +++++ build/ps2/ps2topc.py | 587 ++++++++++++++++++++++++++ build/ps2/ps2world.py | 584 ++++++++++++++++++++++++++ build/ps2/spec-pc-world.md | 799 ++++++++++++++++++++++++++++++++++++ build/ps2/spec-ps2-world.md | 234 +++++++++++ 7 files changed, 2903 insertions(+) create mode 100644 build/ps2/pcworld.py create mode 100644 build/ps2/ps2dtx.py create mode 100644 build/ps2/ps2rez.py create mode 100644 build/ps2/ps2topc.py create mode 100644 build/ps2/ps2world.py create mode 100644 build/ps2/spec-pc-world.md create mode 100644 build/ps2/spec-ps2-world.md diff --git a/build/ps2/pcworld.py b/build/ps2/pcworld.py new file mode 100644 index 00000000..da2d2963 --- /dev/null +++ b/build/ps2/pcworld.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Walk a PC NOLF (LithTech 2.x, version 66) world .dat section by section. + +Follows build/ps2/spec-pc-world.md, which follows the engine +reconstruction: + header ............ commonmodel.cpp w_ReadWorldHeader + info/extents/tree . commonmodel.cpp CServerWorldFull::Load, worldtree.cpp LoadLayout + world models ...... worldbsp.cpp w_LoadWorldModel, CWorldSimple::Load, worldmodels.cpp CSubGrid::Load + object list ....... servermodel.cpp sm_LoadWorldObjects, commonmodel.cpp w_ReadWorldLights + lightmap anims .... worldmodels.cpp CWorldData::LoadObjects + +Usage: pcworld.py [-v] world.dat [world.dat ...] +Every byte is accounted for: the script reports gaps/overlaps between sections +and whether the last section ends at EOF. +""" +import struct +import sys +from collections import Counter + + +class R: + def __init__(self, data, pos=0): + self.d = data + self.p = pos + + def u8(self): + v = self.d[self.p]; self.p += 1; return v + + def u16(self): + v = struct.unpack_from('= nTex: + texidx_bad += 1 + fl = r.u32(); sflags[fl] += 1 + b.setdefault('s44', Counter())[r.u8()] += 1 # +0x44 + b.setdefault('sRGB', Counter())[(r.u8(), r.u8(), r.u8())] += 1 # ignored colour + has = r.u8() + if has == 1: + effects += 1 + r.lstr(); r.lstr() # effect class, params + b.setdefault('texFlags', Counter())[r.u16()] += 1 # texture flags (surface type) + b['surfaceFlags'] = sflags; b['surfEffects'] = effects; b['surfBadTex'] = texidx_bad + + # polies + edgeTotal = 0 + for i in range(nPolies): + r.vec() # lightmap origin (+0x34) + lw = r.u16(); lh = r.u16() # lightmap w/h (stored as bytes) + b.setdefault('lmSizes', Counter())[(lw, lh)] += 1 + ne = r.u16(); edgeTotal += ne + for _ in range(ne): + a = r.u16(); k = r.u16() + b.setdefault('lmrefs', []).append((i, a, k)) + surf = r.u32(); assert surf < nSurfaces, 'poly surface out of range' + nv = vcounts[i][0] + vcounts[i][1] + for _ in range(nv): + pt = r.u16(); assert pt < nPoints, 'poly point out of range' + r.u8(); r.u8(); r.u8() + assert edgeTotal <= nEdges, (edgeTotal, nEdges) + b['edgesUsed'] = edgeTotal + + # nodes + term = Counter() + nodes = [] + for i in range(nNodes): + p = r.u32(); assert p < nPolies, 'node poly out of range' + leaf = r.u16() # leaf index (0xffff none) + assert leaf == 0xffff or leaf < nLeafs, 'node leaf out of range' + cs = [] + for _ in range(2): + c = r.i32() + if c < 0: + assert c in (-1, -2); term[c] += 1 + else: + assert c < nNodes + cs.append(c) + nodes.append((p, leaf, cs[0], cs[1])) + b['nodes'] = nodes + b['nodesWithLeaf'] = sum(1 for n in nodes if n[1] != 0xffff) + b['nodeTerminals'] = dict(term) + + # user portals + for i in range(nPortals): + r.lstr(); r.u32(); r.u16(); r.vec(); r.vec() + + # points (position + an ignored normal) + for i in range(nPoints): + r.vec(); b.setdefault('ptNormal', Counter())[r.vec()] += 1 + + # PBlock table (CSubGrid::Load) + cx, cy, cz = r.i32(), r.i32(), r.i32() + csize = r.vec(); corg = r.vec() + ncells = cx * cy * cz + nodes6 = 0; empty = 0 + cells = [] + for i in range(ncells): + cnt = r.u16(); root = r.u16() + recs = [struct.unpack_from('<3H', r.raw(6)) for _ in range(cnt)] + nodes6 += cnt + if cnt == 0: + empty += 1 + # WSNode {u16 bspNode, u16 back, u16 front}; children index this cell's run + # or are 0xfffe (empty) / 0xffff (solid); root likewise + assert root in (0xfffe, 0xffff) or root < cnt, ('pblock root', root, cnt) + for (bn, c0, c1) in recs: + assert bn < nNodes, 'pblock node -> bsp node out of range' + for c in (c0, c1): + assert c in (0xfffe, 0xffff) or c < cnt + cells.append((root, recs)) + b['cells'] = cells + b['pblock'] = dict(dims=(cx, cy, cz), cell=csize, origin=corg, + cells=ncells, nodes=nodes6, emptyCells=empty) + + root = r.i32() + assert root in (-1, -2) or 0 <= root < nNodes + b['root'] = root + + nTerrain = r.u32() + b['nTerrain'] = nTerrain + for t in range(nTerrain): + nn = r.u32() + for _ in range(nn): + r.u32(); r.i32(); r.i32() + r.i32() + np_ = r.u32() + for _ in range(np_): + r.u32() + cx, cy, cz = r.i32(), r.i32(), r.i32(); r.vec(); r.vec() + for i in range(cx * cy * cz): + cnt = r.u16(); r.u16(); r.raw(cnt * 6) + r.raw(8) # wb_ReadSimpleHeader: 4 bytes + 1 dword + r.vec() # centre + b['size'] = r.p - start + return b + + +def parse_world(path, verbose=False, data=None): + d = data if data is not None else open(path, 'rb').read() + r = R(d) + sec = [] # (name, start, end) + out = {'file': path, 'size': len(d)} + + s = r.p + version = r.u32(); assert version == 66, version + objOfs = r.u32(); lmOfs = r.u32() + hdr_unused = [r.u32() for _ in range(8)] + sec.append(('header', s, r.p)) + out.update(version=version, objOfs=objOfs, lmOfs=lmOfs, hdrUnused=hdr_unused) + + s = r.p + n = r.u32(); info = r.raw(n).decode('latin-1') + lmgrid = r.f32(); emin = r.vec(); emax = r.vec() + sec.append(('worldinfo+lmgrid+extents', s, r.p)) + out.update(info=info, lmGrid=lmgrid, extMin=emin, extMax=emax) + + s = r.p + tmin = r.vec(); tmax = r.vec(); tUnused = r.u32(); tDepth = r.u32() + # quadtree bit stream: preorder, 1 bit per node, LSB first, bytes read lazily + cur = [0, 8] + count = [0, 0] + + def node(): + if cur[1] == 8: + cur[0] = r.u8(); cur[1] = 0 + bit = (cur[0] >> cur[1]) & 1 + cur[1] += 1 + count[0] += 1 + if bit: + count[1] += 1 + for _ in range(4): + node() + node() + sec.append(('worldtree', s, r.p)) + out.update(treeMin=tmin, treeMax=tmax, treeUnused=tUnused, treeDepthDword=tDepth, + treeNodes=count[0], treeInner=count[1]) + + s = r.p + nWorlds = r.u32() + sec.append(('nWorlds', s, r.p)) + worlds = [] + for i in range(nWorlds): + s = r.p + nxt = r.u32() + b = parse_bsp(r, verbose) + b['next'] = nxt + b['slack'] = nxt - r.p # bytes between record end and next offset + worlds.append(b) + sec.append(('world[%d] %s' % (i, b['name']), s, r.p)) + if nxt != r.p: + sec.append(('world[%d] SLACK' % i, r.p, nxt)) + r.p = nxt + out['worlds'] = worlds + + # object list + assert r.p == objOfs, ('object list not right after last world', r.p, objOfs) + s = r.p + nObj = r.u32() + classes = Counter(); ptypes = Counter(); propcount = 0 + objs = [] + for i in range(nObj): + rs = r.u16(); p0 = r.p + cls = r.lstr(); classes[cls] += 1 + nprops = r.u32() + props = {} + for j in range(nprops): + pn = r.lstr(); pt = r.u8(); pf = r.u32(); pl = r.u16() + ptypes[pt] += 1; propcount += 1 + if pt == 0: + q = r.p + sv = r.lstr() + assert r.p - q == pl, ('string prop length', pn, pl, r.p - q) + props[pn] = sv + else: + props[pn] = r.raw(pl) + assert r.p - p0 == rs, ('record size mismatch', cls, rs, r.p - p0) + objs.append((cls, props)) + sec.append(('objects', s, r.p)) + out.update(nObjects=nObj, classes=classes, propTypes=ptypes, nProps=propcount) + out['objs'] = objs + + if r.p != lmOfs: + sec.append(('GAP before lightmaps', r.p, lmOfs)) + r.p = lmOfs + s = r.p + nRefs, nFramePtrs, nData, nEntries, nAnims = [r.u32() for _ in range(5)] + anims = [] + tRefs = tFrames = tData = tEnt = 0 + for a in range(nAnims): + name = r.lstr() + shadow = r.u32(); nfr = r.u32(); npol = r.u32() + refs = [(r.u16(), r.u16()) for _ in range(npol)] + out.setdefault('animrefs', []).append(refs) + dbytes = 0 + for f in range(nfr): + for p in range(npol): + sz = r.u32(); r.raw(sz); dbytes += sz + tRefs += npol; tFrames += nfr; tEnt += nfr * npol; tData += dbytes + anims.append((name, shadow, nfr, npol, dbytes)) + assert tRefs <= nRefs and tFrames <= nFramePtrs and tData <= nData and tEnt <= nEntries + sec.append(('lightanims', s, r.p)) + out.update(lmTotals=(nRefs, nFramePtrs, nData, nEntries, nAnims), + lmUsed=(tRefs, tFrames, tData, tEnt), anims=anims) + # cross-check: every poly's light-anim back-reference {iAnim, iSlot} must point + # at the anim poly-ref {world, poly} that names that very poly + bad = 0; n = 0 + for wi, b in enumerate(worlds): + for (pi, a, k) in b.get('lmrefs', []): + n += 1 + refs = out['animrefs'][a] if a < len(out.get('animrefs', [])) else None + if not refs or k >= len(refs) or refs[k] != (wi, pi): + bad += 1 + out['lmBackrefs'] = (n, bad) + out['end'] = r.p + out['sections'] = sec + return out + + +def report(o, verbose=False): + print('=' * 72) + print('%s: %d bytes, version %d, objOfs %d, lmOfs %d, unused hdr %s' + % (o['file'], o['size'], o['version'], o['objOfs'], o['lmOfs'], o['hdrUnused'])) + print('info string: %r' % o['info'][:200]) + print('lmGrid %g extents %s .. %s' % (o['lmGrid'], o['extMin'], o['extMax'])) + print('worldtree box %s..%s dwords (%d, %d), %d nodes (%d with children)' + % (o['treeMin'], o['treeMax'], o['treeUnused'], o['treeDepthDword'], + o['treeNodes'], o['treeInner'])) + print('%d world models' % len(o['worlds'])) + tot = Counter() + for i, b in enumerate(o['worlds']): + for k in ('nPoints', 'nPlanes', 'nSurfaces', 'nPortals', 'nPolies', 'nLeafs', + 'nLeafLists', 'visBytes', 'nNodes', 'nLeafPolyRefs', 'nEdges'): + tot[k] += b[k] + tot['nTextures'] += len(b['textures']) + tot['pblockCells'] += b['pblock']['cells']; tot['pblockNodes'] += b['pblock']['nodes'] + if verbose or i == 0: + print(' [%d] %-28s flags %#04x dc %#x | pts %d planes %d surfs %d portals %d polys %d ' + 'leafs %d unk %d vis %d/%d leaflists %d/%d nodes %d leafrefs %d/%d edges %d/%d ' + 'tex %d (pool %d, pad %d) eff %d badTex %d fix %d maxv %d root %d terr %d slack %d' + % (i, b['name'], b['flags'], b['dc'], b['nPoints'], b['nPlanes'], b['nSurfaces'], + b['nPortals'], b['nPolies'], b['nLeafs'], b['nUnk'], b['visUsed'], b['visBytes'], + b['leafListsUsed'], b['nLeafLists'], b['nNodes'], b['leafRefsUsed'], + b['nLeafPolyRefs'], b['edgesUsed'], b['nEdges'], len(b['textures']), + b['texStrBytes'], b['texPad'], b['surfEffects'], b['surfBadTex'], + b['fixverts_polys'], b['maxverts'], b['root'], b['nTerrain'], b['slack'])) + print(' junk8 %s' % b['junk8']) + print(' pblock %s' % b['pblock']) + print(' nodeTerminals %s' % b['nodeTerminals']) + print(' surface flags %s' % ', '.join('%#x:%d' % kv for kv in sorted(b['surfaceFlags'].items()))) + flagc = Counter(b['flags'] for b in o['worlds']) + print(' world-model flags histogram: %s' % ', '.join('%#x:%d' % kv for kv in sorted(flagc.items()))) + print(' totals: %s' % dict(tot)) + print('objects: %d, props %d, prop types %s' % (o['nObjects'], o['nProps'], dict(o['propTypes']))) + print(' top classes: %s' % o['classes'].most_common(12)) + print('lightanims: header totals %s used %s; poly back-refs %d (%d inconsistent)' + % (o['lmTotals'], o['lmUsed'], o['lmBackrefs'][0], o['lmBackrefs'][1])) + for a in o['anims'][:5]: + print(' %s' % (a,)) + if len(o['anims']) > 5: + print(' ... %d more' % (len(o['anims']) - 5)) + print('sections:') + last = 0 + for name, s, e in o['sections']: + flag = '' if s == last else ' <-- %+d from previous end' % (s - last) + if verbose or 'world[' not in name or name.startswith('world[0]') or 'SLACK' in name: + print(' %10d %10d %10d %s%s' % (s, e, e - s, name, flag)) + last = e + print('end of parse %d, file size %d -> %s' % (o['end'], o['size'], + 'EXACT' if o['end'] == o['size'] else 'MISMATCH (%+d)' % (o['size'] - o['end']))) + + +def walk_rez(rezpath, pattern='WORLDS/'): + """Parse every world .dat inside a .rez (tools/rezextract.py), one summary line each.""" + import os + sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..', '..', 'tools')) + from rezextract import Rez + rz = Rez(rezpath) + f = open(rezpath, 'rb') + for p, pos, sz in rz.walk(): + if not (p.upper().startswith(pattern.upper()) and p.upper().endswith('.DAT')): + continue + f.seek(pos); data = f.read(sz) + try: + o = parse_world(p, data=data) + print('%-50s %9d %s worlds %4d objs %5d anims %3d backrefs-bad %d' % ( + p, sz, 'EXACT' if o['end'] == sz else 'MISMATCH%+d' % (sz - o['end']), + len(o['worlds']), o['nObjects'], len(o['anims']), o['lmBackrefs'][1])) + except Exception as e: + print('%-50s %9d FAILED %r' % (p, sz, e)) + + +if __name__ == '__main__': + args = sys.argv[1:] + v = '-v' in args + if args and args[0] == '--rez': + walk_rez(args[1]) + sys.exit(0) + for a in [x for x in args if x != '-v']: + report(parse_world(a, v), v) diff --git a/build/ps2/ps2dtx.py b/build/ps2/ps2dtx.py new file mode 100644 index 00000000..ce9704fa --- /dev/null +++ b/build/ps2/ps2dtx.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""PS2 NOLF textures (DTX v-5, BPP_32P: 8-bit indices + a "PALLETE32" section +of 256 32-bit entries) -> PC DTX v-5, BPP_32 (BGRA), with mipmaps. + + ps2dtx.py + +Also decodes PC DXT1/3/5 and BPP_32 DTX (for comparisons). +""" +import struct, sys + +HDR = 0xa4 + + +def clut_index(i): + """PS2 CSM1 CLUT storage order: bits 3 and 4 of the index swapped (the + textures are not stored so; measured against PC copies, which also say BGRA).""" + return (i & ~0x18) | ((i & 0x08) << 1) | ((i & 0x10) >> 1) + + +def ps2_read(data, swizzle=False, order='bgra'): + """-> (w, h, [list of mips as bytes RGBA], header bytes).""" + w, h, nm, ns = struct.unpack_from('<4H', data, 8) + bpp = data[0x1a] + if struct.unpack_from('> i), max(1, h >> i) + mips.append((mw, mh, data[p:p + mw * mh])) + p += mw * mh + pal = None + for _ in range(ns): + name = data[p:p + 15].split(b'\0')[0] + ln = struct.unpack_from(' 1 or mips[-1][1] > 1) and len(mips) < 8: + mips.append(downsample(*mips[-1])) + hdr = bytearray(hdr_in[:HDR]) + struct.pack_into('<4H', hdr, 8, w, h, len(mips), 0) + fl = struct.unpack_from(' BGRA + out += b + return bytes(out) + + +def ps2_to_pc(data, swizzle=False, order='bgra'): + if data[0x1a] == 3: + # Already 32-bit, stored BGRA as on the PC (the PS2 reverses it at + # load). Only the alpha needs the PS2 meaning: none at all = opaque, + # 0x80 = opaque. + w, h = struct.unpack_from('> 11) & 31, (c >> 5) & 63, c & 31 + return (r * 255 // 31, g * 255 // 63, b * 255 // 31) + + +def pc_read_top(data): + """Top mip of a PC DTX as (w, h, RGBA bytes). DXT1/3/5 and BPP_32.""" + w, h = struct.unpack_from(' c1 or bpp != 4: + cols = [a, b, tuple((2 * x + y) // 3 for x, y in zip(a, b)), tuple((x + 2 * y) // 3 for x, y in zip(a, b))] + else: + cols = [a, b, tuple((x + y) // 2 for x, y in zip(a, b)), (0, 0, 0)] + for i in range(16): + x, y = bx * 4 + (i & 3), by * 4 + (i >> 2) + if x < w and y < h: + c = cols[(bits >> (2 * i)) & 3] + o = 4 * (y * w + x) + out[o:o + 4] = bytes((c[0], c[1], c[2], 255)) + return w, h, bytes(out) + + +if __name__ == '__main__': + src, dst = sys.argv[1], sys.argv[2] + open(dst, 'wb').write(ps2_to_pc(open(src, 'rb').read())) diff --git a/build/ps2/ps2rez.py b/build/ps2/ps2rez.py new file mode 100644 index 00000000..476c301f --- /dev/null +++ b/build/ps2/ps2rez.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Read-only PS2 LITH_PS2.REZ reader, a Python port of the logic in +StenApp/ps2rezdecoder (Rez.cpp / ps2rezdecoder.cpp). + +Layout (little endian): + 0x000 u32 unknown(=3), u32 entries_per_table, u32 hash_magic, + u32 data_offset (2048-byte sectors), u32 table_count + 0x800 + t*0x19000: table t, entries_per_table x 16-byte entries + {u32 name_hash, u32 sector, u32 size, u32 size_in_sectors} + hash: h = 0; for c in upper(name) ('/'->'\\'): h = h + c + h*magic (u32) + +Usage: + ps2rez.py info REZ + ps2rez.py tables REZ per-table summary + ps2rez.py list REZ NAMES.txt OUT full listing (all tables) + ps2rez.py x REZ NAMES.txt OUTDIR [regex] [table] extract named files +""" +import os, re, struct, sys +from collections import defaultdict + +TABLE_START, TABLE_STRIDE, ENTRY = 0x800, 0x19000, 16 + + +def rez_hash(name, magic): + h = 0 + for ch in name: + c = ord(ch.upper()) + if c == ord('/'): + c = ord('\\') + h = (h + c + h * magic) & 0xFFFFFFFF + return h + + +def open_rez(path): + f = open(path, 'rb') + unk, n, magic, data_off, ntab = struct.unpack('<5I', f.read(20)) + tables = [] + for t in range(ntab): + f.seek(TABLE_START + t * TABLE_STRIDE) + raw = f.read(n * ENTRY) + tables.append([struct.unpack_from('<4I', raw, i * ENTRY) for i in range(n)]) + return f, dict(unk=unk, n=n, magic=magic, data_off=data_off, ntab=ntab), tables + + +def load_names(path, magic): + names = {} + with open(path, encoding='latin-1') as fh: + for line in fh: + s = line.strip() + if not s or s.startswith('#'): + continue + s = s.split()[0] + names.setdefault(rez_hash(s, magic), s) + return names + + +def main(): + cmd, rez = sys.argv[1], sys.argv[2] + f, hdr, tables = open_rez(rez) + if cmd == 'info': + print(hdr) + return + if cmd == 'tables': + for t, tab in enumerate(tables): + live = [e for e in tab if e[2]] + hashes = len(set(e[0] for e in tab)) + secs = [e[1] for e in live] + print(t, 'live', len(live), 'uniqhash', hashes, + 'sectors', (min(secs), max(secs)) if secs else None, + 'bytes', sum(e[2] for e in live)) + return + names = load_names(sys.argv[3], hdr['magic']) + if cmd == 'list': + with open(sys.argv[4], 'w') as out: + out.write('# header %r\n# table\tidx\thash\tsector\tsize\tsectors\tname\n' % hdr) + for t, tab in enumerate(tables): + for i, (h, sec, size, xs) in enumerate(tab): + if size == 0 and sec == 0 and h == 0: + continue + out.write('%d\t%d\t0x%08X\t%d\t%d\t%d\t%s\n' % ( + t, i, h, sec, size, xs, names.get(h, '?'))) + return + if cmd == 'x': + outdir = sys.argv[4] + rx = re.compile(sys.argv[5], re.I) if len(sys.argv) > 5 else None + want_t = int(sys.argv[6]) if len(sys.argv) > 6 else None + done = set() + for t, tab in enumerate(tables): + if want_t is not None and t != want_t: + continue + for h, sec, size, xs in tab: + nm = names.get(h) + if not nm or size == 0 or h in done: + continue + if rx and not rx.search(nm): + continue + done.add(h) + dst = os.path.join(outdir, *nm.replace('\\', '/').split('/')) + os.makedirs(os.path.dirname(dst), exist_ok=True) + f.seek(sec * 2048) + with open(dst, 'wb') as o: + o.write(f.read(size)) + print('%s\t%d\ttable %d' % (nm, size, t)) + return + + +if __name__ == '__main__': + main() diff --git a/build/ps2/ps2topc.py b/build/ps2/ps2topc.py new file mode 100644 index 00000000..0fe86ef7 --- /dev/null +++ b/build/ps2/ps2topc.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +"""Convert a PS2 NOLF world (worlds/*.ltb, v66) into a PC NOLF world (.dat, v66). + + ps2topc.py [--pc-rez DIR] [--ps2-rez LITH_PS2.REZ] + +First version: geometry, BSPs, collision grids, leaves/PVS, portals, terrain +and objects are carried over; each polygon gets its own PC surface whose +O/P/Q reproduce its PS2 per-vertex UVs; there are no PC lightmaps -- the +PS2's baked lighting (lightmap pages, vertex colours) becomes the PC vertex +colours. Layouts: spec-pc-world.md, spec-ps2-world.md. +""" +import math, os, struct, sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import ps2world as P # noqa: E402 + +SURF_INVISIBLE, SURF_TRANSLUCENT, SURF_LIGHTMAP = 0x4, 0x8, 0x80 + +# ------------------------------------------------------------------ texture sizes + +class TexSizes: + """Width/height of a texture by name: the PC archives first (that is what the + engine will load), then the PS2 archive, else 64x64.""" + def __init__(self, pc_dir, ps2_rez): + self.pc = {} + self.cache = {} + if pc_dir: + d = HERE + while d != os.path.dirname(d) and not os.path.isfile(os.path.join(d, 'tools', 'rezextract.py')): + d = os.path.dirname(d) + sys.path.insert(0, os.path.join(d, 'tools')) + from rezextract import Rez + for n in sorted(os.listdir(pc_dir)): + if n.lower().endswith('.rez'): + path = os.path.join(pc_dir, n) + for p, pos, sz in Rez(path).walk(): + self.pc.setdefault(p.replace('/', '\\').lower(), (path, pos)) + self.ps2 = None + if ps2_rez: + import ps2rez + f, hdr, tables = ps2rez.open_rez(ps2_rez) + self.ps2 = (ps2rez, f, hdr, {e[0]: e for e in tables[0] if e[2]}) + + def head(self, name): + key = name.replace('/', '\\').lower() + if key in self.pc: + path, pos = self.pc[key] + with open(path, 'rb') as fh: + fh.seek(pos) + return fh.read(16), 'pc' + if self.ps2: + mod, f, hdr, t0 = self.ps2 + e = t0.get(mod.rez_hash(name, hdr['magic'])) + if e: + f.seek(e[1] * 2048) + return f.read(16), 'ps2' + return None, None + + def ps2_bytes(self, name): + if not self.ps2: + return None + mod, f, hdr, t0 = self.ps2 + e = t0.get(mod.rez_hash(name, hdr['magic'])) + if not e: + return None + f.seek(e[1] * 2048) + return f.read(e[2]) + + def size(self, name): + if name not in self.cache: + h, where = self.head(name) + if h and len(h) >= 12 and struct.unpack_from(' best: + best, tri = ar, (i, j, k) + if n > 12: + break + if tri is None or best < 1e-8: + return (0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 0.0, 1.0), 0.0 + i, j, k = tri + e1, e2 = sub(pts[j], pts[i]), sub(pts[k], pts[i]) + nn = cross(e1, e2) + Pv = solve3(e1, e2, nn, (uvs[j][0] - uvs[i][0], uvs[k][0] - uvs[i][0], 0.0)) + Qv = solve3(e1, e2, nn, (uvs[j][1] - uvs[i][1], uvs[k][1] - uvs[i][1], 0.0)) + if Pv is None or Qv is None: + return (0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 0.0, 1.0), 0.0 + X = solve3(Pv, Qv, nn, (uvs[i][0], uvs[i][1], 0.0)) + if X is None: # P and Q parallel: a degenerate mapping + X = (0.0, 0.0, 0.0) + O = sub(pts[i], X) + err = 0.0 + for p, (u, v) in zip(pts, uvs): + d = sub(p, O) + err = max(err, abs(dot(d, Pv) - u), abs(dot(d, Qv) - v)) + return O, Pv, Qv, err + +# ------------------------------------------------------------------ lighting + +class Lightmaps: + """PS2 lightmap pages: colour = L - d per channel (spec-ps2-world.md §8).""" + def __init__(self, pages): + self.pages = [] + for pg in pages: + pal = [] + for i in range(256): + e = pg.clut[4 * P.clut_index(i):4 * P.clut_index(i) + 4] + dr, dg, db, L = e + pal.append((max(0, L - dr), max(0, L - dg), max(0, L - db))) + self.pages.append([pal[i] for i in pg.pixels]) + + def sample_px(self, page, x, y): + """Bilinear, at page pixel coordinates (pixel centres on integers).""" + px = self.pages[page] + x = min(63.0, max(0.0, x)) + y = min(63.0, max(0.0, y)) + x0, y0 = int(x), int(y) + x1, y1 = min(63, x0 + 1), min(63, y0 + 1) + fx, fy = x - x0, y - y0 + out = [] + for c in range(3): + a = px[y0 * 64 + x0][c] * (1 - fx) + px[y0 * 64 + x1][c] * fx + b = px[y1 * 64 + x0][c] * (1 - fx) + px[y1 * 64 + x1][c] * fx + out.append(a * (1 - fy) + b * fy) + return out + + def sample(self, page, u, v): + px = self.pages[page] + x = min(63.0, max(0.0, u * 64.0 - 0.5)) + y = min(63.0, max(0.0, v * 64.0 - 0.5)) + x0, y0 = int(x), int(y) + x1, y1 = min(63, x0 + 1), min(63, y0 + 1) + fx, fy = x - x0, y - y0 + out = [] + for c in range(3): + a = px[y0 * 64 + x0][c] * (1 - fx) + px[y0 * 64 + x1][c] * fx + b = px[y1 * 64 + x0][c] * (1 - fx) + px[y1 * 64 + x1][c] * fx + out.append(a * (1 - fy) + b * fy) + return out + + +def to_pc_colour(rgb, scale): + return tuple(max(0, min(255, int(round(c * scale)))) for c in rgb) + + +# The engine's six lightmap plane types (math_FindClosestAxisBasis over +# g_AxisBases, svlast.cpp): the basis normal, and the "up" its lightmap +# vectors are built from (P = up x N, Q = N x P; mr_LMVectors). +AXIS_NORMALS = [(0, 1, 0), (0, -1, 0), (0, 0, 1), (0, 0, -1), (1, 0, 0), (-1, 0, 0)] +AXIS_UPS = [(0, 0, -1), (0, 0, 1), (0, 1, 0), (0, -1, 0), (0, -1, 0), (0, -1, 0)] +LM_MAX = 32 # texels a side (the renderers' 1024-texel buffers) + + +def lm_vectors(n): + best, t = -2.0, 0 + for i, a in enumerate(AXIS_NORMALS): + d = dot(n, a) + if d > best: + best, t = d, i + P = cross(AXIS_UPS[t], n) + Q = cross(n, P) + return P, Q + + +def lm_compress(texels): + """The engine's 16-bit RLE (mr_DecompressLM): a word per texel, or a word + with bit 15 set and a count byte for a run.""" + out = bytearray() + i, n = 0, len(texels) + while i < n: + j = i + while j < n and j - i < 255 and texels[j] == texels[i]: + j += 1 + if j - i > 1: + out += struct.pack(' LM_MAX or H > LM_MAX: + return None + origin = tuple(s0 * grid * P[i] / pp + t0 * grid * Q[i] / qq + dist * N[i] for i in range(3)) + # world -> PS2 page pixels, from the poly's own vertices + O2, P2, Q2, _ = poly_opq(pts, [(v.lm_u * 64.0, v.lm_v * 64.0) for v in p.verts], N) + x0, y0 = p.lm_x, p.lm_y + x1, y1 = p.lm_x + max(1, p.lm_w) - 1, p.lm_y + max(1, p.lm_h) - 1 + texels = [] + for j in range(H): + for i in range(W): + pos = tuple(origin[k] + i * grid * P[k] / pp + j * grid * Q[k] / qq for k in range(3)) + d = sub(pos, O2) + u = min(x1, max(x0, dot(d, P2) - 0.5)) + v = min(y1, max(y0, dot(d, Q2) - 0.5)) + rgb = to_pc_colour(lm.sample_px(p.lm_page, u, v), scale) + texels.append(((rgb[0] >> 3) << 10) | ((rgb[1] >> 3) << 5) | (rgb[2] >> 3)) + return origin, W, H, lm_compress(texels) + +# ------------------------------------------------------------------ writer + +class W: + def __init__(self): + self.b = bytearray() + + def pack(self, fmt, *a): + self.b += struct.pack('<' + fmt, *a) + + def u8(self, x): self.pack('B', x) + def u16(self, x): self.pack('H', x) + def u32(self, x): self.pack('I', x) + def i32(self, x): self.pack('i', x) + def f32(self, x): self.pack('f', x) + def vec(self, v): self.pack('3f', *v) + + def lstr(self, s): + e = s.encode('latin-1') + self.u16(len(e)) + self.b += e + + def pos(self): + return len(self.b) + + +def strip(s): + return s.rstrip('\0') if isinstance(s, str) else s + + +def write_pblock(w, pb): + sx, sy, sz = pb.size + w.pack('3i', sx, sy, sz) + w.vec(pb.block_size) + w.vec(pb.origin) + recs = pb.data + at = 0 + for _, n, root in pb.blocks: + w.u16(n) + w.u16(root if n else 0xFFFF) + w.b += recs[at * 6:(at + n) * 6] # {u16 node, u16 back, u16 front}: the PC encoding already + at += n + if at * 6 != len(recs): + raise ValueError('PBlock records not consumed: %d of %d' % (at, len(recs) // 6)) + + +def convert_bsp(w, m, world, tex, lm, stats, scale, world_index, anim): + is_world_model = not (m.info_flags & (0x10 | 0x20)) + lmaps = [make_lightmap(m, p, lm, world.lm_grid_size, scale) if anim is not None else None + for p in m.polys] + stats['lm_polys'] += sum(1 for x in lmaps if x) + stats['lm_too_big'] += sum(1 for p, x in zip(m.polys, lmaps) + if x is None and p.lm_page >= 0 and p.lm_w and p.verts and p.verts[0].lm_u is not None) + # textures used by this BSP, in first-use order + tex_index = {} + tex_names = [] + surf_of_poly = [] + for p in m.polys: + s = m.surfaces[p.surface] + t = strip(world.textures[s.texture]) + if t not in tex_index: + tex_index[t] = len(tex_names) + tex_names.append(t) + + w.pack('8I', *([0] * 8)) + w.u32(m.info_flags) + w.u32(0) + w.lstr(strip(m.name)) + n_total_verts = sum(len(p.verts) for p in m.polys) + vis_bytes = sum(len(d) for lf in m.leaves if lf.share is None for _, d in lf.lists) + n_lists = sum(len(lf.lists) for lf in m.leaves if lf.share is None) + n_refs = sum(len(lf.poly_refs) for lf in m.leaves) + w.pack('12I', len(m.points), len(m.planes), len(m.polys), len(m.portals), len(m.polys), + len(m.leaves), n_total_verts, vis_bytes, n_lists, len(m.nodes), n_refs, + sum(1 for x in lmaps if x)) + w.vec(m.box_min) + w.vec(m.box_max) + if is_world_model: + tr = tuple((a + b) * 0.5 for a, b in zip(m.box_min, m.box_max)) + else: + tr = m.translation + w.vec(tr) + + pool = b''.join(t.encode('latin-1') + b'\0' for t in tex_names) + w.u32(len(pool)) + w.u32(len(tex_names)) + w.b += pool + + for p in m.polys: + if len(p.verts) > 255: + raise ValueError('poly with %d verts' % len(p.verts)) + w.u8(len(p.verts)) + w.u8(0) + + for lf in m.leaves: + if lf.share is not None: + w.u16(0xFFFF) + w.u16(lf.share) + else: + w.u16(len(lf.lists)) + for portal, data in lf.lists: + w.pack('hH', portal, len(data)) + w.b += data + w.u32(len(lf.poly_refs)) + for a, b in lf.poly_refs: + w.pack('HH', a, b) + w.f32(lf.unk) + + for nx, ny, nz, d in m.planes: + w.pack('4f', nx, ny, nz, d) + + # one surface per poly + for p, lmap in zip(m.polys, lmaps): + s = m.surfaces[p.surface] + t = strip(world.textures[s.texture]) + flags = s.flags & ~SURF_LIGHTMAP & 0x1FFFFFFF + if lmap: + flags |= SURF_LIGHTMAP + pts = m.poly_xyz(p) + if p.verts and p.verts[0].u is not None: + tw, th, _ = tex.size(t) + uvs = [(v.u * tw, v.v * th) for v in p.verts] + O, Pv, Qv, err = poly_opq(pts, uvs, m.planes[p.plane][:3]) + stats['uv_err'] = max(stats['uv_err'], err if tw * th else 0) + if err > 0.5: + stats['uv_bad'] += 1 + else: + O, Pv, Qv = (0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 0.0, 1.0) + w.vec(O) + w.vec(Pv) + w.vec(Qv) + w.u16(tex_index[t]) + w.u32(p.plane) + w.u32(flags) + w.u8(0) + w.pack('3B', 0, 0, 0) + if s.effect: + w.u8(1) + w.lstr(strip(s.effect)) + w.lstr(strip(s.effect_param or '')) + else: + w.u8(0) + w.u16(s.texture_flags) + + for i, (p, lmap) in enumerate(zip(m.polys, lmaps)): + if lmap: + origin, lw, lh, data = lmap + w.vec(origin) + w.pack('HHH', lw, lh, 1) + w.pack('HH', 0, len(anim)) # LightAnim_BASE (anim 0), this poly's slot + anim.append((world_index, i, data)) + else: + w.vec((0.0, 0.0, 0.0)) + w.pack('HHH', 0, 0, 0) # lmWidth, lmHeight, nLMAnimRefs + w.u32(i) # its own surface + for v in p.verts: + if v.color is not None: + rgb = to_pc_colour(v.color[:3], scale) + elif v.lm_u is not None and p.lm_page >= 0: + rgb = to_pc_colour(lm.sample(p.lm_page, v.lm_u, v.lm_v), scale) + else: + rgb = (255, 255, 255) + w.pack('H3B', v.point, *rgb) + + for poly, leaf, back, front in m.nodes: + w.pack('IHii', poly, 0xFFFF if leaf < 0 else leaf, back, front) + + for pt in m.portals: + w.lstr(strip(pt.name)) + w.u32(pt.unk1) + w.u16(pt.flags) + w.vec(pt.center) + w.vec(pt.dims) + + for x, y, z in m.points: + w.pack('6f', x, y, z, 0.0, 0.0, 0.0) + + write_pblock(w, m.pblock) + w.i32(m.root_node) + w.u32(len(m.terrain_sections)) + for t in m.terrain_sections: + w.u32(len(t.nodes)) + for a, b, c in t.nodes: + w.pack('Iii', a, b, c) + w.i32(t.root) + w.u32(len(t.polys)) + for pi in t.polys: + w.u32(pi) + write_pblock(w, t.pblock) + w.pack('4BI', *t.header) + w.vec(t.center) + + +def write_objects(w, objs): + w.u32(len(objs)) + for o in objs: + rec = W() + rec.lstr(strip(o.cls)) + rec.u32(len(o.props)) + for pr in o.props: + rec.lstr(strip(pr.name)) + rec.u8(pr.type) + rec.u32(0) + d = W() + if pr.type == 0: + d.lstr(strip(pr.value)) + elif pr.type in (1, 2): + d.vec(pr.value) + elif pr.type in (3, 4, 6): + d.f32(pr.value) + elif pr.type == 5: + d.u8(pr.value) + elif pr.type == 7: + d.pack('4f', *pr.value) + rec.u16(len(d.b)) + rec.b += d.b + if len(rec.b) > 0xFFFF: + raise ValueError('object record too long') + w.u16(len(rec.b)) + w.b += rec.b + + +def light_objects(world): + """The PS2's baked static lights as PC Light objects. The engine reads them + at load into the light table that lights models and characters + (w_ReadWorldLights: Pos, LightRadius, LightColor, LightObjects; fast by + default); the game has no Light class, so nothing else is made of them.""" + out = [] + for i, L in enumerate(world.lights): + if L.radius <= 0: + continue + props = [P.Obj(name='Name', type=0, value='PS2Light%d' % i), + P.Obj(name='Pos', type=1, value=tuple(L.pos)), + P.Obj(name='Rotation', type=7, value=(0.0, 0.0, 0.0, 0.0)), + P.Obj(name='LightRadius', type=3, value=float(L.radius)), + P.Obj(name='LightColor', type=2, value=tuple(float(c) for c in L.color)), + P.Obj(name='LightObjects', type=5, value=1), + P.Obj(name='FastLightObjects', type=5, value=1)] + out.append(P.Obj(cls='Light', props=props)) + return out + + +def write_light_anims(w, anim): + """One animation, LightAnim_BASE: a single frame of every lightmapped poly + (spec-pc-world.md §7).""" + if not anim: + w.pack('5I', 0, 0, 0, 0, 0) + return + total = sum(len(d) for _, _, d in anim) + w.pack('5I', len(anim), 1, total, len(anim), 1) + w.lstr('LightAnim_BASE') + w.pack('III', 0, 1, len(anim)) + for wi, pi, _ in anim: + w.pack('HH', wi, pi) + for _, _, d in anim: + w.u32(len(d)) + w.b += d + + +def convert(src, dst, pc_dir, ps2_rez, scale=2.0, lightmaps=True): + world = P.load(src) + tex = TexSizes(pc_dir, ps2_rez) + lm = Lightmaps(world.lm_pages) + stats = dict(uv_err=0.0, uv_bad=0, lm_polys=0, lm_too_big=0) + anim = [] if lightmaps else None + w = W() + w.pack('11I', 66, 0, 0, *([0] * 8)) + info = strip(world.info) + w.u32(len(info)) + w.b += info.encode('latin-1') + w.f32(world.lm_grid_size) + w.vec(world.extents_min) + w.vec(world.extents_max) + t = world.tree + w.vec(t.box_min) + w.vec(t.box_max) + w.u32(t.n_nodes) + w.u32(t.terrain_depth) + bits = t.layout + for i in range(0, len(bits), 8): + byte = 0 + for j, bit in enumerate(bits[i:i + 8]): + byte |= bit << j + w.u8(byte) + w.u32(len(world.models)) + for wi, m in enumerate(world.models): + at = w.pos() + w.u32(0) + convert_bsp(w, m, world, tex, lm, stats, scale, wi, anim) + struct.pack_into(' w.textures[...] + w.main.planes[poly.plane] # (nx, ny, nz, dist) + +Command line: ps2world.py file.ltb [--obj out.obj [--model NAME]] [--quiet] +""" +import struct +import sys +from collections import Counter + +# WorldBsp info flags (first dword of a BSP record) +WI_VISBSP = 0x02 # (a world model) uses/has a separate vis BSP +WI_MAINWORLD = 0x04 +WI_PHYSICS = 0x10 +WI_VIS = 0x20 + +# Surface flags (the low ones are the LithTech SURF_* bits) +SURF_SOLID = 0x1 +SURF_NONEXISTENT = 0x2 +SURF_INVISIBLE = 0x4 # PS2: polys of such a surface store bare 4-byte point indices +SURF_TRANSLUCENT = 0x8 +SURF_SKY = 0x10 +SURF_BRIGHT = 0x20 +SURF_FLATSHADE = 0x40 +SURF_LIGHTMAP = 0x80 # PS2: with !TRANSLUCENT, the vertex carries lightmap-page UVs +SURF_NOSUBDIV = 0x100 +SURF_GOURAUDSHADE = 0x1000 + +PROP_STRING, PROP_VECTOR, PROP_COLOR, PROP_REAL, PROP_FLAGS, PROP_BOOL, PROP_LONGINT, PROP_ROTATION = range(8) +PROP_TYPE_NAMES = ['string', 'vector', 'color', 'real', 'flags', 'bool', 'longint', 'rotation'] + + +class FormatError(Exception): + pass + + +class Reader: + __slots__ = ('b', 'p') + + def __init__(self, data, pos=0): + self.b = data + self.p = pos + + def take(self, fmt): + s = struct.Struct('<' + fmt) + v = s.unpack_from(self.b, self.p) + self.p += s.size + return v + + def one(self, fmt): + return self.take(fmt)[0] + + def u8(self): + v = self.b[self.p] + self.p += 1 + return v + + def u16(self): + return self.one('H') + + def u32(self): + return self.one('I') + + def i32(self): + return self.one('i') + + def f32(self): + return self.one('f') + + def vec(self): + return self.take('3f') + + def raw(self, n): + if self.p + n > len(self.b): + raise FormatError('read past EOF at %d (+%d)' % (self.p, n)) + v = self.b[self.p:self.p + n] + self.p += n + return v + + def str16(self): + """u16 length, then that many bytes (the count includes a trailing NUL when present).""" + n = self.u16() + return self.raw(n).rstrip(b'\0').decode('latin-1') + + def str32(self): + n = self.u32() + return self.raw(n).rstrip(b'\0').decode('latin-1') + + +class Obj: + """Plain attribute bag with a readable repr.""" + def __init__(self, **kw): + self.__dict__.update(kw) + + def __repr__(self): + items = ', '.join('%s=%r' % (k, v) for k, v in self.__dict__.items() + if not isinstance(v, (list, bytes)) or len(v) < 8) + return '%s(%s)' % (type(self).__name__, items) + + +class Vert(Obj): + """One polygon vertex. point: index into Bsp.points. u, v: base texture coords + (texture repeats, already divided by the texture size). lm_u, lm_v: lightmap-page + coords (0..1 of the 64x64 page) or None. color: (r, g, b, a) bytes or None.""" + + +class Poly(Obj): + pass + + +class Surface(Obj): + pass + + +class Bsp(Obj): + def poly_xyz(self, poly): + pts = self.points + return [pts[v.point] for v in poly.verts] + + +# ---------------------------------------------------------------- sections + +def read_world_tree(r): + box_min = r.vec() + box_max = r.vec() + n_nodes, terrain_depth = r.take('2I') + # Layout: one bit per node, LSB first, depth-first; a set bit means "has 4 children". + bits = [] + state = [0, 8] + + def node(): + if state[1] == 8: + state[0] = r.u8() + state[1] = 0 + sub = (state[0] >> state[1]) & 1 + state[1] += 1 + bits.append(sub) + if sub: + for _ in range(4): + node() + node() + if len(bits) != n_nodes: + raise FormatError('world tree: %d nodes in layout, header says %d' % (len(bits), n_nodes)) + return Obj(box_min=box_min, box_max=box_max, n_nodes=n_nodes, + terrain_depth=terrain_depth, layout=bits) + + +def read_bsp(r, textures): + """One BSP record (r at the name). Returns Bsp.""" + m = Bsp() + m.offset = r.p + m.name = r.str16() + (m.info_flags, m.unk_dc, n_points, n_planes, n_surfaces, n_portals, n_polys, + n_leaves, m.n_verts, m.vis_data_size, m.n_leaf_lists, n_nodes, + m.n_leaf_poly_refs, m.n_lm_polys) = r.take('14I') + m.box_min = r.vec() + m.box_max = r.vec() + m.translation = r.vec() + m.unk_size = r.u32() # replaces PC's texture-name block; meaning unknown (looks like a memory size) + + vcounts = [r.take('BB') for _ in range(n_polys)] # (n_verts, render mode) + m.poly_vcounts = vcounts + if sum(a for a, _ in vcounts) != m.n_verts: + raise FormatError('%s: vertex counts sum %d != %d' % (m.name, sum(a for a, _ in vcounts), m.n_verts)) + + # leaves (same as PC) + m.leaves = [] + for _ in range(n_leaves): + lf = Obj(share=None, lists=[], poly_refs=None, unk=None) + n = r.u16() + if n == 0xFFFF: + lf.share = r.u16() + else: + for _ in range(n): + portal, size = r.take('hH') + lf.lists.append((portal, r.raw(size))) + npr = r.u32() + lf.poly_refs = list(struct.unpack_from('<%dH' % (2 * npr), r.b, r.p)) + lf.poly_refs = list(zip(lf.poly_refs[0::2], lf.poly_refs[1::2])) + r.p += 4 * npr + lf.unk = r.f32() + m.leaves.append(lf) + + m.planes = [r.take('4f') for _ in range(n_planes)] + + m.surfaces = [] + for _ in range(n_surfaces): + s = Surface() + s.O = r.vec() + s.P = r.vec() + s.Q = r.vec() + s.plane, s.flags, s.texture, s.texture_flags, has_effect = r.take('IIIHH') + s.effect = s.effect_param = None + if has_effect: + s.effect = r.str16() + s.effect_param = r.str16() + m.surfaces.append(s) + + m.polys = [] + for i in range(n_polys): + p = Poly() + p.index = i + (p.lm_page, p.lm_w, p.lm_h, p.lm_x, p.lm_y, p.surface, p.plane, + p.unk_f, cx, cy, cz, p.radius) = r.take('i4BII5f') + p.center = (cx, cy, cz) + p.mode = vcounts[i][1] # 0 lightmapped, 2 bright/unlit, 3 vertex-coloured, 4 invisible + fl = m.surfaces[p.surface].flags + nv = vcounts[i][0] + verts = [] + if fl & SURF_INVISIBLE: + p.vkind = 'index' + for pt in struct.unpack_from('<%dI' % nv, r.b, r.p): + verts.append(Vert(point=pt, u=None, v=None, lm_u=None, lm_v=None, color=None)) + r.p += 4 * nv + elif (fl & SURF_LIGHTMAP) and not (fl & SURF_TRANSLUCENT): + p.vkind = 'lightmap' + for _ in range(nv): + pt, pad, u, v, lu, lv = r.take('HHffff') + verts.append(Vert(point=pt, pad=pad, u=u, v=v, lm_u=lu, lm_v=lv, color=None)) + else: + p.vkind = 'color' + for _ in range(nv): + pt, pad, u, v, cr, cg, cb, ca = r.take('HHff4B') + verts.append(Vert(point=pt, pad=pad, u=u, v=v, lm_u=None, lm_v=None, + color=(cr, cg, cb, ca))) + p.verts = verts + m.polys.append(p) + + m.nodes = [r.take('Iiii') for _ in range(n_nodes)] # poly, leaf, side0, side1 + + m.portals = [] + for _ in range(n_portals): + name = r.str16() + a, b = r.take('IH') # as on PC: dword, word (seen: 0, 0x1c) + m.portals.append(Obj(name=name, unk1=a, flags=b, center=r.vec(), dims=r.vec())) + + m.points = [] + ws = [] + for _ in range(n_points): + x, y, z, w = r.take('4f') + m.points.append((x, y, z)) + ws.append(w) + m.point_w = set(ws) + + m.pblock = read_pblock(r) + + m.root_node = r.i32() + n_sections = r.u32() + m.terrain_sections = [read_terrain_section(r) for _ in range(n_sections)] + return m + + +def read_pblock(r): + """PBlock table (a coarse grid of poly lists, used for collision); same as PC.""" + sx, sy, sz = r.take('3I') + pb = Obj(size=(sx, sy, sz), block_size=r.vec(), origin=r.vec()) + n_rec = r.u32() + pb.n_data = n_rec + pb.data = r.raw(n_rec * 6) # n_rec records {u16 node, i16 side0, i16 side1}, block-local BSPs + pb.blocks = [r.take('IHH') for _ in range(sx * sy * sz)] # (0 [runtime ptr], n_records, root = n-1) + return pb + + +def read_terrain_section(r): + """One terrain section (a sub-BSP over the parent's polys), as on PC (CWorldSimple::Load).""" + t = Obj(offset=r.p) + n = r.u32() + t.nodes = [r.take('Iii') for _ in range(n)] # poly index, side0, side1 (-1 in / -2 out) + t.root = r.i32() + n = r.u32() + t.polys = list(r.take('%dI' % n)) if n else [] + t.pblock = read_pblock(r) + t.header = r.take('4BI') + t.center = r.vec() + return t + + +def read_world_models(r, textures, file_size): + n = r.u32() + models = [] + for i in range(n): + start = r.p + nxt = r.u32() + if not (start < nxt <= file_size): + raise FormatError('world model %d: bad next offset %d' % (i, nxt)) + m = read_bsp(r, textures) + m.record_offset = start + m.record_end = nxt + if r.p != nxt: + raise FormatError('world model %d %r: ended at %d, next record at %d' % (i, m.name, r.p, nxt)) + models.append(m) + return models + + +def read_lights(r): + n = r.u32() + lights = [] + for _ in range(n): + f = r.take('14f') + lights.append(Obj(radius=f[0], color=f[1:4], unk=f[4:7], pos=f[7:10], + dir=f[10:13], cos_cone=f[13])) + return lights + + +def read_light_grid(r): + g = Obj() + g.count = r.u32() + g.dims = r.take('3I') + g.dims_m1 = r.take('3I') + g.stride_z = r.u32() # dims[0]*dims[1] + g.cell = r.vec() + g.inv_cell = r.vec() + g.origin = r.vec() + if g.count != g.dims[0] * g.dims[1] * g.dims[2]: + raise FormatError('light grid count mismatch') + g.data = r.raw(4 * g.count) # R, G, B, 0 per cell; x fastest, then y, then z + return g + + +def clut_index(i): + """PS2 CSM1 CLUT order: bits 3 and 4 of the index are swapped in storage.""" + return (i & ~0x18) | ((i & 0x08) << 1) | ((i & 0x10) >> 1) + + +def read_lm_pages(r): + total, n = r.take('2I') + pages = [] + for _ in range(n): + clut = r.raw(1024) + pix = r.raw(4096) + pages.append(Obj(clut=clut, pixels=pix)) + return total, pages + + +def lm_page_rgba(page): + """Decode a lightmap page to 64*64 (r, g, b, a) tuples, row-major. Byte semantics + of the CLUT entries are not fully understood (see spec): returned raw.""" + pal = [tuple(page.clut[4 * clut_index(i):4 * clut_index(i) + 4]) for i in range(256)] + return [pal[i] for i in page.pixels] + + +def read_objects(r): + count, n_strings = r.take('2I') + hdr_unk = r.take('3I2H') + strings = [r.str16() for _ in range(n_strings)] + objs = [] + for _ in range(count): + dlen, ci, npr = r.take('HII') + start = r.p + o = Obj(cls=strings[ci], data_len=dlen, props=[]) + for _ in range(npr): + ni, code, plen = r.take('IBH') + q = r.p + if code == PROP_STRING: + val = strings[r.u32()] + elif code in (PROP_VECTOR, PROP_COLOR): + val = r.vec() + elif code == PROP_REAL: + val = r.f32() + elif code == PROP_BOOL: + val = r.u8() + elif code in (PROP_FLAGS, PROP_LONGINT): + val = r.f32() # stored as float, like the PC + elif code == PROP_ROTATION: + val = r.take('4f') + else: + raise FormatError('unknown property code %d' % code) + if r.p - q != plen: + raise FormatError('property %s: length %d, read %d' % (strings[ni], plen, r.p - q)) + o.props.append(Obj(name=strings[ni], type=code, value=val)) + o.name = next((p.value for p in o.props if p.name == 'Name'), None) + objs.append(o) + return Obj(header_unk=hdr_unk, strings=strings, objects=objs) + + +# ---------------------------------------------------------------- top level + +class World(Obj): + pass + + +def load(path_or_bytes): + data = path_or_bytes if isinstance(path_or_bytes, (bytes, bytearray)) else open(path_or_bytes, 'rb').read() + r = Reader(data) + w = World() + w.size = len(data) + w.sections = [] + + def sec(name, start): + w.sections.append((name, start, r.p - start)) + + s = r.p + w.header = r.take('11I') + (w.version, w.object_pos, w.lightmap_pos, w.hdr_unk3, w.hdr_unk4, + w.lights_pos, w.grid_pos) = w.header[:7] + if w.version != 66: + raise FormatError('version %d' % w.version) + sec('header', s) + + s = r.p + w.info = r.str32() + w.lm_grid_size = r.f32() + w.extents_min = r.vec() + w.extents_max = r.vec() + sec('world info', s) + + s = r.p + w.tree = read_world_tree(r) + sec('world tree', s) + + s = r.p + n_tex, tex_bytes = r.take('2I') + t0 = r.p + w.textures = [r.str16() for _ in range(n_tex)] + if r.p - t0 != tex_bytes: + raise FormatError('texture list size %d != %d' % (r.p - t0, tex_bytes)) + sec('texture list', s) + + s = r.p + w.models = read_world_models(r, w.textures, len(data)) + sec('world models (%d BSPs)' % len(w.models), s) + w.main = next((m for m in w.models if m.info_flags & WI_MAINWORLD), None) + w.vis = next((m for m in w.models if m.info_flags & WI_VIS), None) + + for name, pos, fn, attr in (('static lights', w.lights_pos, read_lights, 'lights'), + ('light grid', w.grid_pos, read_light_grid, 'light_grid')): + if r.p != pos: + raise FormatError('%s expected at %d, at %d' % (name, pos, r.p)) + s = r.p + setattr(w, attr, fn(r)) + sec(name, s) + + if r.p != w.lightmap_pos: + raise FormatError('lightmap pages expected at %d, at %d' % (w.lightmap_pos, r.p)) + s = r.p + w.lm_total, w.lm_pages = read_lm_pages(r) + if w.lm_total != r.p - s: + raise FormatError('lightmap section size') + sec('lightmap pages', s) + + if r.p != w.object_pos: + raise FormatError('objects expected at %d, at %d' % (w.object_pos, r.p)) + s = r.p + od = read_objects(r) + w.object_strings = od.strings + w.object_header_unk = od.header_unk + w.objects = od.objects + sec('objects', s) + + if r.p != len(data): + raise FormatError('%d bytes left at EOF' % (len(data) - r.p)) + return w + + +# ---------------------------------------------------------------- checks / export + +def bsp_bbox(m): + if not m.points: + return None + xs, ys, zs = zip(*m.points) + return (min(xs), min(ys), min(zs)), (max(xs), max(ys), max(zs)) + + +def planarity(m): + """max |n.p - d| over all poly vertices, and how many polys exceed 0.5 units.""" + worst = 0.0 + bad = 0 + for p in m.polys: + nx, ny, nz, d = m.planes[p.plane] + e = max(abs(nx * x + ny * y + nz * z - d) for x, y, z in m.poly_xyz(p)) + worst = max(worst, e) + if e > 0.5: + bad += 1 + return worst, bad + + +def write_obj(w, models, path, mtl_name=None): + """Write the given BSPs to a Wavefront OBJ: one 'o' per BSP, 'usemtl' = texture name, + vt = base texture UV (v flipped for OBJ). Invisible-surface polys get no UVs.""" + out = open(path, 'w') + if mtl_name: + out.write('mtllib %s\n' % mtl_name) + base = 1 + tbase = 1 + used = set() + for m in models: + out.write('o %s\n' % m.name) + for x, y, z in m.points: + out.write('v %.4f %.4f %.4f\n' % (x, y, z)) + # group polys by texture + by_tex = {} + for p in m.polys: + by_tex.setdefault(m.surfaces[p.surface].texture, []).append(p) + for ti, polys in sorted(by_tex.items()): + tname = w.textures[ti] if 0 <= ti < len(w.textures) else 'tex%d' % ti + used.add(tname) + out.write('g %s__%s\nusemtl %s\n' % (m.name, tname.replace('\\', '/'), tname.replace('\\', '/'))) + for p in polys: + if p.vkind == 'index': + out.write('f ' + ' '.join(str(base + v.point) for v in p.verts) + '\n') + continue + for v in p.verts: + out.write('vt %.5f %.5f\n' % (v.u, -v.v)) + out.write('f ' + ' '.join('%d/%d' % (base + v.point, tbase + k) + for k, v in enumerate(p.verts)) + '\n') + tbase += len(p.verts) + base += len(m.points) + out.close() + return used + + +def summary(w, out=sys.stdout): + pr = lambda *a: print(*a, file=out) + pr('file size', w.size, 'version', w.version) + pr('sections:') + for name, off, size in w.sections: + pr(' %-28s @%9d %9d bytes' % (name, off, size)) + pr('total accounted', sum(s for _, _, s in w.sections)) + pr('world info: %r lm grid %g extents %s %s' % (w.info, w.lm_grid_size, w.extents_min, w.extents_max)) + pr('world tree nodes', w.tree.n_nodes) + pr('textures', len(w.textures)) + pr('world models', len(w.models)) + for i, m in enumerate(w.models): + pr(' %3d %-26s flags %#04x pts %5d planes %5d surfs %3d polys %5d verts %5d leaves %4d nodes %5d portals %d pblock %s/%d terrain %d' + % (i, m.name, m.info_flags, len(m.points), len(m.planes), len(m.surfaces), len(m.polys), m.n_verts, + len(m.leaves), len(m.nodes), len(m.portals), 'x'.join(map(str, m.pblock.size)), m.pblock.n_data, + len(m.terrain_sections))) + if w.main: + bb = bsp_bbox(w.main) + pr('main world (%s) bbox %s .. %s' % (w.main.name, bb[0], bb[1])) + pr('main world planarity (max dist, polys > 0.5):', planarity(w.main)) + pr('static lights', len(w.lights), ' light grid', w.light_grid.dims, 'cell', w.light_grid.cell) + pr('lightmap pages', len(w.lm_pages)) + pr('objects', len(w.objects)) + for c, n in Counter(o.cls for o in w.objects).most_common(): + pr(' %5d %s' % (n, c)) + + +def main(argv): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument('file') + ap.add_argument('--obj') + ap.add_argument('--model', action='append', help='BSP name(s) to export (default: main world)') + ap.add_argument('--textures', action='store_true') + ap.add_argument('--models', action='store_true', help='list world-model names') + a = ap.parse_args(argv) + w = load(a.file) + summary(w) + if True: + used = Counter() + for m in w.models: + for p in m.polys: + used[m.surfaces[p.surface].texture] += 1 + print('texture list (index, polys using it, name):') + for i, t in enumerate(w.textures): + print(' %3d %5d %s' % (i, used[i], t)) + if a.obj: + names = a.model or [w.main.name] + ms = [m for m in w.models if m.name in names] + write_obj(w, ms, a.obj) + print('wrote', a.obj, [m.name for m in ms]) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/ps2/spec-pc-world.md b/build/ps2/spec-pc-world.md new file mode 100644 index 00000000..cd879fc2 --- /dev/null +++ b/build/ps2/spec-pc-world.md @@ -0,0 +1,799 @@ +# PC NOLF world file (`Worlds\*.dat`, LithTech 2.x, version 66): byte layout as the engine loads it + +Status: derived from the byte-matching reconstruction in this repo (server.dll / lithtech.exe / +d3d.ren), and validated with `build/ps2/pcworld.py`, which walks **all 71 world +files in `rez/games/nolf-eu-goty/NOLF.REZ` to the exact last byte** (no gaps, no slack, no +leftovers). It also checks every cross-reference the engine checks, plus several it doesn't. + +Conventions: +- All integers are little-endian. Nothing is aligned or padded in the file. Every struct below is + packed, and the field offsets are file offsets relative to the record start. +- `f32` is an IEEE float. `vec3` is 3×f32 (x, y, z). LithTech space is left-handed with Y up. +- `lstr` = `ILTStream::ReadString` (src/engine/server/basestream.cpp `CBaseStream::ReadString`): + `u16 len` and then `len` raw chars. There is **no NUL** in the file. The reader truncates to its + buffer size minus 1 but always consumes all `len` chars, so an over-long string is safe on the + wire and is only cut short in memory. +- `cstr` = NUL-terminated chars (used only in the texture-name pool). +- "Reader" = the function that consumes the bytes. Every loader runs identically on the server + (server.dll) and on a remote client (lithtech.exe: the same sources are compiled twice). A + local client inherits the server's copy (`InheritWorld`). + +Contents: +1. File-level layout and load sequence +2. Header +3. World info string, lightmap grid, extents +4. World-tree layout (object quadtree) +5. World-model table and the BSP record (the core: geometry, surfaces, BSP, leaves/PVS, PBlocks) +6. Object list +7. Light animations (lightmaps) +8. What the renderer gets +9. Cross-section consistency rules and limits (the "must be consistent" list) +10. Required vs optional: a minimal first test level +11. Surprises / engine quirks +12. Validation results + +--------------------------------------------------------------------------------------------------- + +## 1. File-level layout and load sequence + +``` +offset 0 Header (44 bytes) §2 +offset 44 WorldInfo string, LMGridSize, Extents §3 +... WorldTree layout §4 +... u32 nWorlds +... nWorlds × { u32 nextOffset; BSP record } §5 +hdr.objectListOffset ==> Object list §6 +hdr.lightAnimOffset ==> Light-animation (lightmap) block §7 (runs to EOF) +``` + +Every shipped file has the sections strictly back to back: +- the last world model's `nextOffset` equals `hdr.objectListOffset`; +- the object list ends exactly at `hdr.lightAnimOffset`; +- the light-animation block ends at EOF. + +The engine **depends on the first of these**. `w_ReadWorldLights` reads the object list +*sequentially* from wherever the stream stands after the last `SeekTo(nextOffset)`. Meanwhile +`sm_LoadWorldObjects` seeks to `hdr.objectListOffset`. The two must therefore point at the same +bytes. + +Load sequence (server: `CServerMgr::DoStartWorld` → `CServerMgr::LoadWorld`, src/engine/server/smgr.cpp:505,1087): + +1. `CServerWorldFull::Load` (src/engine/server/commonmodel.cpp ~5933): + - header (§2) → info string, grid, extents (§3) → world tree (§4) → world models (§5); + - then, from the current stream position, `w_ReadWorldLights` reads the object list (§6). It + keeps only Light/DirLight/ObjectLight, for the light table. + - then `ResolvePolyKeys` and `CollectFlaggedPolies`. +2. `CWorldData::LoadObjects` (src/engine/server/worldmodels.cpp:855): `SeekTo(hdr.lightAnimOffset)`, + then the light animations (§7). The name is misleading: it loads lightmaps. +3. `sm_LoadWorldObjects` (src/engine/server/servermodel.cpp:668): `SeekTo(0)`, re-reads dword 0 + and dword 1, then `SeekTo(hdr.objectListOffset)` and creates every object (§6). + +Remote client: `CClientShell::DoLoadWorld` (src/engine/lithtech/startload.cpp:779) calls +`LoadWorldQ2B` = the same `CServerWorldFull::Load` (lithtech 0x004224d0), then `Sub4250e0` = the same +`CWorldData::LoadObjects` (0x004250e0). Then: +- `CreateRenderWorld` hands the in-memory world to the renderer; +- `CreateVisContainerObjects` (src/engine/lithtech/csvis.cpp:331) builds client objects for the + 0x10/0x20 BSPs. + +`GetWorldInfoString` (client, src/engine/server/commonmodel.cpp `sub_421be0`) reads only §2 and the +info string. + +Any reader failure makes the whole load fail with `LT_INVALIDWORLDFILE`. A failure is a version +mismatch, an index out of range, or a stream error. + +--------------------------------------------------------------------------------------------------- + +## 2. Header — 44 bytes + +Reader: `w_ReadWorldHeader` (src/engine/server/commonmodel.cpp ~5215). Dword 1 is read again by +`sm_LoadWorldObjects`. + +| off | type | name | use | +|-----|--------|---------------------|-----| +| 0 | u32 | version | **must be 66** or the file is refused (`return 0` → LT_INVALIDWORLDFILE) | +| 4 | u32 | objectListOffset | absolute offset of §6. `CServerWorldFull::Load` reads it and drops it; `sm_LoadWorldObjects` seeks to it | +| 8 | u32 | lightAnimOffset | absolute offset of §7. Stored at CWorldData+0x68 (`m_nObjectsOffset`, a misnomer) and used by `LoadObjects` | +| 12 | u32[8] | unused | read into a throwaway. 0 in every shipped file | + +Minimal: version 66, two correct offsets, 8 zero dwords. **Required.** + +--------------------------------------------------------------------------------------------------- + +## 3. World info string, lightmap grid size, extents + +Reader: `CServerWorldFull::Load`. + +| type | name | use | +|-----------|-------------|-----| +| u32 | infoLen | | +| char[infoLen] | info | world properties text, **no NUL**, stored NUL-terminated at MainWorld+0x1b8 | +| f32 | lmGridSize | MainWorld+0xf8: world units per lightmap texel (shipped: 20.0). Used by the renderer's lightmap UV math | +| vec3 | extentsMin | MainWorld+0x144 | +| vec3 | extentsMax | MainWorld+0x150 | + +What the engine derives: +- `BoxMin = extentsMin - 100` and `BoxMax = extentsMax + 100` (+0x168/+0x174); +- `InvExtents = 1/(max-min)` per axis (+0x15c). **A degenerate (zero-thickness) axis divides by + zero.** +- The point-shade "light table" grid is sized from min/max, with cell size `LightTableRes` from + the info string (default 250.0, `w_GetShadeCellSize`, src/engine/server/smgrmisc.cpp:372). It is + filled with `AmbientLight` plus "fast" Light objects (§6). + +Info-string commands are CommandParser syntax (`Cmd arg arg;Cmd ...`). Keys seen in shipped files: +- `AmbientLight r g b`: engine `w_GetAmbientLight`. Values clamped 0..255; used for model lighting. +- `LightTableRes n`: engine. +- `WORLDNAME name`: game, `CGameClientShell::GetNiceWorldName`. +- `PBlockSize`, `LMGridSize`, `TerrainSubDivSize`, `MaxLMSize`: preprocessor-only inputs. The + engine ignores them. lmGridSize and the PBlock cell size are written as binary fields elsewhere. + +Example (T01S01): `"AmbientLight 0 16 32"`, grid 20.0, extents (-4096,-256,-1280)..(10048,1088,3648). + +Minimal: an empty string (or `AmbientLight 128 128 128`, so models are not black), grid 20.0, and +extents equal to the geometry's bounding box (each axis non-zero). **Required** (the fields must be +present and sane). + +--------------------------------------------------------------------------------------------------- + +## 4. World-tree layout (object quadtree) + +Reader: `CWorldTree_F::LoadLayout` (src/engine/server/worldtree.cpp:2693; client twin +src/engine/lithtech/worldtree.cpp:870) → `CWTNode_D::LoadNodes` (worldtree.cpp:1799). + +| type | name | use | +|-------|-----------|-----| +| vec3 | boxMin | root node box | +| vec3 | boxMax | | +| u32 | nNodes | read and **ignored**. Always equals the number of nodes in the bitstream in shipped files | +| u32 | m_p070 | stored at tree+0x70, never used. Shipped values 0..4; looks like the subdivision depth | +| bits | topology | pre-order, **1 bit per node, LSB first**, bytes fetched lazily (a new byte only when the bit cursor reaches 8). Bit 1 = "has 4 children" (quadtree, split in X and Z around the centre), recursing into child 0..3. The last partial byte ends the section | + +Size = 32 + ceil(nNodes/8). For T01S01: 237 nodes, 59 with children, 30 bytes of bits. + +Use: this is the spatial index for objects and lights (`FindObjectsInBox`, vis queries). The +renderer's whole visibility query starts here: `MainWorld+0x6c` vtable +0x14 → per tree node, vis +containers → the VisBSP's PVS walk (§8). + +Objects outside the box go on the tree's root list and still work. + +Minimal: box = extents (padded), nNodes = 1, m_p070 = 0, one byte `0x00` (a single leaf node). +**Required**: 33 bytes. + +--------------------------------------------------------------------------------------------------- + +## 5. World models + +``` +u32 nWorlds +repeat nWorlds: + u32 nextOffset absolute offset of the next world's nextOffset field (for the last world: of §6) + BSP record §5.1 .. §5.16 +``` + +Reader: `CServerWorldFull::Load`, which calls `w_LoadWorldModel` (src/engine/server/worldbsp.cpp:976) +for each record. After each record it does `SeekTo(nextOffset)`, so any bytes between the record end +and nextOffset would be skipped. Shipped files have none. + +**Flag 0x2 double load.** If the BSP's `worldInfoFlags & 0x2`, the loader does `SeekTo(position +after nextOffset)` and loads **the same record again**, with `bCurrent = 0`. It becomes the +"original" copy (CServerWorldRec+0x08) that the transformed copy is reset from. It is *not* a +second record in the file. +- Flag 0x2 is set on every movable world model in shipped data: 16341 of 16514 BSPs. +- The `bCurrent = 0` copy gets node projection code 6 everywhere. +- docs/world.md's (Mac-derived) reading of 0x2 as "separate vis BSP / 1 or 2 records" does not + match the PC loader. + +Each record is a complete, independent BSP world (WorldBsp, 0x150 bytes in memory) with its own +points, planes, surfaces, texture list, polys, nodes and PBlocks. In every shipped file: +- the record list is: world models (doors, props, sky boxes, terrains...) first; +- then **`PhysicsBSP`** (flags 0x14); +- then **`VisBSP`** (flags 0x20), always the last two (71/71 files). + +`worldInfoFlags` bits (WorldBsp+0x48): + +| bit | meaning | who looks | +|------|---------|-----------| +| 0x02 | movable: load an untransformed original copy (see above) | CServerWorldFull::Load | +| 0x04 | main world (WORLDFLAG_MAINWORLD) | `IsWorldObject`, `IsMainWorldModel`, modern renderer's DrawAll path | +| 0x08 | "terrain": has terrain sections (§5.16); the client makes one object per section | CreateVisContainerObjects, PrecacheModels | +| 0x10 | physics BSP | `FindFlagged10`/GetPhysicsBSP; PrecacheModels and CreateVisContainerObjects create a world-model object for it | +| 0x20 | vis BSP (has leaves + PVS) | `FindFlagged20`/GetVisBSP; `IsFlagSet` → MainWorld+0x180 bit 2 ("has PVS"); `ResolvePolyKeys`; a world-model object is created for it | + +Server `CPrecacheView::PrecacheModels` (smgr.cpp ~8630) creates an OT_WORLDMODEL `BaseClass` +object for every 0x10/0x20 BSP, flags RAYHIT|SOLID, filename = BSP name. Client +`CreateVisContainerObjects` does the same, VISIBLE|RAYHIT|SOLID, at the BSP translation. + +Game world-model objects (Door, HingedDoor, TranslucentWorldModel...) set `m_Filename = their Name +property` (e.g. src/game/ObjectDLL/AIVolume.cpp:103). The engine attaches the BSP by +**case-insensitive name match** (`CWorldModelHost::AttachWorldModel` / `wd_FindWorldModel`, +worldmodels.cpp:719,776). Then `se_InitWorldModel` (serverengine.cpp:500): +- **sets the object position to the BSP's `translation`**, ignoring the object's Pos property; +- sets the object dims to `(boxMax - boxMin)/2`. + +### 5.1 BSP record header — 32 + 4 + 4 + (2+len) + 48 + 36 bytes + +| type | name | WorldBsp field | use | +|---------|------|----------------|-----| +| u32[8] | unused | — | read into a throwaway; 0 in all shipped files | +| u32 | worldInfoFlags | +0x48 | see table above | +| u32 | unk_dc | +0xdc | stored, copied on inherit, never read. Shipped values look like a tree depth (6 for a box, 0xED for a PhysicsBSP). Write 0 | +| lstr | name | +0x04 (char[0x41], read with max 0x40 → ≤ 63 chars kept) | world-model name; must match the game object's Name. Terrain section names are derived as `"#$#%d %s"` and the name must leave 8 chars of room (§5.16) | +| u32 | nPoints | +0xac | | +| u32 | nPlanes | +0x6c | | +| u32 | nSurfaces | +0x7c | | +| u32 | nUserPortals | +0xb4 | | +| u32 | nPolies | +0xa4 | | +| u32 | nLeafs | +0x8c | | +| u32 | nTotalVerts | — | **read and dropped**. Equals Σ base vertex count (nVerts, not nFixVerts) over all polys in every shipped BSP | +| u32 | visDataSize | +0x98 | bytes of the PVS pool (Σ leaf-list sizes) | +| u32 | nLeafLists | +0x84 | Σ non-shared leaf-list counts | +| u32 | nNodes | +0x74 | | +| u32 | nLeafPolyRefs | +0x94 | Σ leaf poly-ref counts | +| u32 | nLMAnimRefs | +0x54 array size | Σ per-poly light-anim back-references (§5.7). docs/world.md calls this "PolyData54, meaning unknown"; see there | +| vec3 | boxMin | +0xc4 | bounding box. Gives world-model object dims; used for containment | +| vec3 | boxMax | +0xd0 | | +| vec3 | translation | +0xe8 | pivot/centre. Shipped: box centre for world models, (0,0,0) for VisBSP, and for PhysicsBSP an arbitrary interior point. The object is placed here (see above); `Recalc` measures the max vertex distance from it (+0x14c clip radius) | + +These counts only size allocations (dalloc'd pools). They are summed into `g_WorldBytes`. + +**Several pools are filled through unchecked cursors**, so a count *smaller* than the data is heap +corruption, not an error: +- the texture string pool; +- leaf lists; +- vis bytes; +- leaf poly refs. + +Use exact counts. + +### 5.2 Texture name table + +| type | name | use | +|------|------|-----| +| u32 | nStringBytes | size of the pool (Σ (strlen+1)) | +| u32 | nTextures | +0xc0 | +| cstr × nTextures | names | read byte by byte until each NUL into the pool. **No bound check against nStringBytes** | + +Names are resource-relative paths, e.g. `Tex\Stone\St18\St0927.dtx`. Some are `.spr` (sprite +animations, used when surface flag 0x4000). + +The client binds them in `sub_414240` (src/engine/lithtech/clientmgr2.cpp:1932): `m_pTexture = +cm_AddSharedTexture(names[surface.iTexture])`. +- A missing file prints "Unable to find world texture %s" (debug ≥ 1) and falls back to + `textures\default_texture.dtx`. +- If that is missing too, the texture is NULL and the poly is drawn **flat/untextured**. It does + not fail. + +### 5.3 Per-poly vertex counts — 2 × nPolies bytes + +| type | name | use | +|------|------|-----| +| u8 | nVerts | base vertex count (WorldPoly+0x50) | +| u8 | nFixVerts | extra "T-junction fixed" vertex count (+0x52). 0 = none | + +These are read **twice** (worldbsp.cpp regions E and H): once to size the poly block, then a seek +back and a read to carve it. +- In-memory record size = `0x54 + 0x18*(nVerts+nFixVerts)`, rounded up to 4. +- When nFixVerts is 0 the loader sets the fix list = the base list. +- The renderer draws the fix list only when cvar `FixTJunc` is set. Physics and the sphere/centre + maths use the base list. + +### 5.4 Leaves — nLeafs variable records + +Reader: worldbsp.cpp region I ("groups"). In memory, Leaf is 0x30 bytes at WorldBsp+0x88. + +``` +u16 nLists +if nLists == 0xFFFF: + u16 shareLeaf take leaf[shareLeaf]'s list count and list pointer (must be an EARLIER leaf: + later leaves are still zero at that point) +else: + nLists × { u16 portal; 0 for list 0 (unconditional); else index into userPortals, + used only if that portal is open (flags bit 0) and on screen + u16 size; + u8 bits[size] } PVS bitstream, into the visDataSize pool +u32 nPolyRefs +nPolyRefs × { u16 world; u16 poly } polys touching this leaf, in ANY world model +f32 unk28 Leaf+0x28, not used by the engine or the renderers. + Shipped values look like a distance (~1000..7500) +``` + +After load: +- `ResolvePolyKeys` (commonmodel.cpp ~6220) rewrites every {world, poly} into the `WorldPoly*` + of `worlds[world].bsp.polies[poly]`. +- **It runs on the first 0x20 BSP only, and fails the whole load (code 2) if either index is out + of range — or if there is no 0x20 BSP at all.** +- In shipped files the VisBSP's leaves reference **PhysicsBSP** polys (and polys of 0x08 terrain + models), never its own. +- `wm_ResetGeometry` (worldmodels.cpp:524) computes each leaf's centre/radius from those polys' + base vertices. + - Retail divides by the vertex total unguarded: a leaf with 0 refs gets a NaN centre. That is + harmless for drawing but sloppy. + +PVS bitstream: `VisitPVS`, in lithtech.exe (not reconstructed). docs/world.md D.1.2 has it from +the Mac build. +- The bit cursor is global per list, LSB first. +- The tree is walked from the root. At each node: if node.leaf != 0xFFFF, read one bit (1 → that + leaf is visible). Then, for each child that is a real node (flag 8), read one bit; on 1, descend. +- If the view leaf is NULL or has 0 lists, **every leaf referenced by any node is visited**: the + "draw all" fallback. + +Leaves are only meaningful in a 0x20 BSP. Every other BSP has nLeafs = 0 in shipped data. + +### 5.5 Planes — 16 × nPlanes bytes + +| type | name | +|------|------| +| vec3 | normal | +| f32 | dist | + +Point side = `dot(normal, p) - dist`. Positive = front. + +### 5.6 Surfaces — variable records, nSurfaces of them + +Reader: worldbsp.cpp region K. In memory, Surface is 0x48 bytes at WorldBsp+0x78. + +| type | name | Surface field | use | +|------|------|---------------|-----| +| vec3 | O | +0x04 | texture origin | +| vec3 | P | +0x10 | texture U axis, **texels per world unit** (length = scale) | +| vec3 | Q | +0x1c | texture V axis | +| u16 | iTexture | +0x3e | index into §5.2. **Not range-checked** | +| u32 | iPlane | +0x34 → &planes[i] | **must be < nPlanes**, else the load fails. The surface's plane is every one of its polys' plane, for the BSP split, physics, culling and lightmap axes | +| u32 | flags | +0x38 low 29 bits | SURF_* (table below). The top 3 bits are **replaced** by `math_FindClosestAxisBasis(plane.normal)` (the lightmap plane type); file bits 29..31 are ignored | +| u8 | unk44 | +0x44 | stored, meaning unknown. Shipped: 0/1 in PhysicsBSP, 0 in VisBSP | +| u8[3]| unusedRGB | — | read and discarded (all 0 shipped) | +| u8 | hasEffect | — | **== 1** (exactly) means two strings follow | +| lstr | effectName | (if hasEffect==1) | ≤ 126 chars kept. The server looks it up in the request's world-class list (the server passes NULL, so nothing is ever made there). The client links Pan/Rotate/Warble/Portal/Mirror effects (docs/world.md B.1.2). Shipped example: `"Pan"`, params `"8 -16"` | +| lstr | effectParams | (if hasEffect==1) | ≤ 254 chars kept | +| u16 | textureFlags | +0x3c | game "surface type" id (footsteps/impacts; `GetPolyTextureFlags`). 0 is fine | + +Surface flag bits seen and used: + +| bit | meaning | consumer | +|-----|---------|----------| +| 0x1 | SOLID: polys collide. **Without it `w_IsPolyBlocking` returns 0** (worldsim.cpp:290) | physics | +| 0x4 | INVISIBLE: never drawn | renderer | +| 0x8 | TRANSLUCENT: a world model whose first surface has it draws in the translucent pass | renderer | +| 0x10 | SKY portal: collected (`CollectFlaggedPolies`, BuildSkyPortalList); marks where the sky shows; not drawn itself | engine/renderer | +| 0x80 | LIGHTMAP: drawn base × lightmap **only if** MainWorld+0x180 bit 4 (LightAnim_BASE exists), else gouraud | renderer | +| 0x100, 0x200, 0x1000, 0x80000, 0x100000 | seen in data. 0x80000 = additive translucency; the rest are mostly editor/preprocessor bits (0x1000 is on almost every surface) | — | +| 0x4000 | texture is a `.spr` sprite animation | client texture binding | +| 0x8000 | PANNINGSKY | renderer | +| 0x400000 | NOSTEP: physics won't step up onto it (WBSP_NOSTEP) | worldsim.cpp:2927 | + +Texture coordinates are not in the file. `w_SetupPolyTexCoordsQT1` (commonmodel.cpp ~6338) computes +for every vertex: `u = dot(pos - O, P)`, `v = dot(pos - O, Q)`, in texels. The renderer divides by +the DTX size. + +### 5.7 Polys — variable records, nPolies of them + +Reader: worldbsp.cpp region L. The vertex counts come from §5.3. + +| type | name | WorldPoly field | use | +|------|------|-----------------|-----| +| vec3 | lmOrigin | +0x34 | lightmap origin (world space) | +| u16 | lmWidth | +0x48 (stored as **u8**) | lightmap texels; ≤ 32 in practice (the 1024-texel decompress cap) | +| u16 | lmHeight | +0x49 (u8) | | +| u16 | nLMAnimRefs | +0x10 | | +| nLMAnimRefs × {u16 iAnim; u16 iSlot} | lmAnimRefs | +0x0c → slice of the nLMAnimRefs pool | back-references: anim `iAnim` of §7 lists this poly at position `iSlot` of its poly-ref list. **Verified for every poly in every shipped file (0 mismatches).** The running total is checked against nLMAnimRefs (overflow fails the load) | +| u32 | iSurface | +0x28 | **must be < nSurfaces**, else the load fails | +| (nVerts+nFixVerts) × { u16 iPoint; u8 r; u8 g; u8 b } | verts | +0x54.. | `iPoint` **must be < nPoints**, else the load fails. The colour is stored B,G,R,A(=0xFF) at vertex+0x14 (a D3DCOLOR): the gouraud/vertex-lit colour, used whenever lightmaps are off | + +Derived at load and not in the file: +- centre/radius from the base vertices (+0.2) in `wm_ResetGeometry`. A 0-vertex poly → divide by + zero in retail; +- u/v (§5.6); +- the per-surface poly chain (`w_LinkSurfacePoliesQT1`); +- `WorldPoly+0x16` = own index. + +Polys are convex fans in stored order. The visible side is the plane's front: the renderer culls +when `dot(N, cam) - D <= 0.01`. + +### 5.8 BSP nodes — 14 × nNodes bytes + +Reader: worldbsp.cpp region M. In memory, Node/CWorldItem18 is 0x18 bytes at WorldBsp+0x70. + +| type | name | Node field | use | +|------|------|------------|-----| +| u32 | iPoly | +0x00 → polies[i] | **must be < nPolies**, else the load fails. The splitting plane is this poly's surface plane | +| u16 | iLeaf | +0x14 | 0xFFFF = none. In a VisBSP, exactly the nodes whose **front** child is the -1 terminal carry a leaf. **Not range-checked**: keep < nLeafs | +| i32 | back | +0x04 | child index, or **-2** | +| i32 | front | +0x08 | child index, or **-1** | + +Child resolution: `w_GetTreeNodeQT1`, worldbsp.cpp:1798. +- -1 → global terminal node (flags 1). +- -2 → global terminal node (flags 2). +- `(int)i >= nNodes` → NULL → the load fails. +- **Any other negative value is not caught**: it becomes a garbage pointer. + +Shipped convention, without exception: +- back terminals are always -2 (solid/outside); +- front terminals are always -1 (empty/inside). + +The loader presets node flag byte +0x16 = 8 ("real node"). `w_SetNodeProjectionsQT1` sets +0x17 +to 0..5 when the plane is axis-aligned (|n| > 0.99999), else 6. Only for bCurrent = 1. + +Uses: +- ray casts / IntersectSegment; +- `FindPointInTree` (VisBSP: camera leaf); +- object filtering into nodes; +- **PBlock nodes index this array** (§5.11). + +### 5.9 User portals — variable records, nUserPortals of them + +Reader: worldbsp.cpp region O. In memory, 0x24 bytes at WorldBsp+0xb0. + +| type | name | use | +|------|------|-----| +| lstr | name | ≤ 126 chars, dalloc'd copy. Found by name (`w_FindPortalInWorld`) for the game's portal open/close | +| u32 | unused | read and dropped | +| u16 | flags | +0x04. Bit 0 = open (gates leaf lists > 0) | +| vec3 | center | +0x0c | +| vec3 | dims | +0x18, half extents (on-screen test) | + +0 in nearly every shipped BSP. M11S01's VisBSP has 11. **Optional (0).** + +### 5.10 Points — 24 × nPoints bytes + +| type | name | use | +|------|------|-----| +| vec3 | pos | WorldBsp+0xa8 | +| vec3 | normal | **read and discarded** (shipped data holds a per-vertex normal) | + +### 5.11 PBlock table (physics grid) — CSubGrid + +Reader: `CSubGrid::Load` (worldmodels.cpp:413, = worldbsp.h CWorldSub108 at WorldBsp+0x108). + +``` +i32 countX, countY, countZ (one 12-byte read) +vec3 cellSize e.g. 500,500,500 (8192, 2048, 1600 and 1024 also shipped) +vec3 origin = the BSP's boxMin in shipped files +for each of countX*countY*countZ cells, index = x + y*countX + z*(countX*countY), x fastest: + u16 nNodes + u16 root index into this cell's node run, or 0xFFFF / 0xFFFE + nNodes × { u16 iBspNode; u16 back; u16 front } (6 bytes each, WSNode, worldsim.h:231) +``` + +Cell index order matters: `CWorldSub108::GetPointStatus`, worldbsp.cpp:325, returns +`&cells[ix + iz*sliceStride + iy*countX]` with sliceStride = countX*countY. So X varies fastest, +then Y, and **Z is the outermost axis**. + +What each field means: +- `iBspNode` indexes **this BSP's node array (§5.8)**, not polys. Physics reads `node.poly`'s + vertices and `node.poly.surface.plane`. +- `back`/`front` index **this cell's own node run**, or a terminal: + - 0xFFFF = WS_NODE_SOLID; + - 0xFFFE = WS_NODE_EMPTY. +- In shipped data, back terminals are 0xFFFF and front terminals 0xFFFE, mirroring BSP -2/-1. + The cell tree is the BSP pruned to the cell box. +- An empty cell is `nNodes = 0, root = 0xFFFF`: nothing to collide with. + +How the grid is used: +- `GetPBlock(point)` (the WorldBsp vtable slot) truncates `(p - origin)/cellSize` per axis and + clamps it to `[0, count-1]`. So any point maps to *some* cell; one 1×1×1 cell covering + everything works. +- **If countX/Y/Z are 0, `GetPBlock` returns NULL.** `ws_MoveObject` (worldsim.cpp:1844) then + prints "World %s doesn't have PBlocks!" and the object does not collide with that BSP. +- This is what `ws_MoveObject`, `ws_StairStep`, `ws_MoveCylinder` and `ws_MoveBox` walk + (worldsim.cpp ~2313, 2731, 3033), with a 1024-entry u16 stack. +- **The PhysicsBSP and every solid world model need a non-empty grid for collision.** +- Shipped VisBSPs (and 0x08 terrain models) have `0,0,0` and no cells. + +Shipped: PhysicsBSP 29×3×10 at 500 (T01S01); world models mostly 1×1×1 at 500. + +### 5.12 Root node — 4 bytes + +`i32 root`, resolved like a child (§5.8); NULL fails the load. Normally 0. Terrain models use -2. + +### 5.13 .. 5.16 Terrain sections ("CWorldSimple", the 0x08 BSPs) + +``` +u32 nSections +nSections × { + u32 nNodes + nNodes × { u32 iPart (parent poly index, < parent nPolies); i32 back; i32 front } + (children resolved within this section's own nodes; -1/-2 terminals) + i32 root + u32 nParts + nParts × u32 iPoly (< parent nPolies) + CSubGrid (exact §5.11 format) + u8 hdr[4]; u32 hdr2 (wb_ReadSimpleHeader: 4 bytes + 1 dword, stored, unused) + vec3 centre (radius computed from the parts) +} +``` + +Reader: `CWorldSimple::Load` (worldbsp.cpp:1965). Each section is named `"#$# "`; +the name must fit, which caps the parent name at 63-8 chars. + +Shipped: T01S01 "Terrain1" has 2 sections. All other BSPs have 0. **Optional: write `u32 0`.** + +--------------------------------------------------------------------------------------------------- + +## 6. Object list + +Readers: +- `sm_LoadWorldObjects` (servermodel.cpp:668) creates objects. +- `w_ReadWorldLights` (commonmodel.cpp ~5673) reads Light, DirLight and ObjectLight for the light + table. + +``` +u32 nObjects +nObjects × { + u16 recordSize bytes from after this field to the end of the record (= the lstr through the last prop) + lstr className ≤ 255 chars + u32 nProps + nProps × { + lstr name ≤ 255 read; ≤ 79 kept (ObjectProp::m_Name) + u8 type PT_*: 0 string, 1 vector, 2 color, 3 real, 4 flags, 5 bool, 6 longint, 7 rotation + u32 propFlags editor PF_* bits (0x40, 0x2000, 0x1, ...); read and ignored + u16 dataLen + u8 data[dataLen] + } +} +``` + +What the engine does with them: +- `recordSize` is ignored by `sm_LoadWorldObjects`. **`w_ReadWorldLights` uses it** + (`SeekTo(recordStart + recordSize)`) to skip non-light records, so it must be exact. Every + shipped record matches exactly (asserted by the parser). +- An unknown class only prints "Server is missing class %s". +- Classes flagged CF_NORUNTIME, or not CF_ALWAYSLOAD unless the caller asks for all, are skipped. +- Properties go on a list that the game's PreCreate reads with `GetPropGeneric` / `GetPropReal`, + etc. + +Data encoding by type (verified on M01S01): + +| type | dataLen | data | +|------|---------|------| +| 0 string | 2 + n | **an lstr** (u16 n + chars) inside the data. The engine re-reads it with ReadString(0x258) and copies `dataLen-1` | +| 1 vector, 2 color | 12 | vec3 (colour 0..255 floats) | +| 3 real | 4 | f32 | +| 4 flags, 6 longint | 4 | **f32**: the engine reads longints as `(long)m_Float` (smgr.cpp `si_GetPropLongInt`). Shipped e.g. `Type` = `0000803f` (1.0) | +| 5 bool | 1 | u8 | +| 7 rotation | 16 | f32 euler pitch, yaw, roll (radians), plus one f32 that is always 0 and ignored | + +`w_ReadWorldLights` quirks: any property of a Light-class record longer than 0x80 bytes makes it +seek to the record end mid-loop and keep reading the next "property" from there. Keep light props +≤ 0x80 bytes. + +What the game needs (src/game/ObjectDLL): nothing is engine-required. +- Without a `GameStartPoint`, `CGameServerShell::CreatePlayer` spawns the player at (0,0,0) + (GameServerShell.cpp:1581). +- `GameStartPoint` forces its own name to "GameStartPoint" (GameStartPoint.cpp:114), and the + finder matches by name. +- `WorldProperties` carries FarZ, fog, sky panning, environment map and music. +- Every Door/TranslucentWorldModel/etc. object must name an existing BSP. +- Shipped T01S01: 426 objects; M01S01: 731; M11S01: 3626. + +Minimal: `nObjects = 1` with a GameStartPoint (Name, Pos, Rotation, GameType "Single"), plus +WorldProperties for sane FarZ/fog. `nObjects = 0` also loads, with the player at the origin. + +--------------------------------------------------------------------------------------------------- + +## 7. Light animations (lightmaps) — at hdr.lightAnimOffset, to EOF + +Reader: `CWorldData::LoadObjects` (worldmodels.cpp:855). The format and codecs are documented in +docs/world.md part C (src/modern/render/docs/world.md:1291). + +``` +u32 nTotalPolyRefs pool sizes; every cursor is bounds-checked against them (LT_ERROR on overflow) +u32 nTotalFrames +u32 nTotalDataBytes +u32 nTotalFrameEntries (= Σ nFrames*nPolies) +u32 nAnims +nAnims × { + lstr name ≤ 31 chars ("LightAnim_BASE", "__LA", ...) + u32 bShadowMap 0 = colour lightmaps (16-bit RLE of X1R5G5B5), 1 = shadow map (8-bit two-colour RLE) + u32 nFrames + u32 nPolies + nPolies × { u16 world; u16 poly } which polys (any world model) this anim covers + nFrames × nPolies × { u32 size; u8 data[size] } per frame, per poly, compressed lightmap of lmWidth×lmHeight +} +``` + +Behaviour: +- If an anim named `LightAnim_BASE` exists (case-insensitive), MainWorld+0x180 gets bit 0x4. + Its frame pair and blend are zeroed. +- **Only then** does the renderer lightmap SURF_LIGHTMAP polys: PC d3d.ren 0x100256d0 cascade; + Mac `r_SetPolyFunctions`; modern `mr_lightmap.cpp:363`. +- Without it, those polys are drawn gouraud with the file's vertex colours. +- The {world, poly} refs are not validated at load and must be valid. +- Each poly's §5.7 back-refs must point back here. + +Shipped: T01S01 has 2 anims (`LightAnim_BASE` covering 3888 polys, 128480 data bytes; plus +`cigarette__LA`). M11S01 has 28. + +Minimal: **five zero dwords** (nAnims = 0). Every poly then has nLMAnimRefs = 0 (and nLMAnimRefs = +0 in each BSP header), and lmWidth/lmHeight can be 0. This gives vertex-colour lighting: set the +vertex RGB to 255,255,255 for fullbright. + +--------------------------------------------------------------------------------------------------- + +## 8. What the renderer gets (nothing is read from the file by d3d.ren) + +`CreateRenderWorld` hands d3d.ren a `MainWorld*`. +- d3d.ren (src/engine/d3d/*.cpp) and the modern renderers (src/modern/render/ltworld.h, + mr_scene.cpp, mr_lightmap.cpp) read the in-memory structures directly: + - WorldData {flags, bsp, originalBsp, renderBsp} per world; + - WorldBsp, Node, Leaf, WorldPoly, Surface (offsets in ltworld.h / docs/world.md part B); + - the LMAnim arrays. +- Field-for-field, the renderer consumes: + - poly vertices → `points[]`; + - u/v computed from O/P/Q; + - vertex RGB; + - lmOrigin/lmWidth/lmHeight plus lmGridSize and the surface's plane-type bits (lightmap UVs); + - surface flags and texture; + - node tree, leaves and PVS (visibility); + - world-model objects' transforms. + +The per-frame visibility path (retail PC d3dqip4.cpp:778 `sub_1007b580`): +- **If MainWorld+0x180 & 2** (a 0x20 BSP exists) and not DrawAll: + - world-tree query → the VisBSP container's `VisitPVS` from `FindPointInTree(VisBSP, camera)`; + - each visible leaf queues its `polyRefs`. These are PhysicsBSP/terrain polys: **what is drawn + is the PhysicsBSP geometry, selected by VisBSP leaves**. +- **Else (or DrawAll)**: `r_DrawBSP(GetVisBSP())` queues every poly of the **VisBSP itself**. + - If no 0x20 BSP exists, GetVisBSP returns NULL: "ERROR: GetVisBSP returned LTNULL!" and no + world is drawn. This is moot: such a file never loads (§5.4 `ResolvePolyKeys`). + - The modern renderer's DrawAll path draws the 0x04 main world's polys instead (mr_scene.cpp:917). +- Then come the sky (SURF_SKY polys plus SkyPointer objects), world models (each its own BSP under + its transform) and objects. + +--------------------------------------------------------------------------------------------------- + +## 9. Consistency rules and limits + +Checked by the engine (load fails): +- version == 66; +- surface.iPlane < nPlanes; +- poly.iSurface < nSurfaces; +- vertex iPoint < nPoints; +- Σ poly lmAnimRef counts ≤ nLMAnimRefs; +- node.iPoly < nPolies; +- node children ∈ [0, nNodes) ∪ {-1, -2} (other negatives are NOT caught); +- root resolvable; +- terrain part/node indices; +- **a 0x20 BSP exists**, and all its leaf poly refs {world < nWorlds, poly < that world's + nPolies}; +- light-anim pool totals ≥ use; +- stream error at the end. + +Not checked (corrupt memory or misbehave instead): +- texture pool overflow; surface.iTexture ≥ nTextures; +- leaf-list, vis-byte and leaf-ref pool overflow (counts must be ≥ actual; use exact); +- shared-leaf index (must be earlier); +- node.iLeaf < nLeafs; +- PBlock root/children < cell nNodes or 0xFFFE/0xFFFF; PBlock iBspNode < nNodes; +- light-anim {world, poly} refs; poly back-refs (only the renderer uses them); +- object recordSize (must be exact for w_ReadWorldLights). + +Limits from field widths and fixed buffers: +- **≤ 255 base verts and ≤ 255 fix verts per poly** (u8). **Physics clips with + `WS_MAX_POLY_VERTS = 40`** (worldsim.h:354) and retail has no bound check (stack overflow + beyond). Shipped max is 24. Keep ≤ 32 to leave room for clipping. +- Poly index u16 in: leaf refs, light-anim refs, `WorldPoly+0x16`, surface poly chains (0xFFFF = + end), and HPOLY (world << 16 | poly). So **< 65535 polys per BSP**. +- PBlock: iBspNode u16 (**< 65534 BSP nodes per BSP if PBlocks reference them**); < 65534 nodes + per cell; traversal stack 1024 (deep cell trees overflow it). The BSP walker uses a 1000-entry + stack (worldsim.cpp:8). + - Shipped maxima: 59278 BSP nodes, 23234 polys and 33585 nodes in one cell. +- Lightmaps ≤ 32×32 texels (1024-texel decompress buffer); lmWidth/lmHeight are stored as u8. +- Names: BSP ≤ 63, portal ≤ 126, effect ≤ 126/254, anim ≤ 31, object prop name ≤ 79 (in memory). +- Leaf lists: count u16 (0xFFFF reserved for "share"), size u16. + +Coordinate/semantic conventions to reproduce: +- Plane test is `n·p - d`, front positive. +- A face is visible from its front. +- BSP back-terminal = -2 (solid), front-terminal = -1 (empty). The leaf index sits on the node + whose front is -1. +- PBlock 0xFFFF back / 0xFFFE front. +- u/v in texels: P/Q have length = texels per unit. +- Vertex colour order in the file is R, G, B. + +--------------------------------------------------------------------------------------------------- + +## 10. Required vs optional — a minimal first test level (loads, renders vertex-lit, collides) + +| section | verdict | minimal content | +|---------|---------|-----------------| +| §2 header | REQUIRED | 66, objOfs, lmOfs, 8×0 | +| §3 info/grid/extents | REQUIRED | "" or "AmbientLight 128 128 128"; 20.0; real bounding box | +| §4 world tree | REQUIRED | box, 1, 0, byte 0x00 | +| §5 nWorlds | REQUIRED | 2 + number of world models | +| world models (doors...) | optional | 0. If present: flags 0x2, a name matching the object's Name, box/translation = their bounds/centre, own PBlock grid for collision | +| PhysicsBSP (flags **0x14**) | REQUIRED | all level geometry. Points, planes, surfaces (SOLID 0x1 for collision; omit 0x80 without lightmaps), polys (vertex RGB 255 = fullbright), BSP nodes (needed for ray casts and as PBlock targets), nLeafs 0, PBlock grid non-empty (simplest: 1×1×1 cell, cellSize ≥ box size, origin = boxMin, the cell's node run = the whole BSP with -2→0xFFFF and -1→0xFFFE, root = BSP root; ≤ 65533 nodes), root, nTerrain 0 | +| VisBSP (flags **0x20**) | REQUIRED (the load fails without one) | can be tiny: 3 points, 1 plane, 1 surface, 1 texture name, 1 triangle poly, **1 node {poly 0, leaf 0, back -2, front -1}**, root 0, PBlock grid 0,0,0, **1 leaf**: nLists 1 → {portal 0, size 1, bits 0xFF} (or nLists 0, which relies on the draw-all fallback), polyRefs = {PhysicsBSP world index, i} for every PhysicsBSP poly, f32 0. nLeafLists 1, visDataSize 1, nLeafPolyRefs = PhysicsBSP nPolies. Later, a real PVS (multiple leaves) only improves culling | +| user portals | optional | 0 | +| terrain sections | optional | 0 | +| §6 objects | optional for loading; recommended | GameStartPoint (+ WorldProperties) | +| §7 light anims | optional | 5×u32 0. All polys: nLMAnimRefs 0 | +| per-poly lightmap fields | optional | lmOrigin any, lmW/H 0 | +| fix (T-junction) verts | optional | 0 | +| per-point normals | ignored | zeros | +| surface effects | optional | hasEffect 0 | +| texture flags / unk44 / unk_dc / nTotalVerts / 8 unused dwords | ignored or cosmetic | 0 (nTotalVerts may be the real sum; it is dropped) | + +The VisBSP's own geometry is never drawn on the PVS path. Its single poly only has to give the +node a plane. Make that plane put the whole level on the **front** side (e.g. a horizontal plane +below the level with its normal up). Then `FindPointInTree` returns leaf 0 for any camera: the root +node carries leaf 0, so the leaf is remembered regardless of the side taken. + +Objects get filtered into VisBSP nodes, so with one node all objects share leaf 0's visibility. +That is fine. + +--------------------------------------------------------------------------------------------------- + +## 11. Surprises / quirks + +1. **The 0x2 flag re-reads the same record** (the "original" copy for movers). There is never a + second record in the file. docs/world.md §7 / B.1.1 (Mac reading) should be read with that in + mind. +2. **No 0x20 VisBSP → the world fails to load** (`ResolvePolyKeys` returns 0 → code 2 → + LT_INVALIDWORLDFILE). +3. The **drawn** geometry is the PhysicsBSP's, chosen through VisBSP leaves. Retail's DrawAll path + instead draws the VisBSP's own (coarser) polys. The modern renderer differs and draws the + 0x04 world. +4. The header's "objects offset" field at CWorldData+0x68 (worldmodels.cpp) actually holds the + **lightmap** offset: dword 2. Dword 1 is the object list. +5. The object list is read twice: once sequentially (lights only, needing exact recordSize), once + by seek. +6. `nTotalVerts` (the 7th count) is read and dropped. The leaf `f32`, the point normals, the + surface RGB and the user-portal dword are read and dropped too. `unk_dc`, `unk44` and the + world-tree `m_p070` are stored and never used. +7. The "edges" array (WorldBsp+0x54, per-poly `{u16,u16}` at WorldPoly+0x0c, called PolyData54 in + docs/world.md) is **the poly → light-animation back-reference list**: `{iAnim, iSlotInAnim}`. + This is verified exhaustively on all 71 files. +8. lmWidth/lmHeight are u16 on the wire but stored as u8. +9. Surface flag bits 29..31 from the file are overwritten (lightmap plane type). +10. PT_LONGINT and PT_FLAGS values are stored as **floats**. +11. PBlock node records point at **BSP nodes**, not polys. The cell index order is + `x + y*cx + z*cx*cy`. A grid with a zero dimension = "doesn't have PBlocks" (no collision, + console spam per move). +12. `se_InitWorldModel` moves every world-model object to the BSP's `translation` and sizes it from + the BSP box. The object's own Pos property does not position the geometry. +13. Retail divide-by-zero on 0-vertex polys and on 0-poly leaves (centres become NaN). The modern + build guards these (`NOLFSRC_MODERN`, RETAIL_BUGS.md). +14. `w_GetTreeNodeQT1` uses a signed compare: child indices < -2 are not rejected. +15. Unchecked pools (texture names, leaf lists, vis bytes, leaf refs) make exact counts mandatory. +16. Info-string keys `PBlockSize`/`LMGridSize`/`MaxLMSize`/`TerrainSubDivSize` are inert at + runtime. The engine reads only `AmbientLight` and `LightTableRes` (case-insensitive parser; + shipped files use `ambientlight` too). + +--------------------------------------------------------------------------------------------------- + +## 12. Validation (`scripts/pcworld.py`) + +`python3 -I build/ps2/pcworld.py [-v] file.dat ...`; `--rez ` parses +every world inside a rez. + +The script walks every section in file order and asserts all the §9 ranges, including the unchecked +ones: +- pool totals; +- PBlock child/root ranges; +- leaf indices; +- string prop lengths; +- record sizes; +- the light-anim back-ref identity. + +It then reports any gap or overlap and whether the parse ends at EOF. + +Results: +- **All 71 world files in NOLF.REZ (NOLF GOTY EU) parse to EXACT EOF**, with 0 back-ref mismatches. +- Every file ends with PhysicsBSP (0x14) and VisBSP (0x20). +- The 8 unused header dwords and the 8 unused BSP dwords are 0 everywhere. +- The world-tree `nNodes` dword equals the bitstream node count in all 71 files. + +| | T01S01 (1,565,749 B) | M01S01 (3,101,263 B) | M11S01 (11,463,901 B) | +|---|---|---|---| +| objOfs / lmOfs | 1,072,940 / 1,405,793 | 1,937,679 / 2,736,729 | 7,070,564 / 9,683,562 | +| world tree nodes | 237 | 229 | 305 | +| BSP records | 23 (20×0x2, 1×0x8, 0x14, 0x20) | 144 (139×0x2, 3×0x8, 0x14, 0x20) | 639 (637×0x2, 0x14, 0x20) | +| PhysicsBSP pts/planes/surfs/polys/nodes | 4397/1137/2389/3848/7214 | 5141/973/2490/5049/11309 | 39170/9632/22404/23234/42583 | +| PhysicsBSP PBlocks | 29×3×10 @500, 48404 nodes | 14×4×11 @500, 96359 nodes | 2×1×1 @8192, 50946 nodes | +| VisBSP polys/nodes/leaves | 998/1411/335 | 2034/3388/616 | 3948/5873/962 (11 portals) | +| VisBSP leaf lists / PVS bytes / leaf refs | 233 / 14841 / 4769 | 597 / 66988 / 6618 | 1476 / 61824 / 28681 | +| totals pts/planes/surfs/polys/textures | 6495/1662/3438/5094/205 | 10727/2672/5442/8431/439 | 60856/18390/35052/36755/1288 | +| objects / props | 426 / 12956 | 731 / 30959 | 3626 / 103488 | +| light anims (refs / data bytes) | 2 (3915 / 128561) | 2 (6052 / 316032) | 28 (30765 / 1533559) | +| section bytes: header+info+tree+worlds | 1,072,940 | 1,937,679 | 7,070,564 | +| objects | 332,853 | 799,050 | 2,612,998 | +| light anims | 159,956 | 364,534 | 1,780,339 | +| parse end | EXACT | EXACT | EXACT | diff --git a/build/ps2/spec-ps2-world.md b/build/ps2/spec-ps2-world.md new file mode 100644 index 00000000..8d45e3a3 --- /dev/null +++ b/build/ps2/spec-ps2-world.md @@ -0,0 +1,234 @@ +# PS2 NOLF world file (`worlds/*.ltb`, version 66): binary layout + +This is the layout as decoded by `scripts/ps2world.py`. The parser reads all 16 PS2 worlds in +`extract/Worlds/` (thiefmission1-3, t01s01-t10s01, t01s02, t01s03, m01s01) from start to EOF and +accounts for every byte. Each section boundary is checked against the header offsets and the +per-model "next record" offsets. + +Sources used: the LithTools GDScript reader (`LTB_PS2.gd`, `LTB_PS2_Level_V66.bt`) as a starting +point. Its header, world-tree, texture-list, polygon and object layouts are mostly right, but +its user-portal layout is wrong, and its PBlock, root/terrain and render-data parts are wrong or +skipped. The PC structures are taken from this repo's reconstructed engine +(`src/engine/server/worldbsp.cpp` `w_LoadWorldModel` and `CWorldSimple::Load`) and from +`src/modern/render/docs/world.md` (Part B §7 gives the PC record order). PS2 and PC share the same +LithTech 2 BSP structures. The PS2 version mainly differs in vertex data, surfaces and lighting. + +All values are little-endian. `str16` = u16 byte count + bytes. The count includes the trailing NUL +for texture and object strings but not for user-portal names, so strip NULs. `vec3` = 3×f32. +Coordinates are plain IEEE floats in LithTech world units. There is **no quantisation or packing** +of positions, planes or UVs. + +## 0. File order (thiefmission1.ltb, 3,804,224 bytes) + +| # | section | offset | size | +|---|---|---|---| +| 1 | header | 0 | 44 | +| 2 | world info string + LM grid + extents | 44 | 53 | +| 3 | world tree (quadtree layout bits) | 97 | 75 | +| 4 | texture list | 172 | 2,526 | +| 5 | world models: 153 BSP records | 2,698 | 2,883,279 | +| | … 151 small models (AIVolume, TranslucentWorldModel, doors, sky…) | 2,698 | 274,723 | +| | … `PhysicsBSP` (main world, render + collision) | 277,421 | 1,869,340 | +| | … `VisBSP` (leaves/PVS) | 2,146,761 | 739,216 | +| 6 | static light list (426 lights) | 2,885,977 | 23,860 | +| 7 | light grid (29×6×36 cells) | 2,909,837 | 25,124 | +| 8 | lightmap pages (133 × 64×64 CLUT8) | 2,934,961 | 680,968 | +| 9 | objects (396) | 3,615,929 | 188,295 | + +## 1. Header (44 bytes) + +``` +u32 version = 66 +u32 objectDataPos -> section 9 +u32 lightmapDataPos -> section 8 ("RenderDataPos" in LithTools) +u32 0 (always 0) +u32 13 (always 13; meaning unknown) +u32 lightListPos -> section 6 +u32 lightGridPos -> section 7 +u32 0, 0, 0, 0 +``` +PC v66 uses version, objPos, renderPos and then 8 ignored dwords. The PS2 version stores its extra +section offsets in those dwords. + +## 2. World info +`u32 len; char info[len]` (e.g. `ambientlight 56 56 56`), `f32 lmGridSize` (20), `vec3 extentsMin`, `vec3 extentsMax`. + +## 3. World tree +`vec3 boxMin, boxMax; u32 nNodes; u32 terrainDepth` (0, or 1 in t08/t10), then the layout bits. The +bits are read depth-first, one per node, LSB first. Each set bit means the node has 4 children. The +bit count equals nNodes (341 in thiefmission1). The same structure is used on PC. + +## 4. Texture list +`u32 nTextures; u32 nBytes; str16 name[nTextures]` (NUL included; nBytes = total bytes of the +strings incl. their u16 prefixes). On PS2 this is **one list for the whole world**. The PC version +has a per-BSP list. `Surface.texture` indexes this list. + +## 5. World models +`u32 nRecords`, then per record `u32 nextRecordOffset` (absolute) followed by **exactly one** BSP. +Even models with info flag 0x02 have only one BSP. On PC, flag 0x02 adds a second BSP. + +### 5.1 BSP record +``` +str16 name +u32 infoFlags 0x02 world model, 0x14 = MAINWORLD|PHYSICS ("PhysicsBSP"), 0x20 = VIS ("VisBSP"), 0x08 terrain model +u32 unkDC PC WorldBsp+0xdc, meaning unknown on both platforms +u32 nPoints, nPlanes, nSurfaces, nUserPortals, nPolies, nLeaves +u32 nVerts sum of poly vertex counts (checked) +u32 visDataSize total leaf-list bytes +u32 nLeafLists +u32 nNodes +u32 nLeafPolyRefs total of the leaves' poly-ref counts +u32 nLightmappedPolys == number of polys with lmPage >= 0 (true in all 1469 BSPs) +vec3 boxMin, boxMax, worldTranslation (translation = the object's Pos for world models) +u32 unkSize sits where PC has the per-BSP texture-name block; looks like a memory + size (PhysicsBSP 938,832; a 6-poly box 896); not needed +u8 nVerts, u8 mode × nPolies +``` +`mode` is the PC "nFixVerts" byte, but on PS2 it is a **render mode**: 0 = lightmapped, +2 = bright/unlit (flags 0x121, glass), 3 = vertex-coloured gouraud (0x1101, 0x401001), +4 = invisible (0x1085). No extra vertices are stored for it. + +**Leaves** (×nLeaves). Same as PC: +`u16 n` (0xFFFF → `u16 shareIndex`), else n × {`i16 portal; u16 size; u8 data[size]`} (compressed +PVS), then `u32 nRefs; {u16 world, u16 poly}[nRefs]; f32 unk`. In thiefmission1 every VisBSP leaf +ref points at world 151 = PhysicsBSP. **The PVS works on PhysicsBSP polygons.** + +**Planes** (×nPlanes): `vec3 normal; f32 dist` (n·p = dist). + +**Surfaces** (×nSurfaces, 52 bytes + effect): +``` +vec3 O, P, Q texture origin/axes - valid only for the first poly(s) of the surface +u32 plane a plane index (the surface's first poly's); NOT the plane of each poly +u32 flags SURF_*: 1 solid, 4 invisible, 8 translucent, 0x10 sky, 0x20 bright, + 0x80 lightmap, 0x100 nosubdiv, 0x1000 gouraud, 0x400000 / 0x200000 unknown +u32 texture index into §4 +u16 textureFlags (game surface type) +u16 hasEffect if nonzero: str16 effect, str16 effectParam +``` +The PS2 compiler **merged surfaces**: PhysicsBSP has 77 surfaces for 5930 polys. There is roughly +one surface per (texture, flags). O/P/Q do not reproduce the UVs of most polys. **Use the per-vertex +UVs**, and recompute OPQ per poly for the PC (LithTools `UVMath.convert_uv_to_opq` does the same: +3 vertices → O,P,Q scaled by texture size). + +**Polygons** (×nPolies): a 40-byte header plus vertices. +``` +i32 lmPage lightmap page index (§8), -1 = none +u8 lmW, lmH lightmap rectangle size in texels (often 1×1) +u8 lmX, lmY rectangle position in the 64×64 page +u32 surface +u32 plane the poly's own plane +f32 unk values cluster at -0.0 / 0.0 / -1.0 (±1e-6); meaning unknown +vec3 center = average of the poly's vertices +f32 radius = max distance from center + 0.2 +``` +The vertex format depends on the surface flags: +* `flags & 4` (invisible): `u32 point` per vertex (no UV). +* `flags & 0x80 && !(flags & 8)` (lightmap kind): 20 bytes: `u16 point; u16 0; f32 u, v; f32 lmU, lmV`. +* otherwise (colour kind): 16 bytes: `u16 point; u16 0; f32 u, v; u8 r, g, b, a`. + +`point` indexes the BSP's point list. `u, v` are base-texture coordinates in **texture repeats** +(already divided by the texture size; multiply by the DTX width/height for PC texel UVs). `lmU, lmV` +are 0..1 coordinates on the 64×64 lightmap page. They fall inside [lmX, lmX+lmW] / 64 with a +half-texel inset. Polys with lmPage = -1 still carry lmU/lmV. They have lmW=lmH=0 and their values +are junk (sky, 25 polys). In colour-kind polys `a` is always 127, and `rgb` is a 0-255 light value: +ambient-only vertices hold the ambient value (56,56,56), and bright surfaces hold (128,128,128). +Polys are convex fans in stored order, as on PC. + +**Nodes** (×nNodes, 16 bytes): `u32 poly; i32 leaf (-1 none); i32 side[2]` (node index, -1 = in, +-2 = out). PC stores the leaf as u16, the PS2 as a full dword. + +**User portals** (×nUserPortals): `str16 name; u32 unk (0); u16 flags (0x1c); vec3 center; vec3 dims`. +Only t01s03's VisBSP has them (9). LithTools reads `int,int,ushort` here, which is wrong. + +**Points** (×nPoints): `vec3 pos; f32 w` (always 1.0). PC has vec3 + vec3. + +**PBlock table** (collision grid, same as PC `m_Sub108`): +``` +u32 sx, sy, sz; vec3 blockSize (500,500,500); vec3 origin; u32 nRecords +nRecords × {u16 node; i16 side0; i16 side1} per-block mini BSPs over the BSP's nodes; + sides are block-local record indices, -1 in / -2 out +sx*sy*sz × {u32 0 (runtime pointer); u16 nRecords; u16 root (= nRecords-1)} +``` +The VisBSP has a 0×0×0 table, which is still 40 bytes. + +**Tail**: `i32 rootNode` (0, or -2 for terrain models), `u32 nTerrainSections`, sections. + +**Terrain section** (models with flag 0x08: t01s01 Terrain1, m01s01 Dome1/Canopy_a/b, t02s01 grave, +t08/t10 Terrain1 with 4 sections, t10 EggChair). Same as PC `CWorldSimple::Load`: +`u32 nNodes; nNodes × {u32 poly, i32 side0, i32 side1}; i32 root; u32 nPolys; u32 poly[nPolys]; +PBlock table; u8 h[4], u32 h4 (header, zeros); vec3 center`. + +### 5.2 What is render and what is collision +There are **no separate PS2 render batches or triangle strips** in the file. The rendered world is +the `PhysicsBSP` polygons (convex fans) with their UVs, and the lightmap pages or vertex colours. +The same PhysicsBSP also carries the BSP nodes and the PBlock grid used for collision. VisBSP is a +coarser hull: its polygons are not lightmapped (lmPage -1), 1487 of its 3685 poly centres coincide +with PhysicsBSP polys, and it holds 568 leaves. Its leaf poly refs index PhysicsBSP polys. + +World models (doors, translucent panes, rotating models, sky pieces) are separate BSP records. Each +is named exactly like the object whose `Name` property matches. All 151 non-main BSPs in +thiefmission1 have a matching object: AIVolume 73, TranslucentWorldModel 73, Door 2, +RotatingWorldModel, Weather, DemoSkyWorldModel. Their points are in **world space**, and +`worldTranslation` equals the object's `Pos`. + +## 6. Static light list +`u32 n`, n × 56 bytes: +`f32 radius; f32 r, g, b (0-255); f32 unk[3] (always 0); vec3 pos; vec3 dir; f32 cosCone (-1 = omni, +e.g. 0.866 = 30° spot)`. These are the level's static lights. thiefmission3 has none. + +## 7. Light grid (model ambient lighting volume) +``` +u32 count = nx*ny*nz; u32 nx, ny, nz; u32 nx-1, ny-1, nz-1; u32 nx*ny +vec3 cellSize (250); vec3 1/cellSize; vec3 origin (= world extents min) +count × {u8 r, g, b, 0} x fastest, then y, then z +``` + +## 8. Lightmap pages +`u32 sectionSize (incl. these 8 bytes); u32 nPages`, then nPages × 5120 bytes: +`u8 clut[256][4]` followed by `u8 index[64*64]` (row-major, linear, not GS-swizzled). + +* The CLUT is in PS2 **CSM1 order**: logical index i is stored at entry `i` with bits 3 and 4 swapped + (`ps2world.clut_index`). This was verified on every page: every stored non-zero entry is reached. +* Entry bytes are `(dR, dG, dB, L)`, all multiples of 8 (5-bit source). Entry 0 of every level equals + its ambient: thiefmission1 `00 00 00 38` for ambient 56,56,56, and m01s01 `08 00 00 20` for ambient + 28,32,36. This fits **colour = (L-dR, L-dG, L-dB)**, where L is the brightest channel and d is how + far each channel falls below it. Decoded pages look like proper lightmaps with warm and cool lights + (checked visually). The decode is not yet compared against PC lightmaps. On the PS2, L is likely + drawn as a modulating alpha and d as a subtractive pass. +* Each lightmapped poly owns a unique rectangle (no two polys share one in thiefmission1). + +## 9. Objects +``` +u32 nObjects; u32 nStrings +u32 stringBytes (= Σ(len+3)); u32 unk; u32 unk; u16 unk; u16 unk +str16 strings[nStrings] (class names, property names, string values) +per object: u16 dataLen; u32 classString; u32 nProps; + per prop: u32 nameString; u8 type; u16 len; value +``` +Property types: 0 string = `u32 stringIndex`, 1 vector and 2 color = vec3, 3 real = f32, 4 flags and +6 longint = f32 (as on PC), 5 bool = u8, 7 rotation = 4×f32 **Euler (pitch, yaw, roll, 0)**, not a +quaternion. `len` is checked for every property. thiefmission1 has 396 objects; the largest classes +are AIVolume 73, TranslucentWorldModel 73, PropType 61, ScaleSprite 53, AINode 43, DoorKnob 20 and +Trigger 13. There are no `Light` objects: lighting is baked into sections 6 to 8. + +## 10. Geometry check (thiefmission1) +`thiefmission1.obj` (PhysicsBSP) and `thiefmission1_visbsp.obj` were written by the parser. +* PhysicsBSP: 9729 points, 5930 polys, 23620 verts. bbox (-3456,-64,-6624)..(3712,1216,2368), + which equals the header extents and the world-tree box. No NaN or inf. +* Planarity (|n·p - d| per vertex): max 0.005 in most levels. In thiefmission1, 7 polys deviate by + up to 10.4 units, all on 15° walls whose stored plane normal is (±0.25, 0, ∓0.968). The polys + themselves are planar (Newell normal (0.259, 0, -0.966)), so the stored plane is a rounded normal + from the source. Recompute planes when converting. Other levels show up to 4 units on a few polys. +* All UVs are finite (u range -430..110 repeats on sky/huge polys). Polys that share a point and a + (merged) surface agree on its UV, up to an integer repeat, in only 43% of cases. UVs are per + original face, so the converter must keep per-poly UV/OPQ and not weld UVs per point. + +## 11. Unknown / undecoded +* Header dword 4 (= 13). BSP `unkDC` and `unkSize`. Poly `unk` f32 (0 / -1 values). +* Light record `unk[3]` (always 0). Object header dwords 2-4. +* Leaf `f32 unk` (thousands; a radius/size?). User-portal dword/word. +* Surface flag bits 0x200000 and 0x400000. +* Exact lightmap colour semantics (§8): the (L - d) decode is a well-supported hypothesis, not proven. +* PVS leaf-list compression: assumed identical to PC, not decoded here. +* PS2 lightmaps are lower resolution than the PC's (LM grid 20 nominal, many 1×1 rects). A converter + must resample them into PC lightmap sizes (lmGridSize-based) or regenerate them. From 8deb8f452bb3d4ca1dd205b001b0dea91290ba3e Mon Sep 17 00:00:00 2001 From: nolffan Date: Sat, 24 Oct 2026 00:06:16 +0000 Subject: [PATCH 02/19] PS2: convert the PS2 release's models to PC models ps2model.py reads a PS2 model (.ltb version 16: rigid and skinned pieces from their VU DMA chains, LODs, skeleton, weight sets, child models, animations, sockets) and writes a PC model (.abc version 12) with abcmodel.py, a reader and writer of PC models that rewrites every .abc of the retail archives byte for byte. spec-models.md is both formats. The PS2 keeps names only as hashes, with each model's own multiplier; they are looked up in a dictionary of candidate names (collect_names: every name in the PC models, the identifiers in the PS2 executable and attribute files), collected from the discs and never committed. A multiplier can be as small as 2, so names collide ("Report" and "Select" hash the same): the names the game code asks for come first. A PC vertex weight holds the position in the bone's space times the weight; the PS2's skinned pieces are converted so. Young Cate's model (hero_thief) has the PC player's skeleton, so it uses the PC's own animation models. Checked against PS2 models with PC twins (a hat, a purse, the lighter): same faces, coordinates, weights and animation. Co-Authored-By: Claude Opus 5.5 --- build/ps2/abcmodel.py | 371 ++++++++++++++++++++ build/ps2/ps2model.py | 715 +++++++++++++++++++++++++++++++++++++++ build/ps2/spec-models.md | 180 ++++++++++ 3 files changed, 1266 insertions(+) create mode 100644 build/ps2/abcmodel.py create mode 100644 build/ps2/ps2model.py create mode 100644 build/ps2/spec-models.md diff --git a/build/ps2/abcmodel.py b/build/ps2/abcmodel.py new file mode 100644 index 00000000..d74324ba --- /dev/null +++ b/build/ps2/abcmodel.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""NOLF PC model file (.abc, versions 11/12): an in-memory model, a reader and a +writer. The layout is in spec-models.md. The writer is byte-exact against the +shipped PC files (abcmodel.py check FILE... reads, rewrites and compares). + + abcmodel.py check FILE... round-trip test + abcmodel.py dump FILE summary +""" +import struct, sys + + +class Node: + def __init__(self, name='', index=0, flags=0, matrix=None, nchildren=0): + self.name, self.index, self.flags = name, index, flags + # 4x4 row-major, rows are (r0..r3); translation in column 3 + self.matrix = matrix or [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]] + self.nchildren = nchildren + + +class Vertex: + def __init__(self, pos, normal, weights, sublod=0xCDCD): + self.pos, self.normal = pos, normal + self.weights = weights # [(node, (x,y,z) node-space pos, bias)] + self.sublod = sublod + + +class LOD: + def __init__(self): + self.faces = [] # [((u,v,vi), (u,v,vi), (u,v,vi))] + self.verts = [] + + +class Piece: + def __init__(self, name='', material=0, spec_power=0.0, spec_scale=1.0, + lod_weight=1.0, unknown=0): + self.name, self.material = name, material + self.spec_power, self.spec_scale, self.lod_weight = spec_power, spec_scale, lod_weight + self.unknown = unknown + self.lods = [] + + +class ChildModel: + def __init__(self, name='', build=0, xforms=None): + self.name, self.build = name, build + self.xforms = xforms or [] # per node: ((x,y,z), (qx,qy,qz,qw)) + + +class Anim: + def __init__(self, name='base', dims=(0, 0, 0)): + self.name, self.dims = name, dims + self.val = -1 # v12: always -1 in what we write + self.interp = 200 + self.keys = [] # [(time ms, command string)] + self.tracks = [] # per node: [((x,y,z), (qx,qy,qz,qw))] per key + + +class Socket: + def __init__(self, node, name, rot, pos): + self.node, self.name, self.rot, self.pos = node, name, rot, pos + + +class Model: + def __init__(self): + self.version = 12 + self.command = '' + self.radius = 0.0 + self.lod_dists = [] + self.nodes, self.pieces, self.weight_sets = [], [], [] + self.children = [ChildModel()] # [0] is the model itself (empty name) + self.anims, self.sockets = [], [] + self.bindings = [] # [(name, dims, origin)] for own anims + self.child_bindings = [] # one list per children[1:] + self.lod_count = 1 + # header counts as read (None = compute when writing) + self.hdr = None + + # ---- counts the header carries + def strings(self): + """The distinct names the engine copies into the model's string pool: + the command string, node, animation, keyframe-command and child-model + names, de-duplicated case-sensitively, empty ones not counted (piece, + socket and weight-set names go into fixed buffers). This reproduces the + header's two string counts of every v12 .abc NOLF ships.""" + seen, s = [], set() + for n in ([self.command] + [x.name for x in self.nodes] + + [a.name for a in self.anims] + + [k[1] for a in self.anims for k in a.keys] + + [c.name for c in self.children]): + if n and n not in s: + s.add(n) + seen.append(n) + return seen + + def counts(self): + st = self.strings() + nfaces = sum(len(l.faces) for p in self.pieces for l in p.lods) + nverts = sum(len(l.verts) for p in self.pieces for l in p.lods) + nweights = sum(len(v.weights) for p in self.pieces for l in p.lods for v in l.verts) + return [sum(len(a.keys) for a in self.anims), len(self.anims), len(self.nodes), + len(self.pieces), len(self.children), nfaces, nverts, nweights, + self.lod_count, len(self.sockets), len(self.weight_sets), + len(st), sum(len(x) for x in st)] + + +# ------------------------------------------------------------------ reading + +class R: + def __init__(self, d, o=0): + self.d, self.o = d, o + + def u(self, fmt): + v = struct.unpack_from('<' + fmt, self.d, self.o) + self.o += struct.calcsize('<' + fmt) + return v if len(v) > 1 else v[0] + + def s(self): + n = self.u('H') + v = self.d[self.o:self.o + n].decode('latin-1') + self.o += n + return v + + +def read_abc(data): + r = R(data) + m = Model() + secs = {} + while True: + name = r.s() + nxt = r.u('I') + secs[name] = r.o + if name == 'Header': + m.version = r.u('I') + c = list(r.u('11I')) + if m.version >= 13: + r.u('I') + c += list(r.u('2I')) + m.hdr = c + m.lod_count = c[8] + m.command = r.s() + m.radius = r.u('f') + nd = r.u('I') + r.o += 60 + m.lod_dists = list(r.u('%df' % nd)) if nd > 1 else ([r.u('f')] if nd else []) + elif name == 'Pieces': + r.u('I') # weight count + for _ in range(r.u('I')): + p = Piece() + p.material, p.spec_power, p.spec_scale = r.u('H'), r.u('f'), r.u('f') + if m.version > 9: + p.lod_weight = r.u('f') + p.unknown = r.u('H') + p.name = r.s() + for _ in range(m.lod_count): + l = LOD() + for _ in range(r.u('I')): + fv = [] + for _ in range(3): + u, v = r.u('2f') + fv.append((u, v, r.u('H'))) + l.faces.append(tuple(fv)) + for _ in range(r.u('I')): + nw, sub = r.u('2H') + ws = [] + for _ in range(nw): + ni = r.u('I') + ws.append((ni, r.u('3f'), r.u('f'))) + l.verts.append(Vertex(r.u('3f'), r.u('3f'), ws, sub)) + p.lods.append(l) + m.pieces.append(p) + elif name == 'Nodes': + for _ in range(c[2]): + n = Node(r.s(), r.u('H'), r.u('B')) + f = r.u('16f') + n.matrix = [list(f[0:4]), list(f[4:8]), list(f[8:12]), list(f[12:16])] + n.nchildren = r.u('I') + m.nodes.append(n) + for _ in range(r.u('I')): + wn = r.s() + k = r.u('I') + m.weight_sets.append((wn, list(r.u('%df' % k)) if k > 1 else ([r.u('f')] if k else []))) + elif name == 'ChildModels': + m.children = [] + for _ in range(r.u('H')): + cm = ChildModel(r.s(), r.u('I')) + for _ in range(c[2]): + cm.xforms.append((r.u('3f'), r.u('4f'))) + m.children.append(cm) + elif name == 'Animation': + for _ in range(r.u('I')): + a = Anim() + a.dims = r.u('3f') + a.name = r.s() + a.val = r.u('i') + if m.version >= 12: + a.interp = r.u('I') + nk = r.u('I') + else: + nk = r.u('I') if a.val == -1 else a.val + for _ in range(nk): + t = r.u('I') + a.keys.append((t, r.s())) + for _ in range(c[2]): + a.tracks.append([(r.u('3f'), r.u('4f')) for _ in range(nk)]) + m.anims.append(a) + elif name == 'Sockets': + for _ in range(r.u('I')): + ni = r.u('I') + sn = r.s() + m.sockets.append(Socket(ni, sn, r.u('4f'), r.u('3f'))) + elif name == 'AnimBindings': + def binds(): + return [(r.s(), r.u('3f'), r.u('3f')) for _ in range(r.u('I'))] + m.bindings = binds() + m.child_bindings = [binds() for _ in m.children[1:]] + if nxt == 0xFFFFFFFF: + break + r.o = nxt + m.sections = secs + return m + + +# ------------------------------------------------------------------ writing + +class W: + def __init__(self): + self.b = bytearray() + + def p(self, fmt, *v): + self.b += struct.pack('<' + fmt, *v) + + def s(self, txt): + e = txt.encode('latin-1') + self.p('H', len(e)) + self.b += e + + def section(self, name): + if hasattr(self, 'last'): + struct.pack_into(' OUTDIR + +PS2 files keep piece, animation, socket and weight-set names only as hashes +(ps2rez's hash with the model's own multiplier); they are looked up in a name +dictionary (collect_names: every PC .abc name, the PS2 executable's strings +and the PS2 attribute files; build/ps2/ps2install.py collects it from the +discs, and the research default, out/tmp/ps2-research/ps2model_names.txt, +is made by `names`). It is game data, so it is never committed. A hash +with no match becomes "h". +""" +import math, os, re, struct, sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import abcmodel as A # noqa: E402 +from ps2rez import rez_hash # noqa: E402 + +MT_RIGID, MT_SKEL, MT_VERTANIM = 4, 5, 6 + + +# CWeaponModel's animation names (WeaponModel.cpp: Select, Deselect, Reload, +# Idle_N, FireN, AltIdle_N, AltFireN), the weapon-ani overrides' (Select1: +# WEAPONS.txt BarretteLockAnis), and KeyPad's decipherer "Animation". +PRIORITY = (['Select', 'Deselect', 'Reload', 'Fire', 'AltFire', 'Animation', 'Select1', 'Dec1'] + + ['Idle_%d' % i for i in range(10)] + ['AltIdle_%d' % i for i in range(10)] + + ['Fire%d' % i for i in range(10)] + ['AltFire%d' % i for i in range(10)]) + + +class Names: + def __init__(self, path=None, names=None): + path = path or os.path.join(os.path.dirname(os.path.dirname(HERE)), + 'out', 'tmp', 'ps2-research', 'ps2model_names.txt') + self.names = list(names or []) + if names is None and os.path.isfile(path): + with open(path, encoding='latin-1') as fh: + self.names = [l.rstrip('\n') for l in fh if l.strip()] + # The names the game code asks models for come first: a model's hash + # multiplier can be as small as 2 (h = 3h + c), and then names collide + # ("Report" and "Select" hash the same); the one the code uses wins. + self.names = PRIORITY + self.names + self.tables = {} + + def add(self, names): + self.names = list(names) + self.names + self.tables = {} + + def look(self, h, magic): + t = self.tables.get(magic) + if t is None: + t = {} + for n in reversed(self.names): # earlier entries win + t[rez_hash(n, magic)] = n + self.tables[magic] = t + return t.get(h) + + +# ------------------------------------------------------------------ PS2 reading + +class PS2LOD: + def __init__(self): + self.mesh_type = 0 + self.node = 0 # rigid: the node; skeletal: number of bones + self.strips = [] # [(reversed, [(pos, normal, (u,v), extra)])] + self.qwords = 0 + # skeletal only + self.chain_qwords = 0 + self.wmap = [] # per unique vertex: [qword offsets into the DMA chain] + self.uverts = [] # per unique vertex: (pos, normal) + self.bones = [] # local bone index -> node index + self.weights = [] # per unique vertex: [(node, bias)] + self.strip_qw = [] # per strip vertex: its position qword's offset in the chain + + +class PS2Piece: + pass + + +def ps2_strips_from_chain(d, tag, lod): + """Walk a DMA chain (from the tag at `tag` until a RET/END tag), run its VIF + codes into a VU-memory map and, at each MSCAL*, decode the buffer: a header + qword (strip count-1, vertex count) and then GIF tags (NLOOP vertices of 3 + qwords: xyz1, normal, s t q). Returns the offset after the chain.""" + start = tag + while True: + lo, = struct.unpack_from('> 28) & 7 + p, end = tag + 8, tag + 16 + qwc * 16 + mem = {} + while p < end: + w, = struct.unpack_from('> 24) & 0x7F, (w >> 16) & 0xFF, w & 0xFFFF + p += 4 + if cmd & 0x60 == 0x60: + vn, vl = (cmd >> 2) & 3, cmd & 3 + n = num or 256 + if (vn, vl) != (3, 0): + raise ValueError('UNPACK V%d-%d not handled' % (vn + 1, 32 >> vl)) + for i in range(n): + mem[(imm & 0x3FF) + i] = p + 16 * i + p += 16 * n + elif cmd in (0x14, 0x15, 0x17): + decode_vu_buffer(d, mem, lod, start) + mem = {} + elif cmd == 0x20: + p += 4 + elif cmd in (0x30, 0x31): + p += 16 + elif cmd == 0x4A: + p += (num or 256) * 8 + elif cmd in (0x50, 0x51): + p += (imm or 65536) * 16 + tag = end + if tid in (0, 6, 7): + return tag + + +def decode_vu_buffer(d, mem, lod, chain_start): + a = min(mem) + nm1, nverts = struct.unpack_from('<2I', d, mem[a]) + a += 1 + got = 0 + while True: + lo, hi = struct.unpack_from('> 15) & 1 + rev = bool(hi & 0x8000) # REGS nibble 3: the strip starts reversed + a += 1 + verts = [] + for _ in range(nloop): + o = mem[a] + pos = struct.unpack_from('<3f', d, o) + nrm = struct.unpack_from('<3f', d, o + 16) + s, t, q = struct.unpack_from('<3f', d, o + 32) + extra, = struct.unpack_from(' 1e-12 else (0.0, 0.0, 1.0) + + +def sub3(a, b): return (a[0] - b[0], a[1] - b[1], a[2] - b[2]) +def cross3(a, b): return (a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]) +def dot3(a, b): return a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + + +STATS = {} + + +def strip_faces(lod, vindex): + """Triangles of a LOD's strips, as ((strip vertex, ...) x3) in PC (clockwise + seen from the front) order. vindex(strip, k) gives the PC vertex index.""" + faces = [] + for si, (rev, verts) in enumerate(lod.strips): + for k in range(2, len(verts)): + a, b, c = k - 2, k - 1, k + # A strip alternates its winding every triangle; REGS nibble 3 (0x8) + # says the first one is the other way round. + if (k % 2 == 1) != rev: + a, b = b, a + ia, ib, ic = vindex(si, a), vindex(si, b), vindex(si, c) + if ia == ib or ib == ic or ia == ic: + STATS['degenerate'] = STATS.get('degenerate', 0) + 1 + continue + faces.append(((si, a), (si, b), (si, c))) + return faces + + +# The PS2 player-view projection is not the PC's: the same first-person model +# with the same weapon Pos carries FovXOffset/FovYOffset 31/41 on the PS2 +# where the PC file has 20/50 (lighter, coin, poodle, sterling, s&w, atomlaser, +# smgsky, snow, moto, perfume*, spacechop, karatechop, cameradis, windshield: +# all PS2 twins at 31/41 but bucklezip are 20/50 on the PC). So the PS2 offsets move by the +# same amount (degrees) to look on the PC as they did on the PS2. +PV_FOV_SHIFT = (-11.0, 9.0) + + +def pc_command(cmd): + def shift(mo): + k = 0 if mo.group(1) == 'X' else 1 + v = float(mo.group(2)) + PV_FOV_SHIFT[k] + return 'Fov%sOffset %s' % (mo.group(1), ('%d' % v) if v == int(v) else ('%g' % v)) + return re.sub(r'Fov([XY])Offset\s+(-?[0-9.]+)', shift, cmd) + + +def convert(m, children=(), flip=None): + """PS2Model -> abcmodel.Model. children: [(file name, .abc bytes)] of the PC + child models to reference (the engine loads them from the parent's own + directory); their animation bindings and build numbers are copied, as + ModelEdit does.""" + out = A.Model() + out.version = 12 + out.command = pc_command(m.command) + out.radius = m.radius + out.lod_count = m.n_lods + out.lod_dists = list(m.lod_dists) + binds = [n['m'] for n in m.nodes] + inv = [mat_inverse(b) for b in binds] + for i, n in enumerate(m.nodes): + out.nodes.append(A.Node(n['name'], n['index'], n['unk'] | (1 if i == 0 else 0), + [list(r) for r in n['m']], n['nchildren'])) + # piece names: the node names of this model first (rigid pieces are + # usually named after their node) + m.names.add([n['name'] for n in m.nodes]) + for p in m.pieces: + pc = A.Piece(m.name(p.hash), p.texture, 5.0, 1.0, 1.0, 0) + for l in p.lods: + L = A.LOD() + if l.mesh_type == MT_RIGID: + b = binds[l.node] + key2idx = {} + idx = [] + for rev, verts in l.strips: + row = [] + for pos, nrm, uv, extra in verts: + key = (tuple(round(x, 4) for x in pos), tuple(round(x, 3) for x in nrm)) + vi = key2idx.get(key) + if vi is None: + vi = key2idx[key] = len(L.verts) + L.verts.append(A.Vertex(mat_apply(b, pos), norm(mat_apply(b, nrm, 0.0)), + [(l.node, pos, 1.0)])) + row.append(vi) + idx.append(row) + vindex = lambda si, k: idx[si][k] + else: + qw2u = {} + for u, offs in enumerate(l.wmap): + for q in offs: + qw2u[q] = u + flat = [] + for qw in l.strip_qw: + if qw not in qw2u: + raise ValueError('strip vertex at qword %d not in the weight map' % qw) + flat.append(qw2u[qw]) + starts, n0 = [], 0 + for rev, verts in l.strips: + starts.append(n0) + n0 += len(verts) + for u, (pos, nrm) in enumerate(l.uverts): + ws = l.weights[u] + tot = sum(w for _, w in ws) or 1.0 + # A PC weight's position is the node-space position + # pre-multiplied by the weight: the engine skins with + # sum(M_i * (pos_i, w_i)) / sum(w_i) (every shipped .abc + # stores it so). + L.verts.append(A.Vertex(pos, norm(nrm), + [(ni, tuple(x * (w / tot) for x in mat_apply(inv[ni], pos)), w / tot) + for ni, w in ws])) + vindex = lambda si, k: flat[starts[si] + k] + for tri in strip_faces(l, vindex): + L.faces.append(tuple((l.strips[si][1][k][2][0], l.strips[si][1][k][2][1], vindex(si, k)) + for si, k in tri)) + pc.lods.append(L) + out.pieces.append(pc) + out.weight_sets = [(m.name(h), list(w)) for h, w in m.weight_sets] + ident = [((0.0, 0.0, 0.0), (0.0, 0.0, 0.0, 1.0))] * len(m.nodes) + out.children = [A.ChildModel('', 1, list(ident))] + for a in m.anims: + an = A.Anim(m.name(a['hash']), a['dims']) + an.interp = a['interp'] + an.keys = list(a['keys']) + an.tracks = [list(tr) for marker, tr in a['tracks']] + out.anims.append(an) + out.bindings.append((an.name, a['dims'], a['trans'])) + for s in m.sockets: + out.sockets.append(A.Socket(s['node'], m.socket_name(s), s['rot'], s['pos'])) + for name, data in children: + cm = A.read_abc(data) + if len(cm.children) > 1: + raise ValueError('%s has child models of its own' % name) + out.children.append(A.ChildModel(name, cm.children[0].build, list(ident))) + out.child_bindings.append(list(cm.bindings)) + return out + + +# ------------------------------------------------------------------ name dictionary + +def repo_root(): + d = HERE + while d != os.path.dirname(d) and not os.path.isfile(os.path.join(d, 'tools', 'rezextract.py')): + d = os.path.dirname(d) + return d + + +def collect_names(abcs, texts): + """Candidate names for a PS2 model's hashes: every name in the PC models + `abcs` (their bytes), and the identifiers in `texts` (the bytes of the + PS2 executable and attribute files).""" + names, seen = [], set() + + def add(n): + n = n.strip() + if n and n not in seen and len(n) < 64: + seen.add(n) + names.append(n) + # 1. every name in every PC model + for data in abcs: + try: + m = A.read_abc(data) + except Exception: + continue + for n in ([x.name for x in m.nodes] + [x.name for x in m.pieces] + + [a.name for a in m.anims] + [s.name for s in m.sockets] + + [w[0] for w in m.weight_sets] + [c.name for c in m.children] + + [b[0] for bl in m.child_bindings for b in bl]): + add(n) + # 2. identifiers in the PS2 executable and attribute files + tok = re.compile(rb'[A-Za-z_+][A-Za-z0-9_+&\-. ]{1,40}') + for data in texts: + for t in tok.findall(data): + t = t.decode('latin-1') + for part in [t] + re.split(r'[ .]', t): + add(part) + if '%d' in part: + for i in range(10): + add(part.replace('%d', str(i))) + # 3. weapon animations with numbered variants + for b in ('Idle_', 'Fire', 'PreFire', 'PostFire', 'Fire_'): + for i in range(10): + add(b + str(i)) + return names + + +def build_names(out, pc_dir=None, ps2_dir=None): + """collect_names from the PC game's archives and a PS2 research + extraction (disc/SLUS_200.28, extract/Attributes/*.txt), into a file + (the default Names() reads it, for research; build/ps2/ps2install.py + collects its own).""" + root = repo_root() + pc_dir = pc_dir or os.path.join(root, 'rez', 'games', 'nolf-eu-goty') + ps2_dir = ps2_dir or os.path.join(root, 'out', 'tmp', 'ps2-research') + sys.path.insert(0, os.path.join(root, 'tools')) + from rezextract import Rez + + def abcs(): + for rz in sorted(os.listdir(pc_dir)): + if not rz.lower().endswith('.rez'): + continue + r = Rez(os.path.join(pc_dir, rz)) + for p, pos, size in r.walk(): + if p.lower().endswith('.abc'): + r.f.seek(pos) + yield r.f.read(size) + srcs = [os.path.join(ps2_dir, 'disc', 'SLUS_200.28')] + att = os.path.join(ps2_dir, 'extract', 'Attributes') + if os.path.isdir(att): + srcs += [os.path.join(att, x) for x in sorted(os.listdir(att)) if x.lower().endswith('.txt')] + texts = [open(x, 'rb').read() for x in srcs if os.path.isfile(x)] + names = collect_names(abcs(), texts) + with open(out, 'w', encoding='latin-1') as fh: + for n in names: + fh.write(n + '\n') + print('%d names -> %s' % (len(names), out)) + + +def dump(m): + print('cmd %r radius %g magic %d ext %d,%d lod dists %s' % (m.command, m.radius, m.magic, m.ext1, m.ext2, m.lod_dists)) + for i, n in enumerate(m.nodes): + print('node %2d %-28s idx %d unk %d children %d t=(%.2f %.2f %.2f)' % ( + i, n['name'], n['index'], n['unk'], n['nchildren'], n['m'][0][3], n['m'][1][3], n['m'][2][3])) + for p in m.pieces: + print('piece %-24s tex %d colors %s unk %d %d four %d lods %s' % ( + m.name(p.hash), p.texture, [round(x, 2) for x in p.colors], p.unk1, p.unk2, p.four, + [(l.mesh_type, l.node, len(l.strips), sum(len(v) for r, v in l.strips), len(l.uverts)) for l in p.lods])) + print('weight sets', [m.name(h) for h, w in m.weight_sets]) + print('children', m.children) + for a in m.anims: + print('anim %-20s dims %s trans %s interp %d keys %d %s markers %s' % ( + m.name(a['hash']), [round(x, 2) for x in a['dims']], [round(x, 2) for x in a['trans']], + a['interp'], len(a['keys']), [k for k in a['keys'] if k[1]][:4], + sorted(set('%x' % t[0] for t in a['tracks']))[:4])) + for s in m.sockets: + print('socket %-12s node %d rot %s pos %s' % (m.socket_name(s), s['node'], + [round(x, 3) for x in s['rot']], [round(x, 3) for x in s['pos']])) + + +def main(argv): + names_path = None + if '--names' in argv: + i = argv.index('--names') + names_path = argv[i + 1] + del argv[i:i + 2] + if argv[0] == 'names': + kw = {} + for k in ('--pc-rez', '--ps2-dir'): + if k in argv: + i = argv.index(k) + kw[k[2:].replace('-', '_').replace('rez', 'dir')] = argv[i + 1] + del argv[i:i + 2] + build_names(argv[1], **kw) + return + if argv[0] == 'thief': + convert_thief(argv[1], names=names_path) + return + names = Names(names_path) + m = PS2Model(open(argv[1], 'rb').read(), names) + if argv[0] == 'dump': + dump(m) + elif argv[0] == 'conv': + children = [] + while '--child' in argv: + i = argv.index('--child') + children.append(argv[i + 1]) + del argv[i:i + 2] + out = convert(m, [(os.path.basename(c).lower(), open(c, 'rb').read()) for c in children]) + with open(argv[2], 'wb') as fh: + fh.write(A.write_abc(out)) + print('%s: %d nodes, %d pieces, %d anims, %d sockets, children %s' % ( + argv[2], len(out.nodes), len(out.pieces), len(out.anims), len(out.sockets), + [c.name for c in out.children[1:]])) + + + +# ------------------------------------------------------------------ the thief levels' models + +# (PS2 name, PC child models to reference). The character's PS2 children +# (playerbase.ltb, base_general.ltb, base_death.ltb) are replaced by the PC +# player's own (CHARS/MODELS/PLAYER_ACTION.abc references these three): same +# 25-node skeleton, and the PC animation names the game code asks for. +THIEF_MODELS = [ + ('guns/models_pv/ps2lockpick_pv.ltb', []), + ('guns/models_pv/ps2safecracker_pv.ltb', []), + ('guns/models_pv/delislescope.ltb', []), + ('guns/models_hh/ps2lockpick_hh.ltb', []), + ('guns/models_hh/ps2safecracker_hh.ltb', []), + ('interface/models/hud/briefcase.ltb', []), + ('interface/models/hud/change_01.ltb', []), + ('interface/models/hud/moneybag.ltb', []), + ('interface/models/hud/purses_01.ltb', []), + ('attachments/models/britishpolicehat.ltb', []), + ('props/models/moneybag.ltb', []), + ('props/models/payphone_01.ltb', []), + ('chars/models/hero_thief.ltb', ['basemodel.abc', 'fembase.abc', 'playerbase.abc']), + # the PS2's animation-only child models, converted as they are (not + # referenced by the above; see spec-models.md) + ('chars/models/base_general.ltb', []), + ('chars/models/base_death.ltb', []), + ('chars/models/base_pistol.ltb', []), + ('chars/models/base_rifle.ltb', []), +] + + +def pc_file(pc_dir, name): + sys.path.insert(0, os.path.join(repo_root(), 'tools')) + from rezextract import Rez + want = name.replace('\\', '/').lower() + for rz in sorted(os.listdir(pc_dir)): + if rz.lower().endswith('.rez'): + r = Rez(os.path.join(pc_dir, rz)) + for p, pos, size in r.walk(): + if p.lower() == want: + r.f.seek(pos) + return r.f.read(size) + return None + + +def convert_thief(outdir, ps2_rez=None, pc_dir=None, names=None): + """Every model the thief levels need, from the PS2 archive into + outdir//.abc.""" + import ps2rez + root = repo_root() + ps2_rez = ps2_rez or os.path.join(root, 'out', 'tmp', 'ps2-research', 'disc', 'LITH_PS2.REZ') + pc_dir = pc_dir or os.path.join(root, 'rez', 'games', 'nolf-eu-goty') + f, hdr, tables = ps2rez.open_rez(ps2_rez) + t0 = {e[0]: e for e in tables[0] if e[2]} + for name, kids in THIEF_MODELS: + e = t0.get(ps2rez.rez_hash(name, hdr['magic'])) + if not e: + print('not in the PS2 archive:', name) + continue + f.seek(e[1] * 2048) + m = PS2Model(f.read(e[2]), Names(names)) + children = [] + for k in kids: + data = pc_file(pc_dir, os.path.dirname(name) + '/' + k) + if data is None: + raise ValueError('PC child model %s not found' % k) + children.append((k, data)) + out = convert(m, children) + parts = name.split('/') + dst = os.path.join(outdir, *[x.upper() for x in parts[:-1]], os.path.splitext(parts[-1])[0].upper() + '.abc') + os.makedirs(os.path.dirname(dst), exist_ok=True) + data = A.write_abc(out) + assert A.write_abc(A.read_abc(data)) == data + with open(dst, 'wb') as fh: + fh.write(data) + unnamed = [p.name for p in out.pieces if re.match(r'h[0-9a-f]{8}$', p.name)] + unnamed += [a.name for a in out.anims if re.match(r'h[0-9a-f]{8}$', a.name)] + print('%-44s %3d nodes %3d pieces %4d tris %3d anims %2d sockets%s%s' % ( + dst, len(out.nodes), len(out.pieces), + sum(len(l.faces) for p in out.pieces for l in p.lods), len(out.anims), len(out.sockets), + ' children ' + ','.join(kids) if kids else '', + ' unnamed: ' + ' '.join(unnamed) if unnamed else '')) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/ps2/spec-models.md b/build/ps2/spec-models.md new file mode 100644 index 00000000..ce224a83 --- /dev/null +++ b/build/ps2/spec-models.md @@ -0,0 +1,180 @@ +# NOLF models: PS2 `.ltb` (file type 2, version 16) and PC `.abc` (version 12) + +Implemented by `ps2model.py` (PS2 reader, converter, `thief` batch) and `abcmodel.py` +(PC reader/writer). Both readers check every section boundary: the PS2 reader asserts each +section ends exactly at the next header offset and the last at the file size (all 17 PS2 +models the thief levels use, plus hero_action, player_action, playerbase, fembase, +lighter_pv/_hh, codedec_pv, scope, purses_01 and overcoatagent_action_hat from the PS2 +archive); `abcmodel.py check` re-writes all 692 v12 `.abc` files in the PC archives +byte-identically (header counts included). + +All little-endian. `str16` = u16 length + bytes, no NUL. `vec3` = 3×f32, `quat` = x,y,z,w f32. +Matrices are 4×4 f32 row-major with the translation in column 3 (`m[0][3], m[1][3], m[2][3]`), +the same on both platforms. + +## PS2 `.ltb` v16 + +### Header (0x00) + +| off | type | field | +|---|---|---| +| 0x00 | u32 | file type = 2 (world `.ltb`s are other types) | +| 0x04 | u16 | version = 16 | +| 0x06 | u16, 3×u32 | 0 | +| 0x14 | u32 | 0x10 (offset of the offsets) | +| 0x18 | u32 | offset of the piece section | +| 0x1c | u32 | offset of the node section | +| 0x20 | u32 | offset of the child-model section | +| 0x24 | u32 | offset of the animation section | +| 0x28 | u32 | offset of the socket section | +| 0x2c | u32 | file size | +| 0x30 | u32 | 0x10 | +| 0x34 | 14×u32 | counts: keyframes, animations, nodes, pieces, child models (incl. self), triangles, vertices, weights, LODs, sockets, weight sets, strings, string bytes, 0 | +| 0x6c | str16 | command string (`""` literally when empty) | +| | f32 | internal radius | +| | u32 | **hash multiplier** for this file's hashed names (1, 2, 4, 9 seen) | +| | u32 | 0 or 1, unknown (not carried over) | +| | u32 | LOD-distance count (= LODs − 1) | +| | f32[n] | LOD distances | + +Hashed names: `h = 0; for c in name.upper() ('/'→'\\'): h = h + c + h*mult` (u32), the +REZ directory's hash (`ps2rez.rez_hash`). Piece, animation and weight-set names are only +hashes; node names and child-model names are plain strings. Sockets carry two hashes. + +### Pieces (at the piece offset) + +`u32 count`, then per piece: + +| size | field | +|---|---| +| u32 | name hash | +| 12×f32 | three RGBA colours (0.8,0.8,0.8,1 / 0.5,0.5,0.5,0 / 0,0,0,1 everywhere) | +| u32 | texture (skin) index = PC material index | +| 2×u32 | 0 | +| u32 | 4 (always) | +| LOD × count | see below | + +Per LOD: + +| size | field | +|---|---| +| u32 | mesh type: 4 rigid, 5 skeletal (6 vertex-animated: not seen) | +| skeletal: u32 | qwords in the DMA chain below | +| skeletal: u32 | u16 entries in the weight map | +| u32 | rigid: qwords in the DMA chain + 1; skeletal: unique-vertex count | +| u32 | rigid: the node the piece hangs on; skeletal: bone-map length | +| DMA chain | 16-byte DMA tags (qwc = low 16 bits, id = bits 28-30), each followed by qwc qwords; tags with id 1 (CNT) continue, the chain ends after an id 6 (RET) tag's data | + +The VIF code stream starts in the upper 8 bytes of each tag: NOPs, `FLUSH`, then +`UNPACK V4-32` (cmd 0x6c, num = qwords, imm = VU address; data follows inline) and +`MSCALF` (0x15) to run the VU program. Each MSCALF buffer (contiguous VU memory from +address 80) is: + +- header qword: u32 strips − 1, u32 vertices, 0, 0 +- per strip: a GIF tag (NLOOP = vertex count, EOP on the last strip, PRIM = 0x5c triangle + strip/Gouraud/textured/alpha, 3 registers ST,RGBAQ,XYZF2; REGS = 0x412, or 0x8412 when the + strip's first triangle is reversed; upper u32 small integer, unknown) and NLOOP vertices of + 3 qwords: `x y z 1.0`, `nx ny nz 0`, `s t q(=1.0) u32`. + +Strip triangle *k* (k ≥ 2) is (k−2, k−1, k), swapped for odd *k*, and swapped once more for +a 0x8412 strip; the result is PC (clockwise) order. No degenerate triangles occur; the +triangle count equals the header's. + +Rigid vertices are in the space of their node (= PC weight location). Skeletal vertices are +in model space (bind pose); after the chain come: + +| size | field | +|---|---| +| u16 × entries, padded to 16 | weight map: per unique vertex, `u16 n` + n qword offsets (from the chain's first tag) of the strip vertices that are copies of it (their position qwords) | +| 32 × unique | unique vertices: `x y z 1`, `nx ny nz 0` (model space) | +| 4 × bone-map length | bone map: u32 node index per local bone | +| 12 × unique | weights: 4×s16 (4096 = 1.0), 4×u8 local bone × 4 (0xff = unused) | + +### Nodes (at the node offset) + +Per node: `str16 name`, 16×f32 bind matrix (model space), u32 name hash, u32 child count, +u16 index, u16 flags (= PC node flags; PC sets 1 on the root). Then `u32 weight-set count`, +per set: u32 name hash, u32 n, f32[n]. + +### Child models + +`u32 count` (incl. the model itself), then count−1 × `str16` file name (`playerbase.ltb`, +`base_general.ltb`, ...). + +### Animations + +`u32 count`, then per animation: vec3 dims, vec3 translation (the PC anim binding's origin), +u32 name hash, u32 interpolation ms, u32 keyframe count, keyframes (`u32 time ms, str16 +command`), then per node: `u32 0xffffffff` and per keyframe 16 bytes: `s16 x,y,z, s16 flag, +s16 qx,qy,qz,qw` — position / 16 when flag ≠ 0, else / 4096; quaternion / 16384; parent- +relative, as PC. PS2 keeps fewer keyframes than PC for the same animation. + +### Sockets + +`u32 count`, then 48 bytes each: quat rotation, vec3 position, u32 0, u32 node, u32 name +hash (file multiplier), u32 name hash (multiplier 1), u32 0. + +## PC `.abc` v12 + +A chain of sections: `str16 name`, `u32 offset of the next section` (absolute; +0xffffffff on the last), data. Order: Header, Pieces, Nodes, ChildModels, Animation, +Sockets, AnimBindings. + +**Header**: u32 version 12; u32 × 13 counts: keyframes (sum over animations), +animations, nodes, pieces, child models (incl. self), faces, vertices, weights (all LODs), +LODs (≥ 1), sockets, weight sets, strings, string bytes; `str16` command string; f32 +radius; u32 LOD-distance count (the engine reads u16 + 62 skipped bytes); 60 zero bytes; +f32 LOD distances. The engine sizes one allocation from the 13 counts. Strings = the +distinct non-empty (case-sensitive) command string, node names, animation names, keyframe +commands and child-model names; string bytes = their total length (this reproduces every +shipped file). + +**Pieces**: u32 weight count, u32 piece count; per piece: u16 material, f32 specular +power (5.0), f32 specular scale (1.0), f32 LOD weight (1.0), u16 0, `str16` name, then per +LOD: `u32 nfaces`, faces (3 × {f32 u, f32 v, u16 vertex}), `u32 nverts`, vertices +{u16 nweights, u16 sub-LOD index (0xcdcd when there is one LOD), weights {u32 node, vec3 +position in the node's space **pre-multiplied by the bias** (= bias × inverse(bind) × model +position; exact in every shipped file; the engine skins with Σ M_i·(pos_i, bias_i) / Σ bias_i), +f32 bias}, vec3 position +(model space), vec3 normal (model space)}. + +**Nodes**: per node `str16 name, u16 index, u8 flags, 16×f32 bind matrix, u32 child +count` (depth-first order); then `u32 weight-set count`, per set `str16 name, u32 n, +f32[n]`. + +**ChildModels**: `u16 count` (incl. self), then per child `str16 name` (empty for self), +u32 build number, per node vec3 + quat (all identity in every shipped file). The engine +reads the count as u32 (swallowing self's empty name), loads each named child from the +parent's own directory, and refuses a child that has children of its own. + +**Animation**: `u32 count`; per animation vec3 dims, `str16` name, i32 −1, u32 +interpolation ms, u32 keyframe count, keyframes {u32 time, `str16` command}, then per node +per keyframe vec3 position + quat rotation (parent-relative). + +**Sockets**: `u32 count`; per socket u32 node, `str16` name, quat, vec3. + +**AnimBindings**: for the model, then for each child in order: `u32 n`, n × {`str16` +animation name, vec3 dims, vec3 origin} — for a child, a copy of that child file's own +bindings (the engine sets each child animation's origin from it). + +## Conversion (ps2model.py) + +Nodes, sockets, weight sets, animations, LOD distances and radius copy across. The command +string too, except FovXOffset/FovYOffset, which move by (−11°, +9°): every PS2 first-person +model at 31/41 but bucklezip is 20/50 on the PC, with the same geometry and weapon Pos. Rigid LODs: strip vertices de-duplicated by (position, normal) — the PC files +have the same vertex counts; model-space position/normal = bind × node-space. Skeletal +LODs: PC vertex *i* = unique vertex *i*, strip vertices mapped to it through the weight +map; weights normalised to sum 1. Names: `ps2model.collect_names` (every name in the +PC `.abc` files, identifiers in the PS2 executable and the PS2 attribute files; collected +at install time, never committed), +plus the model's own node names; unresolved hashes become `h%08x`. Checked against PC +twins of PS2 models (overcoatagent_action_hat, purses_01, lighter_pv, lighter_hh): same +faces with the same orientation and UVs, same vertices, normals and node-space weight +positions; animations agree with the PC keyframes they share to within the PS2 +quantisation (1/16 unit, 1/16384). + +Child models: the PS2 splits the human animation set into `base_general/_pistol/_rifle/ +_death.ltb`; the PC has `basemodel.abc` (+ `fembase.abc`, `playerbase.abc`) holding the +same animations under the same names. `hero_thief.abc` references the PC player's three +children (PC `player_action.abc` does the same); the PS2 animation-only children are also +converted (`BASE_*.abc`, no pieces) but not referenced. From b6bd700dd89e487ad5352e735051e6fbba5c913b Mon Sep 17 00:00:00 2001 From: nolffan Date: Sat, 24 Oct 2026 00:06:22 +0000 Subject: [PATCH 03/19] PS2: decode the PS2 release's sound banks and voice streams The PS2 keeps a level's sounds in a bank (lps/.lps) and its voice-over in streams (stream/.../vo_*.lps): one container, a header, 72-byte sound records, a tree of groups that gives each sound its PC path, mono PS-ADPCM data, lip-sync bytes and an SPU preload image. spec-sound.md is the format. ps2sound.py decodes them to 16-bit WAV. ps2soundpc.py finds the sounds the PS2-only levels need that no PC archive has -- from the levels' objects, their dialogue IDs and the PS2 attribute files -- and writes sound-map.txt, the list the installer extracts: the thief levels' 74 lines of dialogue (VOICE\14001-14074), their new events and the safe cracker's sounds. Checked against sounds both releases have: the same length, and in step once the PS2's leading silent frame is skipped. Co-Authored-By: Claude Opus 5.5 --- build/ps2/ps2sound.py | 219 ++++++++++++++++++++++++++++++++++++++++ build/ps2/ps2soundpc.py | 165 ++++++++++++++++++++++++++++++ build/ps2/sound-map.txt | 90 +++++++++++++++++ build/ps2/spec-sound.md | 130 ++++++++++++++++++++++++ 4 files changed, 604 insertions(+) create mode 100644 build/ps2/ps2sound.py create mode 100644 build/ps2/ps2soundpc.py create mode 100644 build/ps2/sound-map.txt create mode 100644 build/ps2/spec-sound.md diff --git a/build/ps2/ps2sound.py b/build/ps2/ps2sound.py new file mode 100644 index 00000000..6cbc7a24 --- /dev/null +++ b/build/ps2/ps2sound.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""PS2 No One Lives Forever sound banks ("LithTech PSX Sound Data 1.20", *.lps). + +Written from scratch from the layout documented in spec-sound.md. The same +container holds the per-level effect banks (lps/.lps) and the per-level +voice-over banks (stream/mNN/vo_.lps); the samples are mono PS-ADPCM +(SPU "VAG" frames: 16 bytes -> 28 samples). + +Importable: + import ps2sound + b = ps2sound.load('thiefmission1.lps') + for s in b.sounds: + s.path # PC-style path from the group tree, e.g. 'SND\\EVENT\\STEAM.WAV' + s.name, s.rate, s.loop, s.adpcm (raw bytes) + pcm = ps2sound.decode(s.adpcm) # list of int16 + ps2sound.write_wav(path, pcm, rate) + +Command line: + ps2sound.py list BANK.lps every sound: index, path, rate, size, length + ps2sound.py x BANK.lps OUTDIR [regex] decode to OUTDIR/ as 16-bit PCM WAV + (duplicate paths get a ~N suffix) + ps2sound.py stat FILE.wav... RMS / peak / clipping / zero-crossing check +""" +import os +import re +import struct +import sys + +MAGIC = b'LithTech PSX Sound Data 1.20\r\n\x1a\0' +HDR = 0x34 # magic(32) + 5 x u32 +SOUND_REC = 72 # name[32] + 10 x u32 +GROUP_REC = 48 # name[32] + 4 x u32 + +# PS-ADPCM prediction filters (SPU), in 1/64 units +FILTERS = ((0, 0), (60, 0), (115, -52), (98, -55), (122, -60)) + +FLAG_END, FLAG_LOOP, FLAG_LOOP_START = 1, 2, 4 + + +class Obj: + def __init__(self, **kw): + self.__dict__.update(kw) + + def __repr__(self): + return '%s(%s)' % (type(self).__name__, ', '.join( + '%s=%r' % kv for kv in self.__dict__.items() if kv[0] not in ('adpcm', 'lip', 'spu_image'))) + + +def cstr(b): + return b.split(b'\0', 1)[0].decode('latin-1') + + +def load(path_or_bytes): + d = path_or_bytes if isinstance(path_or_bytes, bytes) else open(path_or_bytes, 'rb').read() + if d[:32] != MAGIC: + raise ValueError('not a LithTech PSX sound bank') + data_size, n_groups, n_sounds, lip_size, spu_image_size = struct.unpack_from('<5I', d, 32) + sounds = [] + for i in range(n_sounds): + o = HDR + i * SOUND_REC + f = struct.unpack_from('<10I', d, o + 32) + sounds.append(Obj(index=i, name=cstr(d[o:o + 32]), flags=f[0], offset=f[1], + size=f[2], rate=f[3], loop_start=f[4], play_size=f[5], + lip_start=f[6], lip_count=f[7], spu_addr=f[8], spu_size=f[9], + loop=bool(f[0] & 2))) + go = HDR + n_sounds * SOUND_REC + groups = [] + for i in range(n_groups): + o = go + i * GROUP_REC + nch, nsnd, first_child, first_sound = struct.unpack_from('<4I', d, o + 32) + groups.append(Obj(index=i, name=cstr(d[o:o + 32]), n_children=nch, + n_sounds=nsnd, first_child=first_child, first_sound=first_sound)) + data = go + n_groups * GROUP_REC + if data + data_size + lip_size + spu_image_size != len(d): + raise ValueError('size mismatch: %d + %d + %d + %d != %d' % ( + data, data_size, lip_size, spu_image_size, len(d))) + for s in sounds: + s.adpcm = d[data + s.offset:data + s.offset + s.size] + # group tree: the root is the last group; children are consecutive + # groups [first_child, first_child + n_children), a leaf's sounds are + # [first_sound, first_sound + n_sounds) + def walk(g, prefix, seen): + if g.index in seen: + raise ValueError('group cycle') + seen.add(g.index) + p = prefix + [g.name] if g.name else prefix + for c in range(g.first_child, g.first_child + g.n_children): + walk(groups[c], p, seen) + for k in range(g.first_sound, g.first_sound + g.n_sounds): + sounds[k].group = '\\'.join(p) + sounds[k].path = '\\'.join(p + [sounds[k].name]) + for s in sounds: + s.group, s.path = '', s.name + if groups: + walk(groups[-1], [], set()) + lip = d[data + data_size:data + data_size + lip_size] + for s in sounds: + s.lip = lip[s.lip_start:s.lip_start + s.lip_count] + img = data + data_size + lip_size + return Obj(data_size=data_size, n_groups=n_groups, n_sounds=n_sounds, lip_size=lip_size, + spu_image_size=spu_image_size, data_pos=data, spu_image_pos=img, + spu_image=d[img:img + spu_image_size], sounds=sounds, groups=groups) + + +def decode(adpcm, stop_at_end=True): + """PS-ADPCM -> list of int16. Stops after the first frame flagged END + (when stop_at_end); a looping sound's END|LOOP frame is decoded once.""" + out = [] + s1 = s2 = 0 + for o in range(0, len(adpcm) - 15, 16): + hdr, flags = adpcm[o], adpcm[o + 1] + shift, filt = hdr & 0x0F, hdr >> 4 + if filt > 4: + filt = 0 # invalid on hardware; treat as raw + if shift > 12: + shift = 9 # SPU behaviour for shift 13..15 + f0, f1 = FILTERS[filt] + if flags == 7: # "void" terminator frame: not played + break + for b in adpcm[o + 2:o + 16]: + for nib in (b & 0x0F, b >> 4): + v = (nib << 12) & 0xFFFF + if v & 0x8000: + v -= 0x10000 + v = (v >> shift) + ((s1 * f0 + s2 * f1 + 32) >> 6) + v = -32768 if v < -32768 else 32767 if v > 32767 else v + out.append(v) + s2, s1 = s1, v + if stop_at_end and (flags & FLAG_END): + break + return out + + +def write_wav(path, pcm, rate): + os.makedirs(os.path.dirname(path) or '.', exist_ok=True) + raw = struct.pack('<%dh' % len(pcm), *pcm) + with open(path, 'wb') as f: + f.write(b'RIFF' + struct.pack('= 32767 or x <= -32768) / n + zc = sum(1 for a, b in zip(pcm, pcm[1:]) if (a < 0) != (b < 0)) / n + de = sum((b - a) ** 2 for a, b in zip(pcm, pcm[1:])) + return Obj(rms=rms, peak=peak, clip=clip, zcr=zc, diff_ratio=de / e) + + +def unique_paths(sounds): + seen = {} + for s in sounds: + k = s.path.upper() + seen[k] = seen.get(k, 0) + 1 + s.out_path = s.path if seen[k] == 1 else '%s~%d%s' % ( + s.path[:-4], seen[k], s.path[-4:]) + + +def main(): + if len(sys.argv) < 3: + sys.exit(__doc__) + cmd = sys.argv[1] + if cmd == 'stat': + for p in sys.argv[2:]: + w = read_wav(p) + st = stats(w.samples) + print('%-50s %5d Hz %6.2fs rms %6.0f peak %5d clip %.4f zcr %.3f d/s %.3f' % ( + p, w.rate, len(w.samples) / w.rate, st.rms, st.peak, st.clip, st.zcr, + st.diff_ratio)) + return + b = load(sys.argv[2]) + unique_paths(b.sounds) + if cmd == 'list': + print('# %d sounds, %d groups, data %d bytes at 0x%x, lip-sync %d bytes, SPU image %d bytes' % ( + b.n_sounds, b.n_groups, b.data_size, b.data_pos, b.lip_size, b.spu_image_size)) + for s in b.sounds: + n = sum(1 for _ in range(0, s.size, 16)) * 28 + print('%4d %-44s %5d Hz %7d B ~%6.2fs %s%s' % ( + s.index, s.out_path, s.rate, s.size, n / s.rate, 'loop ' if s.loop else '', + 'lip %d' % s.lip_count if s.lip_count else '')) + elif cmd == 'x': + out = sys.argv[3] + rx = re.compile(sys.argv[4], re.I) if len(sys.argv) > 4 else None + for s in b.sounds: + if rx and not rx.search(s.out_path): + continue + dst = os.path.join(out, *s.out_path.split('\\')) + write_wav(dst, decode(s.adpcm), s.rate) + print(dst) + else: + sys.exit(__doc__) + + +if __name__ == '__main__': + main() diff --git a/build/ps2/ps2soundpc.py b/build/ps2/ps2soundpc.py new file mode 100644 index 00000000..38b2abf1 --- /dev/null +++ b/build/ps2/ps2soundpc.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Which sounds do the PS2-only worlds need that the PC archives lack, and +extract those from the PS2 sound banks as PC WAV files. + + ps2soundpc.py [--ps2 EXTRACT] [--pc REZDIR] [--out OUTDIR] [--map MAP] + [--worlds thiefmission1,thiefmission2,thiefmission3] [--dry-run] + +EXTRACT the PS2 extraction (default out/tmp/ps2-research/extract): it has + AssetLists/.txt (whose [Sounds] names the world's banks), + the banks themselves (lps/..., stream/...) and Worlds/.ltb +REZDIR the PC game's .rez files (default rez/games/nolf-eu-goty) +OUTDIR where the WAVs go, in the PC layout with upper-case names and a + lower-case extension, e.g. VOICE/14002.wav, SND/EVENT/SCARE.wav + (default out/tmp/ps2-research/sound) +MAP the mapping list written (default build/ps2/sound-map.txt) + +Every sound in a world's banks whose path (from the bank's group tree) is in +no PC archive is extracted; the map also lists the worlds' references (object +string properties: *.wav names, `playsound` commands, CinematicTrigger +DialogueN ids = VOICE\\.wav) and the PS2 attribute files' *.wav names, +and flags world references found nowhere. +""" +import argparse +import os +import re +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(REPO, 'tools')) +import ps2sound # noqa: E402 +import ps2world # noqa: E402 +import rezextract # noqa: E402 + + +def norm(p): + return p.replace('/', '\\').upper() + + +def pc_names(rezdir): + names = {} + for f in sorted(os.listdir(rezdir)): + if f.lower().endswith('.rez'): + for p, _, _ in rezextract.Rez(os.path.join(rezdir, f)).walk(): + names.setdefault(norm(p), f) + return names + + +def world_banks(extract, world): + banks = [] + with open(os.path.join(extract, 'AssetLists', world + '.txt'), encoding='latin-1') as fh: + sec = None + for line in fh: + m = re.match(r'\s*\[(\w+)\]', line) + if m: + sec = m.group(1) + continue + m = re.match(r'\s*(LPSFile\w*)\s*=\s*"([^"]+)"', line) + if sec == 'Sounds' and m: + banks.append(m.group(2)) + return banks + + +def world_refs(path): + """{PATH: [where, ...]} for the sounds a world's objects name.""" + refs = {} + w = ps2world.load(path) + for o in w.objects: + for pr in o.props: + if not isinstance(pr.value, str): + continue + where = '%s %s.%s' % (o.cls, o.name, pr.name) + for m in re.findall(r'[\w\\/&.-]+\.wav', pr.value, re.I): + refs.setdefault(norm(m), []).append(where) + if re.match(r'Dialogue\d+$', pr.name): + for t in re.split(r'[,;\s]+', pr.value.strip()): + if t.isdigit(): + refs.setdefault('VOICE\\%s.WAV' % t, []).append(where) + return refs + + +def attribute_refs(extract): + """{PATH: ['Weapons.txt:5288', ...]} for the .wav names in the PS2 attribute files.""" + refs = {} + d = os.path.join(extract, 'Attributes') + for f in sorted(os.listdir(d)): + if not f.lower().endswith('.txt'): + continue + with open(os.path.join(d, f), encoding='latin-1') as fh: + for n, line in enumerate(fh, 1): + for m in re.findall(r'"([^"]+\.wav)"', line, re.I): + refs.setdefault(norm(m), []).append('%s:%d' % (f, n)) + return refs + + +def out_name(path): + parts = path.split('\\') + base = parts[-1] + if base.upper().endswith('.WAV'): + base = base[:-4].upper() + '.wav' + return [x.upper() for x in parts[:-1]] + [base] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--ps2', default=os.path.join(REPO, 'out/tmp/ps2-research/extract')) + ap.add_argument('--pc', default=os.path.join(REPO, 'rez/games/nolf-eu-goty')) + ap.add_argument('--out', default=os.path.join(REPO, 'out/tmp/ps2-research/sound')) + ap.add_argument('--map', default=os.path.join(HERE, 'sound-map.txt')) + ap.add_argument('--worlds', default='thiefmission1,thiefmission2,thiefmission3') + ap.add_argument('--dry-run', action='store_true') + a = ap.parse_args() + + pc = pc_names(a.pc) + have = {} # PATH -> (bank, sound) first seen + dups = {} # PATH -> [(bank, sound)] further copies + refs = {} # PATH -> [world: where] + for world in a.worlds.split(','): + for bank in world_banks(a.ps2, world): + b = ps2sound.load(os.path.join(a.ps2, bank)) + for s in b.sounds: + k = s.path.upper() + if k in have: + dups.setdefault(k, []).append((bank, s)) + else: + have[k] = (bank, s) + for k, where in world_refs(os.path.join(a.ps2, 'Worlds', world + '.ltb')).items(): + refs.setdefault(k, []).extend('%s: %s' % (world, x) for x in where) + + attrs = attribute_refs(a.ps2) + lines = ['# PC path <- PS2 source (bank #index, rate, ADPCM bytes) ; decoded length ; referenced by', + '# generated by build/ps2/ps2soundpc.py; WAVs in out/tmp/ps2-research/sound/', + '# worlds: ' + a.worlds, ''] + n = 0 + for k in sorted(have): + if k in pc: + continue + bank, s = have[k] + pcm = ps2sound.decode(s.adpcm) + other = [d for d in dups.get(k, []) if d[1].adpcm != s.adpcm or d[1].rate != s.rate] + st = ps2sound.stats(pcm) + r = refs.get(k, []) + ['PS2 Attributes/' + x for x in attrs.get(k, [])] + lines.append('%s <- %s #%d (%d Hz%s, %d B) ; %.2fs rms %.0f ; %s%s' % ( + '/'.join(out_name(k)), bank, s.index, s.rate, ', loop' if s.loop else '', s.size, + len(pcm) / s.rate, st.rms, + '; '.join(sorted(set(r))[:3]) + (' (+%d)' % (len(set(r)) - 3) if len(set(r)) > 3 else '') + if r else 'not referenced by the worlds or attributes (bank only)', + ' ; DIFFERS in ' + ','.join(d[0] for d in other) if other else '')) + if not a.dry_run: + ps2sound.write_wav(os.path.join(a.out, *out_name(k)), pcm, s.rate) + n += 1 + missing = sorted(k for k in refs if k not in pc and k not in have) + lines += ['', '# referenced by the worlds, in neither the PC archives nor the PS2 banks:'] + lines += ['# %s ; %s' % (k, '; '.join(sorted(set(refs[k]))[:3])) for k in missing] + lines += ['', '# %d sounds extracted; %d world references, %d of them on PC, %d from PS2 banks, %d nowhere' % ( + n, len(refs), sum(1 for k in refs if k in pc), + sum(1 for k in refs if k not in pc and k in have), len(missing))] + with open(a.map, 'w') as fh: + fh.write('\n'.join(lines) + '\n') + print('\n'.join(lines[-1:])) + + +if __name__ == '__main__': + main() diff --git a/build/ps2/sound-map.txt b/build/ps2/sound-map.txt new file mode 100644 index 00000000..56ac8ea9 --- /dev/null +++ b/build/ps2/sound-map.txt @@ -0,0 +1,90 @@ +# PC path <- PS2 source (bank #index, rate, ADPCM bytes) ; decoded length ; referenced by +# generated by build/ps2/ps2soundpc.py; WAVs in out/tmp/ps2-research/sound/ +# worlds: thiefmission1,thiefmission2,thiefmission3 + +GUNS/SND/LIPSTICK/RELOAD1.wav <- lps/thiefmission2.lps #38 (12000 Hz, 1552 B) ; 0.22s rms 978 ; PS2 Attributes/Weapons.txt:3940; PS2 Attributes/Weapons.txt:4003; PS2 Attributes/Weapons.txt:4066 (+1) +GUNS/SND/SAFE/FIRE.wav <- lps/thiefmission2.lps #27 (12000 Hz, 3008 B) ; 0.44s rms 1402 ; PS2 Attributes/Weapons.txt:5288 +GUNS/SND/SAFE/SELECT.wav <- lps/thiefmission2.lps #26 (12000 Hz, 4848 B) ; 0.70s rms 1130 ; PS2 Attributes/Weapons.txt:5291 +SND/EVENT/BOTTLECAP.wav <- lps/thiefmission3.lps #12 (12000 Hz, 5056 B) ; 0.73s rms 3931 ; thiefmission3: SoundFX screwcap.Sound +SND/EVENT/BOTTLESGIVEN.wav <- lps/thiefmission3.lps #11 (12000 Hz, 10416 B) ; 1.52s rms 2282 ; thiefmission3: SoundFX bottles.Sound +SND/EVENT/GHOSTDEED.wav <- lps/thiefmission3.lps #7 (10000 Hz, 16480 B) ; 2.88s rms 3619 ; thiefmission3: SoundFX deed.Sound; thiefmission3: SoundFX deed2.Sound +SND/EVENT/SCARE.wav <- lps/thiefmission3.lps #3 (10000 Hz, 19472 B) ; 3.40s rms 6553 ; not referenced by the worlds or attributes (bank only) +VOICE/14001.wav <- lps/thiefmission1.lps #0 (12000 Hz, 40800 B) ; 5.95s rms 5533 ; thiefmission1: CinematicTrigger conv1.Dialogue1 +VOICE/14002.wav <- stream/m03/vo_thiefmission1.lps #29 (12000 Hz, 20096 B) ; 2.93s rms 2876 ; thiefmission1: CinematicTrigger conv2.Dialogue1 +VOICE/14003.wav <- stream/m03/vo_thiefmission1.lps #28 (12000 Hz, 15488 B) ; 2.26s rms 6586 ; thiefmission1: CinematicTrigger conv2.Dialogue2 +VOICE/14004.wav <- stream/m03/vo_thiefmission1.lps #27 (12000 Hz, 27104 B) ; 3.95s rms 4450 ; thiefmission1: CinematicTrigger conv2.Dialogue3 +VOICE/14005.wav <- stream/m03/vo_thiefmission1.lps #26 (12000 Hz, 11504 B) ; 1.68s rms 6808 ; thiefmission1: CinematicTrigger conv2.Dialogue4 +VOICE/14006.wav <- stream/m03/vo_thiefmission1.lps #25 (12000 Hz, 26144 B) ; 3.81s rms 5860 ; thiefmission1: CinematicTrigger conv2.Dialogue5 +VOICE/14007.wav <- stream/m03/vo_thiefmission1.lps #24 (12000 Hz, 53936 B) ; 7.86s rms 6808 ; thiefmission1: CinematicTrigger conv2.Dialogue6 +VOICE/14008.wav <- stream/m03/vo_thiefmission1.lps #23 (12000 Hz, 22336 B) ; 3.25s rms 6564 ; thiefmission1: CinematicTrigger conv2.Dialogue7 +VOICE/14009.wav <- stream/m03/vo_thiefmission1.lps #22 (12000 Hz, 16272 B) ; 2.37s rms 6776 ; thiefmission1: CinematicTrigger conv2.Dialogue8 +VOICE/14010.wav <- stream/m03/vo_thiefmission1.lps #21 (12000 Hz, 44784 B) ; 6.53s rms 4580 ; thiefmission1: CinematicTrigger conv2.Dialogue9 +VOICE/14011.wav <- stream/m03/vo_thiefmission1.lps #20 (12000 Hz, 5616 B) ; 0.82s rms 6652 ; thiefmission1: CinematicTrigger conv2.Dialogue10 +VOICE/14012.wav <- stream/m03/vo_thiefmission1.lps #19 (12000 Hz, 32672 B) ; 4.76s rms 2856 ; thiefmission1: CinematicTrigger conv2.Dialogue11 +VOICE/14013.wav <- stream/m03/vo_thiefmission1.lps #18 (12000 Hz, 16512 B) ; 2.41s rms 7475 ; thiefmission1: CinematicTrigger conv2.Dialogue12 +VOICE/14014.wav <- stream/m03/vo_thiefmission1.lps #17 (12000 Hz, 8960 B) ; 1.30s rms 7583 ; not referenced by the worlds or attributes (bank only) +VOICE/14015.wav <- stream/m03/vo_thiefmission1.lps #16 (12000 Hz, 9280 B) ; 1.35s rms 8116 ; not referenced by the worlds or attributes (bank only) +VOICE/14016.wav <- stream/m03/vo_thiefmission1.lps #15 (12000 Hz, 15568 B) ; 2.27s rms 9791 ; not referenced by the worlds or attributes (bank only) +VOICE/14017.wav <- stream/m03/vo_thiefmission1.lps #14 (12000 Hz, 15888 B) ; 2.31s rms 4918 ; not referenced by the worlds or attributes (bank only) +VOICE/14018.wav <- stream/m03/vo_thiefmission1.lps #13 (12000 Hz, 8080 B) ; 1.18s rms 6022 ; thiefmission1: CinematicTrigger conv7.Dialogue1; thiefmission1: CinematicTrigger conv7.Dialogue10 +VOICE/14019.wav <- stream/m03/vo_thiefmission1.lps #12 (12000 Hz, 10032 B) ; 1.46s rms 6775 ; thiefmission1: CinematicTrigger conv7.Dialogue2 +VOICE/14020.wav <- stream/m03/vo_thiefmission1.lps #11 (12000 Hz, 7008 B) ; 1.02s rms 8906 ; thiefmission1: CinematicTrigger conv7.Dialogue11; thiefmission1: CinematicTrigger conv7.Dialogue5 +VOICE/14021.wav <- stream/m03/vo_thiefmission1.lps #10 (12000 Hz, 51616 B) ; 7.53s rms 4349 ; thiefmission1: CinematicTrigger conv7.Dialogue3 +VOICE/14022.wav <- stream/m03/vo_thiefmission1.lps #9 (12000 Hz, 7360 B) ; 1.07s rms 7850 ; thiefmission1: CinematicTrigger conv7.Dialogue4 +VOICE/14023.wav <- stream/m03/vo_thiefmission1.lps #8 (12000 Hz, 44224 B) ; 6.45s rms 6575 ; thiefmission1: CinematicTrigger conv7.Dialogue6 +VOICE/14024.wav <- stream/m03/vo_thiefmission1.lps #7 (12000 Hz, 52704 B) ; 7.68s rms 6152 ; thiefmission1: CinematicTrigger conv7.Dialogue7 +VOICE/14025.wav <- stream/m03/vo_thiefmission1.lps #6 (12000 Hz, 13136 B) ; 1.91s rms 8488 ; thiefmission1: CinematicTrigger conv7.Dialogue8 +VOICE/14026.wav <- stream/m03/vo_thiefmission1.lps #5 (12000 Hz, 9872 B) ; 1.44s rms 7235 ; thiefmission1: CinematicTrigger conv7.Dialogue9 +VOICE/14027.wav <- stream/m03/vo_thiefmission1.lps #4 (12000 Hz, 36496 B) ; 5.32s rms 3414 ; thiefmission1: CinematicTrigger conv3.Dialogue1; thiefmission1: CinematicTrigger conv4.Dialogue1 +VOICE/14028.wav <- stream/m03/vo_thiefmission1.lps #3 (12000 Hz, 22176 B) ; 3.23s rms 5051 ; thiefmission1: CinematicTrigger conv4.Dialogue2 +VOICE/14029.wav <- stream/m03/vo_thiefmission1.lps #2 (12000 Hz, 30448 B) ; 4.44s rms 6871 ; thiefmission1: CinematicTrigger conv4.Dialogue3 +VOICE/14030.wav <- stream/m03/vo_thiefmission1.lps #1 (12000 Hz, 34432 B) ; 5.02s rms 6382 ; thiefmission1: CinematicTrigger conv4.Dialogue4 +VOICE/14031.wav <- stream/m03/vo_thiefmission1.lps #0 (12000 Hz, 15968 B) ; 2.33s rms 6189 ; thiefmission1: CinematicTrigger conv11.Dialogue1 +VOICE/14032.wav <- stream/m05/vo_thiefmission2.lps #13 (12000 Hz, 98352 B) ; 14.34s rms 5773 ; thiefmission2: CinematicTrigger camkf.Dialogue1 +VOICE/14033.wav <- stream/m05/vo_thiefmission2.lps #12 (12000 Hz, 23904 B) ; 3.48s rms 5738 ; thiefmission2: CinematicTrigger camkf.Dialogue2 +VOICE/14034.wav <- stream/m05/vo_thiefmission2.lps #11 (12000 Hz, 70720 B) ; 10.31s rms 5985 ; thiefmission2: CinematicTrigger camkf.Dialogue3 +VOICE/14035.wav <- stream/m05/vo_thiefmission2.lps #10 (12000 Hz, 35184 B) ; 5.13s rms 4797 ; thiefmission2: CinematicTrigger camkf.Dialogue4 +VOICE/14036.wav <- stream/m05/vo_thiefmission2.lps #9 (12000 Hz, 12224 B) ; 1.78s rms 5873 ; thiefmission2: CinematicTrigger camkf.Dialogue5 +VOICE/14037.wav <- stream/m05/vo_thiefmission2.lps #8 (12000 Hz, 49712 B) ; 7.25s rms 2870 ; thiefmission2: CinematicTrigger camkf.Dialogue6 +VOICE/14038.wav <- stream/m05/vo_thiefmission2.lps #7 (16000 Hz, 18128 B) ; 1.98s rms 5241 ; thiefmission2: CinematicTrigger camkf.Dialogue7 +VOICE/14039.wav <- stream/m05/vo_thiefmission2.lps #6 (12000 Hz, 27744 B) ; 4.04s rms 5758 ; thiefmission2: CinematicTrigger camkf.Dialogue8 +VOICE/14040.wav <- stream/m05/vo_thiefmission2.lps #5 (12000 Hz, 30288 B) ; 4.41s rms 7554 ; thiefmission2: CinematicTrigger conv1.Dialogue1 +VOICE/14041.wav <- stream/m05/vo_thiefmission2.lps #4 (12000 Hz, 17872 B) ; 2.60s rms 5153 ; thiefmission2: CinematicTrigger conv5.Dialogue1 +VOICE/14042.wav <- stream/m05/vo_thiefmission2.lps #3 (12000 Hz, 17872 B) ; 2.60s rms 6214 ; thiefmission2: CinematicTrigger conv3.Dialogue1 +VOICE/14043.wav <- stream/m05/vo_thiefmission2.lps #2 (12000 Hz, 21056 B) ; 3.07s rms 5390 ; thiefmission2: CinematicTrigger conv4.Dialogue1 +VOICE/14044.wav <- stream/m05/vo_thiefmission2.lps #1 (12000 Hz, 19072 B) ; 2.78s rms 6454 ; thiefmission2: CinematicTrigger conv2.Dialogue1 +VOICE/14045.wav <- stream/m05/vo_thiefmission2.lps #0 (12000 Hz, 19616 B) ; 2.86s rms 5504 ; thiefmission2: CinematicTrigger conv6.Dialogue1 +VOICE/14046.wav <- stream/m13/vo_thiefmission3.lps #28 (12000 Hz, 15008 B) ; 2.19s rms 5549 ; thiefmission3: CinematicTrigger camkf.Dialogue1 +VOICE/14047.wav <- stream/m13/vo_thiefmission3.lps #27 (12000 Hz, 23120 B) ; 3.37s rms 4575 ; thiefmission3: CinematicTrigger camkf.Dialogue2 +VOICE/14048.wav <- stream/m13/vo_thiefmission3.lps #26 (12000 Hz, 7840 B) ; 1.14s rms 8500 ; thiefmission3: CinematicTrigger camkf.Dialogue3 +VOICE/14049.wav <- stream/m13/vo_thiefmission3.lps #25 (12000 Hz, 9904 B) ; 1.44s rms 5129 ; thiefmission3: CinematicTrigger camkf.Dialogue4 +VOICE/14050.wav <- stream/m13/vo_thiefmission3.lps #24 (12000 Hz, 34272 B) ; 5.00s rms 6029 ; thiefmission3: CinematicTrigger camkf.Dialogue5 +VOICE/14051.wav <- stream/m13/vo_thiefmission3.lps #23 (12000 Hz, 12928 B) ; 1.88s rms 4389 ; thiefmission3: CinematicTrigger camkf.Dialogue6 +VOICE/14052.wav <- stream/m13/vo_thiefmission3.lps #22 (12000 Hz, 18752 B) ; 2.73s rms 4613 ; thiefmission3: CinematicTrigger camkf.Dialogue7 +VOICE/14053.wav <- stream/m13/vo_thiefmission3.lps #21 (12000 Hz, 6368 B) ; 0.93s rms 8760 ; thiefmission3: CinematicTrigger camkf.Dialogue8 +VOICE/14054.wav <- stream/m13/vo_thiefmission3.lps #20 (12000 Hz, 17392 B) ; 2.53s rms 6870 ; thiefmission3: CinematicTrigger camkf.Dialogue9 +VOICE/14055.wav <- stream/m13/vo_thiefmission3.lps #19 (12000 Hz, 6704 B) ; 0.98s rms 6973 ; thiefmission3: CinematicTrigger camkf.Dialogue10 +VOICE/14056.wav <- stream/m13/vo_thiefmission3.lps #18 (12000 Hz, 6752 B) ; 0.98s rms 8145 ; thiefmission3: CinematicTrigger camkf.Dialogue11 +VOICE/14057.wav <- stream/m13/vo_thiefmission3.lps #17 (12000 Hz, 85376 B) ; 12.45s rms 5398 ; thiefmission3: CinematicTrigger camkf.Dialogue12 +VOICE/14058.wav <- stream/m13/vo_thiefmission3.lps #16 (12000 Hz, 28160 B) ; 4.10s rms 5046 ; thiefmission3: CinematicTrigger camkf.Dialogue13 +VOICE/14059.wav <- stream/m13/vo_thiefmission3.lps #15 (12000 Hz, 15168 B) ; 2.21s rms 3421 ; thiefmission3: CinematicTrigger camkf.Dialogue14 +VOICE/14060.wav <- stream/m13/vo_thiefmission3.lps #14 (12000 Hz, 182096 B) ; 26.55s rms 4638 ; thiefmission3: CinematicTrigger camkf.Dialogue15 +VOICE/14061.wav <- stream/m13/vo_thiefmission3.lps #13 (12000 Hz, 8064 B) ; 1.17s rms 8151 ; thiefmission3: CinematicTrigger camkf.Dialogue16 +VOICE/14062.wav <- stream/m13/vo_thiefmission3.lps #12 (12000 Hz, 53936 B) ; 7.86s rms 5801 ; thiefmission3: CinematicTrigger camkf.Dialogue17 +VOICE/14063.wav <- stream/m13/vo_thiefmission3.lps #11 (12000 Hz, 16592 B) ; 2.42s rms 6165 ; thiefmission3: CinematicTrigger camkf.Dialogue18 +VOICE/14064.wav <- stream/m13/vo_thiefmission3.lps #10 (12000 Hz, 58384 B) ; 8.51s rms 4871 ; thiefmission3: CinematicTrigger camkf.Dialogue19 +VOICE/14065.wav <- stream/m13/vo_thiefmission3.lps #9 (12000 Hz, 19776 B) ; 2.88s rms 6220 ; thiefmission3: CinematicTrigger camkf.Dialogue20 +VOICE/14066.wav <- stream/m13/vo_thiefmission3.lps #8 (12000 Hz, 15008 B) ; 2.19s rms 4196 ; thiefmission3: CinematicTrigger conv1.Dialogue1; thiefmission3: CinematicTrigger conv3.Dialogue1 +VOICE/14067.wav <- stream/m13/vo_thiefmission3.lps #7 (12000 Hz, 22400 B) ; 3.26s rms 6021 ; thiefmission3: CinematicTrigger conv2.Dialogue1 +VOICE/14068.wav <- stream/m13/vo_thiefmission3.lps #6 (12000 Hz, 23120 B) ; 3.37s rms 4403 ; thiefmission3: CinematicTrigger conv2.Dialogue2 +VOICE/14069.wav <- stream/m13/vo_thiefmission3.lps #5 (12000 Hz, 6208 B) ; 0.90s rms 7821 ; thiefmission3: CinematicTrigger conv2.Dialogue3 +VOICE/14070.wav <- stream/m13/vo_thiefmission3.lps #4 (12000 Hz, 36896 B) ; 5.38s rms 4786 ; thiefmission3: CinematicTrigger conv2.Dialogue4 +VOICE/14071.wav <- stream/m13/vo_thiefmission3.lps #3 (12000 Hz, 52336 B) ; 7.63s rms 3991 ; thiefmission3: CinematicTrigger conv2.Dialogue5 +VOICE/14072.wav <- stream/m13/vo_thiefmission3.lps #2 (12000 Hz, 19152 B) ; 2.79s rms 5392 ; thiefmission3: CinematicTrigger conv2.Dialogue6 +VOICE/14073.wav <- stream/m13/vo_thiefmission3.lps #1 (12000 Hz, 55696 B) ; 8.12s rms 5752 ; thiefmission3: CinematicTrigger conv2.Dialogue7 +VOICE/14074.wav <- stream/m13/vo_thiefmission3.lps #0 (12000 Hz, 73440 B) ; 10.71s rms 4187 ; thiefmission3: CinematicTrigger conv2.Dialogue8 + +# referenced by the worlds, in neither the PC archives nor the PS2 banks: +# SND\DOORS\0WOOD_C.WAV ; thiefmission3: Door Door10.CloseSound + +# 81 sounds extracted; 125 world references, 51 of them on PC, 73 from PS2 banks, 1 nowhere diff --git a/build/ps2/spec-sound.md b/build/ps2/spec-sound.md new file mode 100644 index 00000000..54dd682e --- /dev/null +++ b/build/ps2/spec-sound.md @@ -0,0 +1,130 @@ +# PS2 NOLF sound banks (`*.lps`, "LithTech PSX Sound Data 1.20") + +Reader/decoder: `build/ps2/ps2sound.py`. PC-side gap analysis and extraction: +`build/ps2/ps2soundpc.py` (writes `build/ps2/sound-map.txt`). + +## Where they are + +Each world's asset list (`AssetLists/.txt` in LITH_PS2.REZ) names its sound files: + + [Music] Music000 = "stream/m03/nolf_ps2_03.wav" + [Sounds] LPSFile000 = "lps/thiefmission1.lps" + LPSFileStream000 = "stream/m03/vo_thiefmission1.lps" + +- `lps/.lps`: the level's effect bank. It holds every sample the level can + play (doors, events, ambience, weapons, character barks, and sometimes a voice line), + copied in from the shared sources. Banks of different levels share content. +- `stream/mNN/vo_.lps`: the voice-over bank (VOICE\.WAV). It uses the same + container and adds a stored SPU preload image (see below). +- `stream/mNN/nolf_ps2_NN.wav`: music. These are ordinary RIFF PCM files (16-bit mono, + 24000 Hz), not ADPCM. The PC game plays DirectMusic, so they are not used here. + +All values are little-endian. The banks do not use the "VAGp" header. + +## Layout + + 0x00 char[32] "LithTech PSX Sound Data 1.20\r\n\x1a\0" + 0x20 u32 data_size bytes of ADPCM sample data + 0x24 u32 n_groups + 0x28 u32 n_sounds + 0x2c u32 lip_size bytes of lip-sync data (= sum of lip_count) + 0x30 u32 spu_image_size 0 in effect banks; the preload image in VO banks + 0x34 sound[n_sounds] 72 bytes each + group[n_groups] 48 bytes each + u8 data[data_size] sample data; sound offsets are relative to here + u8 lip[lip_size] + u8 spu_image[spu_image_size] + (end of file: the sizes add up exactly) + +### Sound record (72 bytes) + + +0 char[32] name e.g. "LOCKPICK.WAV", "14002.WAV" (upper case, NUL-terminated, + the bytes after the NUL are uninitialised garbage) + +32 u32 flags bit 1 (2): looping sound; no other bits seen + +36 u32 offset into data[]. Sounds are stored in reverse order: index 0 + has the highest offset and the last index has offset 0 + +40 u32 size ADPCM bytes, a multiple of 16 + +44 u32 rate sample rate in Hz (2000..22050; voice is mostly 12000) + +48 u32 loop_start looping only: byte offset of the loop-start frame (0x20 for + a sound that loops whole); 0 for one-shots + +52 u32 play_size one-shot: size-32 (leaves out the leading zero frame and + the pad frame after END); whole loop: size-16; intro+loop+ + tail: the offset of the loop-end frame + +56 u32 lip_start index into lip[] + +60 u32 lip_count lip-sync bytes for this sound (0 if none) + +64 u32 spu_addr where the sound's head goes in SPU RAM (relative to the bank's + SPU area), laid out from the last sound upward + +68 u32 spu_size min(size, 0x3000): the resident head. Longer sounds are + streamed through a 12 KB SPU buffer + (the EXE's CLTPS2SoundEEADPCMStream; inferred, not traced) + +### Group record (48 bytes): the directory tree + + +0 char[32] name "SND", "EVENT", "GUNS", "WALTHERSMG", "S&W", "VOICE", ... + +32 u32 n_children child groups are groups[first_child .. +n_children) + +36 u32 n_sounds sounds are sounds[first_sound .. +n_sounds) + +40 u32 first_child (in a leaf, a copy of its parent's first_child: ignore it) + +44 u32 first_sound (in an inner node, the first sound below it) + +The last group is the root. It has an empty name. A sound's PC path is the chain of +group names from the root down to the sound's leaf, followed by the sound's name. +Example tree (thiefmission1): + + root -> VOICE, SND{SWITCH,EVENT,DOORS,AMB}, GUNS->SND{WALTHERSMG,S&W,P38,...}, + CHARS->SND{PLAYER,MALE,FEMALE,ENGLISH,AMERICAN} + +so CHARS\SND\ENGLISH\COMBAGGR11.WAV and GUNS\SND\P38\FIRE.WAV match the PC paths +case-insensitively. A path can occur more than once across banks (the same sample), +but never twice in one bank. + +### Lip-sync data + +One byte per 50 ms (20 Hz) gives the mouth-open amplitude, roughly 0x01..0x50. For each +voice sound, lip_count / duration is 19.9..20.8. The exception is VOICE\14038 (stored +at 16000 Hz): it gives 18.2, so it may be pitched up on purpose. Only sounds that +characters speak have lip data. The PC game has no use for it. + +### SPU image (VO banks only) + +spu_image[spu_addr : spu_addr+spu_size] is byte-identical to the first spu_size bytes +of each sound's data. Checked for every sound in the three VO banks. It is a +ready-to-DMA copy of the sound heads so a voice line can start at once while the rest +streams from disc. Effect banks give spu_addr/spu_size but do not store the image; the +runtime builds it. + +## Sample encoding: PS-ADPCM (SPU "VAG" frames) + +Mono only. 16-byte frames, each giving 28 samples: + + byte 0 high nibble = filter (0..4), low nibble = shift (0..12) + byte 1 flags: 1 = END, 2 = LOOP (repeat), 4 = LOOP START + bytes 2..15 28 x 4-bit signed samples, low nibble first + + s = (nibble << 12, as int16) >> shift + s += (s1*f0 + s2*f1 + 32) >> 6, (f0,f1) = (0,0) (60,0) (115,-52) (98,-55) (122,-60) + clamp to int16; s2 = s1; s1 = s + +Frame flags as observed (830 sounds): +- One-shot: frame 0 is all zero bytes; every frame has flags 0 except the + second-to-last, which has 1 (END); the last frame is padding after END. +- Whole loop: frame 0 is zero (flags 0), frame 1 has flags 2, frame 2 has 6 (loop start), + then 2 ... 2, and the last frame has 3 (END+LOOP, jump to the loop start). +- Intro + loop + tail (lighter fire/altfire, zipcord): the loop start (6) is at + loop_start and the loop end (3) is at play_size, followed by a tail up to a final 1. + +`ps2sound.decode` decodes up to and including the first frame with the END bit (for a +loop, one pass through the loop). It skips a frame whose flags are 7 and keeps the +leading zero frame (28 silent samples). + +## Validation + +- Every bank parses with no bytes left over. For every sound, offset+size lies inside + data[]. The sounds tile data[] exactly (VO banks: no gaps). +- Compared with the PC originals, resampled to the PS2 rate (SND\DOORS\LOCKPICK, + SND\SWITCH\CALLBUTTON, SND\EVENT\TARGTDONEBELL): the durations are equal to within + 0.01 s and the normalised cross-correlation is 0.57..0.70 at a constant lag of exactly + 28 samples (the leading zero frame). The remaining difference comes from the PS2 + versions' lower sample rate. +- Voice lines decode to speech-like signals (first-difference/signal energy 0.06..1.0; + white noise gives about 2). The 20 Hz lip-sync count agrees with the decoded length. +- No clipping beyond isolated single samples. From ba78a36d3d228ce2d0b07667492c3fd5284fa82d Mon Sep 17 00:00:00 2001 From: nolffan Date: Sat, 24 Oct 2026 00:06:33 +0000 Subject: [PATCH 04/19] PS2: the PS2 levels' text, and the IDs it takes in CRes.dll The PS2 string table (LTSR: a count, ID/length/offset records, Latin-1 strings; five languages in the archive) has the text the PS2-only levels use: 103 strings of dialogue, objectives, mission-failed messages, intel texts, the gadgets' names and the loading-screen line. ps2strings.py decodes it and writes it as a resource script, to be compiled into the modern build's CRes.dll at build time (the text is the disc's, so it is generated, never committed, as the Game of the Year text is). 19 of the PS2's IDs are taken on the PC by other text, so they and the gadgets' move into package P8's range, 31700-31799 (ps2-string-map.txt, IDs only); remap_objects rewrites them in a converted level's commands. The rest keep their PS2 IDs, free on the PC: 5326, 6535-6537 and the dialogue's 14001-14074, which name its voice files. src/modern/cres/p8.h names the moved ones. Co-Authored-By: Claude Opus 5.5 --- build/ps2/ps2-string-map.txt | 58 ++++++++++++ build/ps2/ps2strings.py | 174 +++++++++++++++++++++++++++++++++++ src/modern/cres/p8.h | 42 +++++++++ 3 files changed, 274 insertions(+) create mode 100644 build/ps2/ps2-string-map.txt create mode 100644 build/ps2/ps2strings.py create mode 100644 src/modern/cres/p8.h diff --git a/build/ps2/ps2-string-map.txt b/build/ps2/ps2-string-map.txt new file mode 100644 index 00000000..9e30c704 --- /dev/null +++ b/build/ps2/ps2-string-map.txt @@ -0,0 +1,58 @@ +# PS2 string IDs used by the "Nine years ago" levels (thiefmission1-3) and +# their attribute entries -> the CRes.dll ID the modern build gives them. +# IDs only; the text is the PS2 disc's (English LTSR table 0x056BAF9E in +# LITH_PS2.REZ), which build/ps2/ps2strings.py reads at build time. +# +# ; +# +# Kept as on the PS2 (free in the PC CRes.dll and in the PC VOICE\ files; +# fixed by the converted level data and by VOICE\.wav): +# 5326, 6535-6537, 14001-14074 +# Moved to the P8 range 31700-31799 (the PC CRes.dll already has these IDs, +# or only our own attribute patches reference them): +# 239, 667-669, 4033/4034/4083/4084/4146/4346, 5047/5048/5089/5212-5216, +# 20541-20547 +# Already on the PC with the same text (nothing to add): 6505, 10176 +# (voice only), 10252, 13316. + +# loading screen and intel folder (code) +239 31700 ui ; "Nine years ago..." load-level line for thiefmission* (PC 239 = "Movement Controls") +667 31701 ui ; FolderIntel name "Moneybag %d" (PC 667 = "up"); %d -> %1!d! +668 31702 ui ; FolderIntel name "Handbag %d" (PC 668 = "down") +669 31703 ui ; FolderIntel name "Money %d" (PC 669 = "Missing file %1!s!") + +# WEAPONS.txt (our patch writes the new IDs) +4033 31710 weapon ; [Weapon45] "Thief Safe Cracker" NameId (PC 4033 empty, but Scot Slam's NameId) +4083 31711 weapon ; [Weapon45] DescriptionId +4034 31712 weapon ; [Weapon46] "Thief Lock Pick" NameId +4084 31713 weapon ; [Weapon46] DescriptionId +4146 31714 ammo ; [Ammo54] "Safe Cra Gadget" NameId +4346 31715 ammo ; [Ammo54] DescId + +# objectives (level commands: objective add/completed/remove N, transmission N) +5047 31720 objective ; thiefmission1 (PC 5047 = "") +5048 31721 objective ; thiefmission1 +5089 31722 objective ; thiefmission2 +5212 31723 objective ; thiefmission3 +5213 31724 objective ; thiefmission3 +5214 31725 objective ; thiefmission3 +5215 31726 objective ; thiefmission3 +5216 31727 objective ; thiefmission3 +5326 5326 objective ; thiefmission2 (free on the PC) + +# mission failed (level commands: MissionFailed N) +6535 6535 failed ; thiefmission1-3 (free on the PC) +6536 6536 failed ; thiefmission1-2 +6537 6537 failed ; thiefmission1-3 + +# intelligence text (Intelligence InfoId; PC 20541-20550 = "") +20541 31730 intel ; thiefmission1 handbag +20542 31731 intel ; thiefmission1 handbag +20543 31732 intel ; thiefmission1 briefcase +20544 31733 intel ; unused on the PS2 ("Two Pounds") +20545 31734 intel ; unused on the PS2 ("Three Pounds") +20546 31735 intel ; thiefmission2 money bags +20547 31736 intel ; thiefmission1-3 paper money + +# dialogue (CinematicTrigger DialogueN; subtitles + VOICE\.wav) +14001-14074 = dialogue ; thiefmission1 14001-14031, thiefmission2 14032-14045, thiefmission3 14046-14074; 14014-14017 unused (bank only) diff --git a/build/ps2/ps2strings.py b/build/ps2/ps2strings.py new file mode 100644 index 00000000..48b89549 --- /dev/null +++ b/build/ps2/ps2strings.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""The PS2 string table (LTSR) and the CRes strings the PS2 levels need. + +The PS2 release keeps its text in one archive entry per language (hash +0x056BAF9E in LITH_PS2.REZ, no name in the directory), in place of the PC's +CRes.dll string table. Layout (little endian): + + 0x00 "LTSR" u16 0, u16 1 (version?) u32 count u32 0 + 0x10 count x {u16 id, u16 length (with the NUL), u32 offset (from the file start)} + ... the strings, NUL-terminated, Latin-1 + +The European disc has five copies (directory tables 0/1,6,11,...: English; +2: French; 3: German; 4: Spanish; 5: Italian); the US disc's is English. + + ps2strings.py dump [lang] idtext, every string + ps2strings.py rc [lang] a STRINGTABLE of the mapped strings + ps2strings.py remap ps2idpcid for the IDs that move + ps2strings.py check each mapped string against the PC CRes.dll + + is an LTSR file or LITH_PS2.REZ (then [lang] = en|fr|de|es|it, default en). + is build/ps2/ps2-string-map.txt. +""" +import os, re, struct, sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +LTSR_HASH = 0x056BAF9E +LANG_TABLE = {'en': 0, 'fr': 2, 'de': 3, 'es': 4, 'it': 5} + + +def parse_ltsr(data): + if data[:4] != b'LTSR': + raise ValueError('not an LTSR string table') + n = struct.unpack_from(' the PC's FormatMessage inserts (FormatString) + n = [0] + def ins(m): + n[0] += 1 + return '%%%d!%s!' % (n[0], m.group(1)) + s = re.sub(r'%([ds])', ins, s) + s = s.replace('\\', '\\\\').replace('"', '""').replace('\n', '\\n').replace('\t', '\\t') + return s + + +def write_rc(strings, mapping, out): + lines = ['// Generated by build/ps2/ps2strings.py from the PS2 disc\'s string table', + '// (LITH_PS2.REZ) -- do not edit, not committed. IDs: build/ps2/ps2-string-map.txt.', + '', 'STRINGTABLE DISCARDABLE', 'BEGIN'] + for ps2, pc, kind in sorted(mapping, key=lambda m: m[1]): + if ps2 not in strings: + sys.exit('ps2strings: PS2 string %d missing' % ps2) + lines.append(' %d "%s"' % (pc, rc_text(strings[ps2]))) + lines += ['END', ''] + with open(out, 'w', encoding='latin-1', newline='\r\n') as fh: + fh.write('\n'.join(lines)) + + +def load_remap(path=None): + """{ps2 id: pc id} for the IDs that move (ps2-string-map.txt).""" + path = path or os.path.join(HERE, 'ps2-string-map.txt') + return {ps2: pc for ps2, pc, kind in read_map(path) if ps2 != pc} + + +# the level commands that carry a CRes string ID +CMD_ID = re.compile(r'(?i)(\b(?:objective\s+(?:add|completed|remove)|transmission|missionfailed|missiontext|popup)\s+)(\d+)') + + +def remap_objects(objs, remap): + """Rewrite the string IDs of PS2 world objects (ps2world.Obj, .props of + .name/.value) to the PC's: IDs inside command strings (any string + property), Intelligence InfoId, and DialogueN. Returns the number changed.""" + n = 0 + for o in objs: + for p in o.props: + v = p.value + if isinstance(v, str): + if re.match(r'Dialogue\d+$', p.name) and v.strip().isdigit(): + if int(v) in remap: + p.value = str(remap[int(v)]) + n += 1 + continue + nv = CMD_ID.sub(lambda m: m.group(1) + str(remap.get(int(m.group(2)), int(m.group(2)))), v) + if nv != v: + p.value = nv + n += 1 + elif p.name == 'InfoId' and isinstance(v, float) and int(v) in remap: + p.value = float(remap[int(v)]) + n += 1 + return n + + +def pc_strings(dll): + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(HERE)), 'tools')) + from rsrcdiff import resources + out = {} + for k, v in resources(dll).items(): + if k[0] != 6: + continue + o = 0 + for i in range(16): + n = struct.unpack_from(' 2 else 'en') + for k in sorted(s): + print('%d\t%r' % (k, s[k])) + elif cmd == 'rc': + s = load(a[1], a[4] if len(a) > 4 else 'en') + write_rc(s, read_map(a[2]), a[3]) + elif cmd == 'remap': + for ps2, pc, kind in read_map(a[1]): + if ps2 != pc: + print('%d\t%d\t%s' % (ps2, pc, kind)) + elif cmd == 'check': + s, m, pc = load(a[1]), read_map(a[2]), pc_strings(a[3]) + for ps2, new, kind in m: + print('%5d -> %5d %-9s PC(new id): %-40r PC(ps2 id): %r' % ( + ps2, new, kind, pc.get(new), pc.get(ps2))) + bad = [new for ps2, new, kind in m if new in pc] + print('collisions at the new IDs:', bad or 'none') + else: + sys.exit(__doc__) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/modern/cres/p8.h b/src/modern/cres/p8.h new file mode 100644 index 00000000..ed220da3 --- /dev/null +++ b/src/modern/cres/p8.h @@ -0,0 +1,42 @@ +// p8.h -- the modern build's P8 strings (the PS2 release's levels): +// IDs 31700-31799 (MODERN.md). Used only by the modern build. +// +// The text is the PS2 disc's: build/ps2/ps2strings.py writes it (with the +// PS2 strings that keep their own IDs: 5326, 6535-6537, 14001-14074) into a +// generated, uncommitted .rc at build time, as build/retailextract.py does +// for the Game of the Year strings. Map: build/ps2/ps2-string-map.txt. + +#ifndef __MODERN_CRES_P8_H__ +#define __MODERN_CRES_P8_H__ + +// loading screen of a thiefmission* level (PS2 239) +#define IDS_P8_NINE_YEARS_AGO 31700 + +// FolderIntel names of the new intel types (PS2 667-669; "%1!d!") +#define IDS_P8_INTEL_MONEYBAG 31701 +#define IDS_P8_INTEL_HANDBAG 31702 +#define IDS_P8_INTEL_MONEY 31703 + +// WEAPONS.txt [Weapon45] / [Weapon46] / [Ammo54] (PS2 4033/4083, 4034/4084, 4146/4346) +#define IDS_P8_WEAPON_SAFECRACKER 31710 +#define IDS_P8_WEAPON_SAFECRACKER_DESC 31711 +#define IDS_P8_WEAPON_LOCKPICK 31712 +#define IDS_P8_WEAPON_LOCKPICK_DESC 31713 +#define IDS_P8_AMMO_SAFECRACKER 31714 +#define IDS_P8_AMMO_SAFECRACKER_DESC 31715 + +// objectives (PS2 5047, 5048, 5089, 5212-5216), used by the converted worlds +#define IDS_P8_OBJ_TM1_HANDBAGS 31720 +#define IDS_P8_OBJ_TM1_FLAT 31721 +#define IDS_P8_OBJ_TM2_SAFES 31722 +#define IDS_P8_OBJ_TM3_DIAMOND 31723 +#define IDS_P8_OBJ_TM3_TRASH 31724 +#define IDS_P8_OBJ_TM3_REPLACE 31725 +#define IDS_P8_OBJ_TM3_CASE 31726 +#define IDS_P8_OBJ_TM3_EXIT 31727 + +// intelligence text (PS2 20541-20547), Intelligence InfoId in the converted worlds +#define IDS_P8_INTEL_TEXT_FIRST 31730 +#define IDS_P8_INTEL_TEXT_LAST 31736 + +#endif // __MODERN_CRES_P8_H__ From 6b2915d594713bf9af5a7a45700807fdd6e73ba8 Mon Sep 17 00:00:00 2001 From: nolffan Date: Sat, 24 Oct 2026 00:06:46 +0000 Subject: [PATCH 05/19] Client: the PS2 thief levels' loading screen, hands and intel What the PS2 release's client does for its "Nine years ago" levels, in the modern build (M-P8-04, M-P8-05): - In a thief level (Worlds\thiefmission*) the loading screen shows "Nine years ago...", the scene of the mission it is played in, and the thief photo, and the first-person hands are young Cate's (guns\skins_pv\ThiefHands_pv.dtx). The PS2 knew the levels by mission and level; here they are known by world name, which also covers +runworld. cshell/p8_thief.*, one call each from LoadWorld and the two places the weapon model picks its hands. - The levels' handbags, moneybags and money are three new intel types, with popups ScaleFX261-263; the intel folder lists them. An intel item's type is stored as a byte, so the folder compares against 261-263 as bytes. The release build is unchanged (make progress). Co-Authored-By: Claude Opus 5.5 --- src/game/ClientShellDLL/FolderIntel.cpp | 15 ++++- src/game/ClientShellDLL/GameClientShell.cpp | 11 +++- src/game/ClientShellDLL/WeaponModel.cpp | 25 ++++++++ src/modern/cshell/p8_thief.cpp | 67 +++++++++++++++++++++ src/modern/cshell/p8_thief.h | 21 +++++++ 5 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 src/modern/cshell/p8_thief.cpp create mode 100644 src/modern/cshell/p8_thief.h diff --git a/src/game/ClientShellDLL/FolderIntel.cpp b/src/game/ClientShellDLL/FolderIntel.cpp index 7ef3c19a..80e1a50b 100644 --- a/src/game/ClientShellDLL/FolderIntel.cpp +++ b/src/game/ClientShellDLL/FolderIntel.cpp @@ -8,14 +8,18 @@ #include "ClientRes.h" #include "GameClientShell.h" #include "MsgIds.h" +#ifdef NOLFSRC_MODERN +#include "cres/p8.h" // M-P8-05 +#endif extern CGameClientShell* g_pGameClientShell; namespace { #ifdef NOLFSRC_MODERN // M-P9-05: the Thief mod's handbags, moneybags, money, wages and diamond - // (its FX.txt makes ScaleFX250-254 their popups) - const int kNumSFXTypes = 19; + // (its FX.txt makes ScaleFX250-254 their popups); M-P8-05: the PS2 thief + // levels' moneybags, handbags and money (FX.txt's ScaleFX261-263) + const int kNumSFXTypes = 22; #else const int kNumSFXTypes = 14; #endif @@ -41,6 +45,10 @@ namespace 252, 253, 254, + // an item's type is a byte (CIntelItemMgr::AddItem) + (uint8)261, + (uint8)262, + (uint8)263, #endif }; int nSFXNameIDs[kNumSFXTypes] = @@ -66,6 +74,9 @@ namespace 689, // handbag 688, // moneybag 690, // paper money + IDS_P8_INTEL_MONEYBAG, + IDS_P8_INTEL_HANDBAG, + IDS_P8_INTEL_MONEY, #endif }; int nSFXCounts[kNumSFXTypes]; diff --git a/src/game/ClientShellDLL/GameClientShell.cpp b/src/game/ClientShellDLL/GameClientShell.cpp index 0f9ae3be..9e9fcf74 100644 --- a/src/game/ClientShellDLL/GameClientShell.cpp +++ b/src/game/ClientShellDLL/GameClientShell.cpp @@ -67,6 +67,10 @@ // M-P9: mods made for the Modernizer (the Thief mod) #include "cshell/p9_mods.h" #endif +#ifdef NOLFSRC_MODERN +// M-P8: the PS2 "Nine years ago" levels +#include "cshell/p8_thief.h" +#endif #ifdef STRICT WNDPROC g_pfnMainWndProc = NULL; @@ -6268,8 +6272,11 @@ LTBOOL CGameClientShell::DoLoadWorld(char* pWorldFile, char* pCurWorldSaveFile, } #ifdef NOLFSRC_MODERN - // M-P9-06, M-P9-08: a mod's thief level's loading screen; the world's - // PS2 track + // M-P8-04: a thief level's loading screen and hands + P8_OnLoadWorld(pWorldFile, m_bIsCustomLevel ? -1 : m_nCurrentLevel); + + // M-P9-06, M-P9-08: a mod's thief level's loading screen (over the + // PS2 one, when the mod is there); the world's PS2 track P9_OnLoadWorld(pWorldFile, m_bIsCustomLevel ? -1 : m_nCurrentLevel); #endif diff --git a/src/game/ClientShellDLL/WeaponModel.cpp b/src/game/ClientShellDLL/WeaponModel.cpp index ebb411e8..a9bba8d0 100644 --- a/src/game/ClientShellDLL/WeaponModel.cpp +++ b/src/game/ClientShellDLL/WeaponModel.cpp @@ -32,6 +32,9 @@ #ifdef NOLFSRC_MODERN #include "cshell/p5_gameplay.h" // M-P5-18 #endif +#ifdef NOLFSRC_MODERN +#include "cshell/p8_thief.h" // M-P8-04 +#endif extern CGameClientShell* g_pGameClientShell; extern LTBOOL g_bInfiniteAmmo; @@ -1180,6 +1183,17 @@ void CWeaponModel::CreateModel() } } +#ifdef NOLFSRC_MODERN + // M-P8-04: young Cate's hands in a thief level, as on the PS2 + if (g_pGameClientShell->GetGameType() == SINGLE && P8_GetHandsSkin()) + { + SAFE_STRCPY(createStruct.m_SkinNames[1], P8_GetHandsSkin()); + if (strcmp(m_pWeapon->szPVSkin, "Hands") == 0) + { + SAFE_STRCPY(createStruct.m_SkinNames[0], P8_GetHandsSkin()); + } + } +#endif m_hObject = CreateModelObject(m_hObject, createStruct); if (!m_hObject) return; @@ -4614,6 +4628,17 @@ void CWeaponModel::SetupModel() } } +#ifdef NOLFSRC_MODERN + // M-P8-04: young Cate's hands in a thief level, as on the PS2 + if (g_pGameClientShell->GetGameType() == SINGLE && P8_GetHandsSkin()) + { + SAFE_STRCPY(createStruct.m_SkinNames[1], P8_GetHandsSkin()); + if (strcmp(m_pWeapon->szPVSkin, "Hands") == 0) + { + SAFE_STRCPY(createStruct.m_SkinNames[0], P8_GetHandsSkin()); + } + } +#endif // Set the filenames... diff --git a/src/modern/cshell/p8_thief.cpp b/src/modern/cshell/p8_thief.cpp new file mode 100644 index 00000000..79003091 --- /dev/null +++ b/src/modern/cshell/p8_thief.cpp @@ -0,0 +1,67 @@ +// ----------------------------------------------------------------------- // +// +// MODULE : p8_thief.cpp +// +// PURPOSE : P8's client call-outs (p8_thief.h): the PS2 "Nine years ago" +// levels (NOLFSRC_MODERN only) +// +// ----------------------------------------------------------------------- // +// +// The PS2 release played three flashbacks of young Cate as a thief inside +// missions 3, 5 and 13 (Worlds\thiefmission1-3, converted from the PS2 disc +// by build/ps2/). Its client knew them by mission and level; here they are +// known by world name, which also covers +runworld. + +#include "stdafx.h" +#include "InterfaceMgr.h" +#include "ClientRes.h" +#include "cshell/p8_thief.h" +#include "cres/p8.h" + +static LTBOOL s_bThiefLevel = LTFALSE; + +static LTBOOL P8_IsThiefWorld(const char* pWorldFile) +{ + if (!pWorldFile) return LTFALSE; + + const char* pName = strrchr(pWorldFile, '\\'); + const char* pSlash = strrchr(pWorldFile, '/'); + if (pSlash > pName) pName = pSlash; + pName = pName ? pName + 1 : pWorldFile; + + return _strnicmp(pName, "thiefmission", 12) == 0; +} + +void P8_OnLoadWorld(const char* pWorldFile, int nLevel) +{ + s_bThiefLevel = P8_IsThiefWorld(pWorldFile); + if (!s_bThiefLevel) return; + + // As on the PS2: "Nine years ago...", the scene of the mission it is + // played in, and the thief photo + + HSTRING hTxt = g_pLTClient->FormatString(IDS_P8_NINE_YEARS_AGO); + if (nLevel >= 0) + { + char tmp[128]; + HSTRING hTxt2 = g_pLTClient->FormatString(IDS_SCENENUMBER, nLevel + 1); + _snprintf(tmp, sizeof(tmp), "%s, %s", g_pLTClient->GetStringData(hTxt), g_pLTClient->GetStringData(hTxt2)); + tmp[sizeof(tmp) - 1] = '\0'; + HSTRING hWorld = g_pLTClient->CreateString(tmp); + g_pInterfaceMgr->SetLoadLevelString(hWorld); + g_pLTClient->FreeString(hWorld); + g_pLTClient->FreeString(hTxt2); + } + else + { + g_pInterfaceMgr->SetLoadLevelString(hTxt); + } + g_pLTClient->FreeString(hTxt); + + g_pInterfaceMgr->SetLoadLevelPhoto("interface\\photos\\missions\\thief.pcx"); +} + +const char* P8_GetHandsSkin() +{ + return s_bThiefLevel ? "guns\\skins_pv\\ThiefHands_pv.dtx" : LTNULL; +} diff --git a/src/modern/cshell/p8_thief.h b/src/modern/cshell/p8_thief.h new file mode 100644 index 00000000..85bc12c6 --- /dev/null +++ b/src/modern/cshell/p8_thief.h @@ -0,0 +1,21 @@ +// ----------------------------------------------------------------------- // +// +// MODULE : p8_thief.h +// +// PURPOSE : P8's client call-outs: the PS2 "Nine years ago" levels +// (NOLFSRC_MODERN only) +// +// ----------------------------------------------------------------------- // + +#ifndef __P8_THIEF_H__ +#define __P8_THIEF_H__ + +// LoadWorld, once the mission and level are known (nLevel: -1 for a custom +// level). In a thief level (Worlds\thiefmission*) the loading screen shows +// "Nine years ago..." and the thief photo instead of the mission's (M-P8-04). +void P8_OnLoadWorld(const char* pWorldFile, int nLevel); + +// The first-person hands skin of a thief level (M-P8-04), or LTNULL. +const char* P8_GetHandsSkin(); + +#endif // __P8_THIEF_H__ From d28f001a1f787ceddcd7c7cd918b29c024a8b71d Mon Sep 17 00:00:00 2001 From: nolffan Date: Sat, 24 Oct 2026 00:07:36 +0000 Subject: [PATCH 06/19] Build: make modern PS2=1 adds the PS2 thief levels With PS2=1 and the PS2 disc image (USA or European: the same files) in rez/isos/ps2/, the modern build gets the PS2 release's three "Nine years ago" levels; without it, nothing changes. - build/ps2/ps2install.py writes everything the levels need into the game directory's custom/, from the disc image (its own ISO9660 reader; the PC files from the game's archives), in about 25 seconds: the three worlds (string IDs remapped, a door sound's typo fixed), their PS2-only textures, the models and skins (the gadgets, young Cate, the police hat, the intel popups), 81 sounds, the gadget icons and the loading photo as PCX. It ends with a check that every file the levels and the attribute recipes name is there. - build/ps2/data/ holds the attribute recipes: MISSIONS (the flashbacks inside missions 3, 5 and 13, as on the PS2; the gadgets one-time), WEAPONS (the Thief Safe Cracker and Thief Lock Pick, appended, and the safe cracker's ammo), INTELITEMS and FX (the new intel), MODELBUTES (the "Thief" style), ATTACHMENTS (the police hats). They need the converted content, so build/ps2/ps2data.py applies them on top of src/modern/data's, not tools/moddata.py. - build/all.sh step 2d writes the levels' text into CRes.dll (ps2strings.py, now reading the ISO too); build/variant.sh installs the rest after the modern data; build/ps2/ps2build.sh does both, and takes out what an earlier install wrote (custom/PS2-DATA.txt lists it). - build/package.sh makes no public archive of a game directory with the PS2 levels: they come from the PS2 disc, which the public archive's install scripts cannot read. Co-Authored-By: Claude Opus 5.5 --- Makefile | 3 + build/all.sh | 6 + build/build.mk | 4 +- build/package.sh | 7 + .../ps2/data/ATTRIBUTES/ATTACHMENTS.txt.patch | 23 + build/ps2/data/ATTRIBUTES/FX.txt.patch | 114 +++ .../ps2/data/ATTRIBUTES/INTELITEMS.txt.patch | 51 ++ build/ps2/data/ATTRIBUTES/MISSIONS.txt.patch | 64 ++ .../ps2/data/ATTRIBUTES/MODELBUTES.txt.patch | 13 + build/ps2/data/ATTRIBUTES/WEAPONS.txt.patch | 195 +++++ build/ps2/ps2build.sh | 73 ++ build/ps2/ps2data.py | 92 +++ build/ps2/ps2install.py | 757 ++++++++++++++++++ build/ps2/ps2strings.py | 1 + build/variant.sh | 8 +- 15 files changed, 1409 insertions(+), 2 deletions(-) create mode 100644 build/ps2/data/ATTRIBUTES/ATTACHMENTS.txt.patch create mode 100644 build/ps2/data/ATTRIBUTES/FX.txt.patch create mode 100644 build/ps2/data/ATTRIBUTES/INTELITEMS.txt.patch create mode 100644 build/ps2/data/ATTRIBUTES/MISSIONS.txt.patch create mode 100644 build/ps2/data/ATTRIBUTES/MODELBUTES.txt.patch create mode 100644 build/ps2/data/ATTRIBUTES/WEAPONS.txt.patch create mode 100644 build/ps2/ps2build.sh create mode 100644 build/ps2/ps2data.py create mode 100644 build/ps2/ps2install.py diff --git a/Makefile b/Makefile index 8399c7ac..c0fe5106 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ SRC_MOUNT := $(if $(NOLF_SOURCE),-v $(realpath $(NOLF_SOURCE)):/nolfsource:ro -e DOCKER_RUN = docker run --rm --init --name $(CONTAINER) \ -u $(shell id -u):$(shell id -g) -e HOME=/tmp/home $(if $(JOBS),-e NOLFSRC_JOBS=$(JOBS)) \ $(if $(NOLF_ARCHIVES),-e NOLF_ARCHIVES="$(NOLF_ARCHIVES)") \ + $(if $(PS2),-e NOLFSRC_PS2=1) \ -v $(CURDIR):/repo -w /repo $(SRC_MOUNT) $(IMAGE) TOOLCHAIN_FILES = toolchains/msvc-6.00-9782/VC98/Bin/CL.EXE \ @@ -56,6 +57,8 @@ help: @echo "make debug the game with symbols and NOLFSRC_DEBUG logging, in out/nolf-legacy-debug/" @echo "make modern the game with NOLFSRC_MODERN (the retail bugs fixed), in out/nolf-modern-release/" @echo "make modern-debug both, in out/nolf-modern-debug/" + @echo " (with PS2=1, the modern ones also get the PS2 release's three" + @echo " \"Nine years ago\" levels, from the PS2 disc in rez/isos/ps2/; no public archive)" @echo " (none of these is ever scored; build/variant.sh)" @echo "make test scripted playtest of out/nolf-legacy-release/ ($(TEST_LEVELS)), under Xvfb" @echo "make test-modern the same of out/nolf-modern-release/" diff --git a/build/all.sh b/build/all.sh index 6c991f8a..3c19d7a0 100755 --- a/build/all.sh +++ b/build/all.sh @@ -137,6 +137,12 @@ python3 build/retailextract.py lithtechres "$RETAIL" src/engine/lithtech/res step "2c. Game of the Year text: the retail CRES.dll in $RETAIL -> src/game/ClientRes/goty/" python3 build/retailextract.py goty "$RETAIL" src/game/ClientRes/goty +step "2d. the PS2 levels' text (modern build, make modern PS2=1): the PS2 disc in rez/isos/ps2/ -> src/game/ClientRes/goty/ps2.rc" +case "${NOLFSRC_CFLAGS:-}" in + *NOLFSRC_MODERN*) bash build/ps2/ps2build.sh strings src/game/ClientRes/goty/ps2.rc ;; + *) rm -f src/game/ClientRes/goty/ps2.rc ;; +esac + step "3. compile: libraries, game DLLs, engine (build/compile.sh: build/build.mk, ${NOLFSRC_JOBS:-$(nproc)} jobs)" GAME_CONFIG=final bash build/compile.sh all for f in src/engine/ltmsg/Release/ltmsg.dll src/engine/sres/Release/sres.dll \ diff --git a/build/build.mk b/build/build.mk index 267708ec..85ce18c3 100644 --- a/build/build.mk +++ b/build/build.mk @@ -309,7 +309,9 @@ $(eval $(call compile,$(CRES)/Release/main.obj,$(CRES),main.cpp,$(ENV_GAME),$(CL CRES_RC := $(CRES)/goty/cres_goty.rc # (a printf format: \\ is one backslash) hash := \# -CRES_RC_FMT := $(hash)include "..\\ClientRes.rc"\r\n$(hash)include "goty.rc"\r\n$(foreach r,$(if $(MODERN),$(notdir $(wildcard src/modern/cres/*.rc))),$(hash)include "..\\..\\..\\modern\\cres\\$(r)"\r\n) +# The modern build adds the PS2 levels' text (goty/ps2.rc) when build/all.sh +# made it from the PS2 disc (MODERN.md, P8). +CRES_RC_FMT := $(hash)include "..\\ClientRes.rc"\r\n$(hash)include "goty.rc"\r\n$(foreach r,$(if $(MODERN),$(notdir $(wildcard src/modern/cres/*.rc))),$(hash)include "..\\..\\..\\modern\\cres\\$(r)"\r\n)$(if $(MODERN),$(if $(wildcard $(CRES)/goty/ps2.rc),$(hash)include "ps2.rc"\r\n)) $(CRES_RC): FORCE @test -f $(CRES)/goty/goty.rc -a -f $(CRES)/goty/credits.txt || \ { echo "error: $(CRES)/goty/ is missing -- build/all.sh writes it" >&2; exit 1; } diff --git a/build/package.sh b/build/package.sh index d445cb36..b17353a5 100755 --- a/build/package.sh +++ b/build/package.sh @@ -66,6 +66,13 @@ if [[ $ARCHIVES == *" private "* ]]; then echo " $PKG: $(du -h "$PKG" | cut -f1) (unpacks to $NAME/)" fi [[ $ARCHIVES == *" public "* ]] || exit 0 +# Nor of a game directory with the PS2 levels (make modern PS2=1): they are +# made from the PS2 disc, which the install scripts do not read. +if [ -f "$OUT/custom/PS2-DATA.txt" ]; then + echo " $PARENT/$NAME-public.tar.gz: skipped (the game directory has the PS2 levels, custom/PS2-DATA.txt)" + rm -f "$PARENT/$NAME-public.tar.gz" + exit 0 +fi # The public archive: the scripts beside the rest, the disc files left out. PUB="$PARENT/$NAME-public.tar.gz" diff --git a/build/ps2/data/ATTRIBUTES/ATTACHMENTS.txt.patch b/build/ps2/data/ATTRIBUTES/ATTACHMENTS.txt.patch new file mode 100644 index 00000000..e4191c27 --- /dev/null +++ b/build/ps2/data/ATTRIBUTES/ATTACHMENTS.txt.patch @@ -0,0 +1,23 @@ +--- a/ATTRIBUTES/ATTACHMENTS.txt ++++ b/ATTRIBUTES/ATTACHMENTS.txt +@@ -555,6 +555,20 @@ + Type = 0 + Model = "attachments\models\rag.abc" + Skin = "attachments\skins\rag.dtx" ++ ++[Attachment74] ++ ++Name = "Britishpolicehat" ++Type = 0 ++Model = "Attachments\Models\BritishPoliceHat.abc" ++Skin = "Attachments\Skins\MaleBystander_Action_BritHat.dtx" ++ ++[Attachment75] ++ ++Name = "ScottishpoliceHat" ++Type = 0 ++Model = "Attachments\Models\BritishPoliceHat.abc" ++Skin = "Attachments\Skins\MaleBystander_Action_ScotHat.dtx" + + + /************************************************************************* diff --git a/build/ps2/data/ATTRIBUTES/FX.txt.patch b/build/ps2/data/ATTRIBUTES/FX.txt.patch new file mode 100644 index 00000000..123f3eba --- /dev/null +++ b/build/ps2/data/ATTRIBUTES/FX.txt.patch @@ -0,0 +1,114 @@ +--- a/ATTRIBUTES/FX.txt ++++ b/ATTRIBUTES/FX.txt +@@ -12575,6 +12575,111 @@ + MinRotVel = 0 + MaxRotVel = 0 + ++/* ++** ScaleFX261 - Popup MoneyBag model (PS2 ScaleFX262) ++*/ ++[ScaleFX261] ++ ++Name = "MoneyBagPopup" ++Type = 0 ++File = "Interface\Models\MoneyBag.abc" ++Skin = "Props\Skins\MoneyBag.dtx" ++InitialScale = <0.0134, 0.0134, 0.0134> ++FinalScale = <0.0134, 0.0134, 0.0134> ++UseColors = 0 ++InitialColor = <128.0, 128.0, 128.0> ++FinalColor = <128.0, 128.0, 128.0> ++InitialAlpha = 1.0 ++FinalAlpha = 1.0 ++DirOffset = 1 ++DirROffset = 0 ++DirUOffset = 0.20 ++Velocity = <0.000000, 0.000000, 0.000000> ++LifeTime = 1000000.00000 ++DelayTime = 0.0 ++Loop = 1 ++AlignToSurface = 0 ++NoZ = 0 ++ReallyClose = 1 ++Additive = 0 ++Multiply = 0 ++Chromakey = 0 ++Rotate = 0 ++FaceCamera = 0 ++RotationAxis = 1 ++MinRotVel = 1.0 ++MaxRotVel = 1.0 ++ ++/* ++** ScaleFX262 - Popup HandBag model (PS2 ScaleFX263) ++*/ ++[ScaleFX262] ++ ++Name = "HandBagPopup" ++Type = 0 ++File = "Interface\Models\Purses_01.abc" ++Skin = "Props\Skins\Purses_01.dtx" ++InitialScale = <0.0111, 0.0111, 0.0111> ++FinalScale = <0.0111, 0.0111, 0.0111> ++UseColors = 0 ++InitialColor = <128.0, 128.0, 128.0> ++FinalColor = <128.0, 128.0, 128.0> ++InitialAlpha = 1.0 ++FinalAlpha = 1.0 ++DirOffset = 1 ++DirROffset = 0 ++DirUOffset = 0.20 ++Velocity = <0.000000, 0.000000, 0.000000> ++LifeTime = 1000000.00000 ++DelayTime = 0.0 ++Loop = 1 ++AlignToSurface = 0 ++NoZ = 0 ++ReallyClose = 1 ++Additive = 0 ++Multiply = 0 ++Chromakey = 0 ++Rotate = 0 ++FaceCamera = 0 ++RotationAxis = 1 ++MinRotVel = 1.0 ++MaxRotVel = 1.0 ++ ++/* ++** ScaleFX263 - Popup Paper Money model (PS2 ScaleFX264) ++*/ ++[ScaleFX263] ++ ++Name = "PaperMoneyPopup" ++Type = 0 ++File = "Interface\Models\Change_01.abc" ++Skin = "Interface\Skins\Change_01.dtx" ++InitialScale = <0.0371, 0.0371, 0.0371> ++FinalScale = <0.0371, 0.0371, 0.0371> ++UseColors = 0 ++InitialColor = <128.0, 128.0, 128.0> ++FinalColor = <128.0, 128.0, 128.0> ++InitialAlpha = 1.0 ++FinalAlpha = 1.0 ++DirOffset = 1 ++DirROffset = 0 ++DirUOffset = 0.20 ++Velocity = <0.000000, 0.000000, 0.000000> ++LifeTime = 1000000.00000 ++DelayTime = 0.0 ++Loop = 1 ++AlignToSurface = 0 ++NoZ = 0 ++ReallyClose = 1 ++Additive = 0 ++Multiply = 0 ++Chromakey = 0 ++Rotate = 0 ++FaceCamera = 0 ++RotationAxis = 1 ++MinRotVel = 1.0 ++MaxRotVel = 1.0 ++ + /************************************************************************ + ** + ** PSHOWERFX diff --git a/build/ps2/data/ATTRIBUTES/INTELITEMS.txt.patch b/build/ps2/data/ATTRIBUTES/INTELITEMS.txt.patch new file mode 100644 index 00000000..0da770a9 --- /dev/null +++ b/build/ps2/data/ATTRIBUTES/INTELITEMS.txt.patch @@ -0,0 +1,51 @@ +--- a/ATTRIBUTES/INTELITEMS.txt ++++ b/ATTRIBUTES/INTELITEMS.txt +@@ -356,3 +356,48 @@ + ScaleName0 = "PopupBackground" + ScaleName1 = "DimsOnlyPopup" + ++/* ++** Intel19 - HandBag (PS2 "Nine years ago" levels) ++*/ ++ ++[Intel19] ++ ++Name = "HandBag" ++Filename = "Props\Models\Purses_01.abc" ++Skin = "Props\Skins\Purses_01.dtx" ++DefaultTextId = 8019 ++Chrome = 0 ++Chromakey = 0 ++ScaleName0 = "PopupBackground" ++ScaleName1 = "HandBagPopup" ++ ++/* ++** Intel20 - Money Bag (PS2 "Nine years ago" levels) ++*/ ++ ++[Intel20] ++ ++Name = "MoneyBag" ++Filename = "Props\Models\MoneyBag.abc" ++Skin = "Props\Skins\MoneyBag.dtx" ++DefaultTextId = 8020 ++Chrome = 0 ++Chromakey = 0 ++ScaleName0 = "PopupBackground" ++ScaleName1 = "MoneyBagPopup" ++ ++/* ++** Intel21 - Paper Money (PS2 "Nine years ago" levels) ++*/ ++ ++[Intel21] ++ ++Name = "Paper Money" ++Filename = "Props\Models\Change_01.abc" ++Skin = "Props\Skins\Change_01b.dtx" ++DefaultTextId = 8021 ++Chrome = 0 ++Chromakey = 0 ++ScaleName0 = "PopupBackground" ++ScaleName1 = "PaperMoneyPopup" ++ diff --git a/build/ps2/data/ATTRIBUTES/MISSIONS.txt.patch b/build/ps2/data/ATTRIBUTES/MISSIONS.txt.patch new file mode 100644 index 00000000..50ca0031 --- /dev/null +++ b/build/ps2/data/ATTRIBUTES/MISSIONS.txt.patch @@ -0,0 +1,64 @@ +--- a/ATTRIBUTES/MISSIONS.txt ++++ b/ATTRIBUTES/MISSIONS.txt +@@ -678,7 +678,8 @@ + Rank5 = "P 1" + Rank6 = "D 1" + Level0 = "Worlds\m03s01" +-Level1 = "Worlds\m03s02" ++Level1 = "Worlds\thiefmission1" ++Level2 = "Worlds\m03s02" + + [Mission5] + // Care to Explain? +@@ -824,11 +825,11 @@ + DefaultGadgets = "Coin,Barrette" + AllowedGadgets = "" + RequiredGadgets = "Sunglasses,Lighter" +-OneTimeGadgets = "" ++OneTimeGadgets = "Thief Lock Pick,Thief Safe Cracker" + DeniedGadgets = "Space Code Breaker" + AllowedAmmo = "BarretteLockpick,BarrettePoison," + RequiredAmmo = "Camera,Lighter,Welder" +-OneTimeAmmo = "" ++OneTimeAmmo = "Safe Cra Gadget" + DeniedAmmo = "" + AllowedMods = "Delisle Scope" + DefaultMods = "P38 Silencer,Delisle Scope" +@@ -854,7 +855,8 @@ + Level1 = "Worlds\m05s02" + Level2 = "Worlds\m05s03" + Level3 = "Worlds\m05s04" +-Level4 = "Worlds\m05s05" ++Level4 = "Worlds\thiefmission2" ++Level5 = "Worlds\m05s05" + + [Mission9] + // H.A.R.M.'s Dire Promise +@@ -1443,11 +1445,11 @@ + DefaultGadgets = "Coin,Lighter,Barrette,Decay Powder,Zipcord" + AllowedGadgets = "" + RequiredGadgets = "" +-OneTimeGadgets = "" ++OneTimeGadgets = "Thief Lock Pick,Thief Safe Cracker" + DeniedGadgets = "Space Code Breaker" + AllowedAmmo = "Lighter,Welder" + RequiredAmmo = "" +-OneTimeAmmo = "" ++OneTimeAmmo = "Safe Cra Gadget" + DeniedAmmo = "" + AllowedMods = "AK47 Scope,Dragunov Nightvision Scope,Walther SMG Silencer" + DefaultMods = "P38 Silencer" +@@ -1470,9 +1472,10 @@ + Rank5 = "S 1" + Rank6 = "D 1" + Level0 = "Worlds\m13s01" +-Level1 = "Worlds\m13s02" +-Level2 = "Worlds\m13s03" +-Level3 = "Worlds\m13s04" ++Level1 = "Worlds\thiefmission3" ++Level2 = "Worlds\m13s02" ++Level3 = "Worlds\m13s03" ++Level4 = "Worlds\m13s04" + + [Mission23] + // Escape - Mission 14 diff --git a/build/ps2/data/ATTRIBUTES/MODELBUTES.txt.patch b/build/ps2/data/ATTRIBUTES/MODELBUTES.txt.patch new file mode 100644 index 00000000..9cd03c0d --- /dev/null +++ b/build/ps2/data/ATTRIBUTES/MODELBUTES.txt.patch @@ -0,0 +1,13 @@ +--- a/ATTRIBUTES/MODELBUTES.txt ++++ b/ATTRIBUTES/MODELBUTES.txt +@@ -653,6 +653,10 @@ + [Style16] + + Name = "Style8" ++ ++[Style17] ++ ++Name = "Thief" + + /* + ** Skeleton [node group] Attributes diff --git a/build/ps2/data/ATTRIBUTES/WEAPONS.txt.patch b/build/ps2/data/ATTRIBUTES/WEAPONS.txt.patch new file mode 100644 index 00000000..2b45dab1 --- /dev/null +++ b/build/ps2/data/ATTRIBUTES/WEAPONS.txt.patch @@ -0,0 +1,195 @@ +--- a/ATTRIBUTES/WEAPONS.txt ++++ b/ATTRIBUTES/WEAPONS.txt +@@ -102,19 +102,21 @@ + Weapon33 = "Space Fists" + Weapon34 = "Space Atomic Lasergun" + Weapon35 = "Space Code Breaker" +-LastPlayerWeapon = 35 ++Weapon36 = "Thief Lock Pick" ++Weapon37 = "Thief Safe Cracker" ++LastPlayerWeapon = 37 + + // Non player weapons (must come after the player weapons)... + // +-Weapon36 = "Heli_Machinegun" +-Weapon37 = "Helicopter RL" +-Weapon38 = "Scot Punch" +-Weapon39 = "Scot Slam" +-Weapon40 = "German Sword" +-Weapon41 = "Shark Bite" +-Weapon42 = "Hand Grenade" +-Weapon43 = "Dynamite" +-Weapon44 = "Super Shark Bite" ++Weapon38 = "Heli_Machinegun" ++Weapon39 = "Helicopter RL" ++Weapon40 = "Scot Punch" ++Weapon41 = "Scot Slam" ++Weapon42 = "German Sword" ++Weapon43 = "Shark Bite" ++Weapon44 = "Hand Grenade" ++Weapon45 = "Dynamite" ++Weapon46 = "Super Shark Bite" + + /************************************************************************ + ** +@@ -2111,6 +2113,34 @@ + FireFXName = "" + TracerFXName = "" + WeaponAniOverride = "" ++ ++/* ++** Ammo54 - Thief Safe Cracker gadget ammo (PS2 Ammo55) ++*/ ++ ++[Ammo54] ++ ++Name = "Safe Cra Gadget" ++NameId = 31714 ++DescId = 31715 ++Type = 3 ++Priority = 54 ++Icon = "Interface\Photos\Ammo\Func_Cracker.pcx" ++SmallIcon = "Interface\Photos\Ammo\Func_Cracker_SM.pcx" ++MaxAmount = 1 ++SelectionAmount = 1 ++SpawnedAmount = 1 ++InstDamage = 0 ++InstDamageType = 22 ++AreaDamage = 0 ++AreaDamageType = 0 ++AreaDamageRadius = 0 ++ProgDamage = 0.000000 ++ProgDamageType = 0 ++ProgDamageDuration = 0.000000 ++ProgDamageRadius = 0.000000 ++ProgDamageLifetime = 0.000000 ++FireRecoilMult = 1.000000 + + + /************************************************************************ +@@ -5208,6 +5238,125 @@ + HideWhenEmpty = 0 + + ++/* ++** Thief Safe Cracker (PS2 "Nine years ago" levels; PS2 Weapon45) ++*/ ++ ++[Weapon45] ++ ++Name = "Thief Safe Cracker" ++NameId = 31710 ++DescriptionId = 31711 ++Gadget = 1 ++AniType = 2 ++Icon = "Interface\Photos\Weapons\PS2SC.pcx" ++SmallIcon = "Interface\Photos\Weapons\PS2SC_SM.pcx" ++Photo = "Interface\Photos\Weapons\PS2SC.pcx" ++Pos = <0.100000, -0.800000, 3.000000> ++MuzzlePos = <2.100000, 1.700000, 17.400000> ++MinPerturb = 0 ++MaxPerturb = 0 ++Range = 100 ++Recoil = <0.000000, 0.000000, 0.000000> ++VectorsPerRound = 1 ++PVModel = "Guns\Models_PV\PS2SafeCracker_pv.abc" ++PVSkin = "Guns\Skins_PV\PS2SafeCracker_pv.dtx" ++PVMuzzleFXName = "" ++HHModel = "Guns\Models_HH\PS2SafeCracker_hh.abc" ++HHSkin = "Guns\Skins_PV\PS2SafeCracker_pv.dtx" ++HHModelScale = <1.000000, 1.000000, 1.000000> ++HHBreachOffset = -10.000000 ++HHMuzzleFXName = "" ++InterfaceModel = "Guns\Models_HH\PS2SafeCracker_hh.abc" ++InterfaceSkin = "Guns\Skins_PV\PS2SafeCracker_pv.dtx" ++InterfaceModelScale = 1.000000 ++InterfaceOffset = <0.000000, 0.000000, 0.000000> ++SilencedFireSnd = "" ++AltFireSnd = "" ++FireSnd = "Guns\Snd\safe\fire.wav" ++DryFireSnd = "" ++ReloadSnd1 = "" ++SelectSnd = "Guns\Snd\safe\select.wav" ++DeselectSnd = "" ++FireSndRadius = 128 ++AIFireSndRadius = 64 ++EnvironmentMap = 1 ++InfiniteAmmo = 0 ++ShotsPerClip = 1 ++AmmoMultiplier = 1 ++FireRecoilPitch = 0.000000 ++FireRecoilDecay = 0.000000 ++AmmoName0 = "Safe Cra Gadget" ++AIMinBurstInterval = 0.500000 ++AIMaxBurstInterval = 1.500000 ++AIMinBurstShots = 1 ++AIMaxBurstShots = 5 ++LooksDangerous = 0 ++CanBeDefault = 0 ++CanBeMapped = 1 ++FireDelay = 810 ++HideWhenEmpty = 1 ++ ++ ++/* ++** Thief Lock Pick (PS2 "Nine years ago" levels; PS2 Weapon47) ++*/ ++ ++[Weapon46] ++ ++Name = "Thief Lock Pick" ++NameId = 31712 ++DescriptionId = 31713 ++Gadget = 1 ++AniType = 2 ++Icon = "Interface\Photos\Weapons\PS2lp.pcx" ++SmallIcon = "Interface\Photos\Weapons\PS2lp_SM.pcx" ++Photo = "Interface\Photos\Weapons\PS2lp.pcx" ++Pos = <0.100000, -0.300000, 1.750000> ++MuzzlePos = <1.000000, -1.100000, 13.800000> ++MinPerturb = 0 ++MaxPerturb = 5 ++Range = 100 ++Recoil = <0.000000, 0.000000, 0.000000> ++VectorsPerRound = 1 ++PVModel = "Guns\Models_PV\PS2lockpick_pv.abc" ++PVSkin = "Guns\Skins_PV\PS2lockpick_pv.dtx" ++PVMuzzleFXName = "" ++HHModel = "Guns\Models_HH\PS2lockpick_hh.abc" ++HHSkin = "Guns\Skins_PV\PS2lockpick_pv.dtx" ++HHModelScale = <1.000000, 1.000000, 1.000000> ++HHBreachOffset = 0.000000 ++HHMuzzleFXName = "" ++InterfaceModel = "Guns\Models_HH\barret_hh.abc" ++InterfaceSkin = "Guns\Skins_PV\PS2lockpick_pv.dtx" ++InterfaceModelScale = 3.500000 ++InterfaceOffset = <0.000000, 0.000000, 0.000000> ++SilencedFireSnd = "" ++FireSnd = "Guns\Snd\lipstick\reload1.wav" ++DryFireSnd = "" ++ReloadSnd1 = "" ++SelectSnd = "Guns\Snd\barrette\select.wav" ++DeselectSnd = "Guns\Snd\barrette\deselect.wav" ++FireSndRadius = 128 ++AIFireSndRadius = 0 ++EnvironmentMap = 1 ++InfiniteAmmo = 1 ++ShotsPerClip = 1 ++AmmoMultiplier = 1 ++FireRecoilPitch = 0.100000 ++FireRecoilDecay = 0.175000 ++AmmoName0 = "BarretteLockpick" ++AIMinBurstInterval = 0.500000 ++AIMaxBurstInterval = 1.500000 ++AIMinBurstShots = 1 ++AIMaxBurstShots = 5 ++LooksDangerous = 0 ++CanBeDefault = 0 ++CanBeMapped = 1 ++FireDelay = 800 ++HideWhenEmpty = 0 ++ ++ + /************************************************************************ + ** + ** MOD diff --git a/build/ps2/ps2build.sh b/build/ps2/ps2build.sh new file mode 100644 index 00000000..eae0e5f2 --- /dev/null +++ b/build/ps2/ps2build.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# The modern build's PS2 levels (MODERN.md, P8): the three "Nine years ago" +# flashbacks of the PlayStation 2 release, converted from the PS2 disc. +# Opt-in: `make modern PS2=1` sets NOLFSRC_PS2, and then the PS2 disc image +# must be in rez/isos/ps2/ (either the USA or the European disc: they give +# the same files). +# +# build/ps2/ps2build.sh strings the levels' text, for CRes.dll +# (build/all.sh, step 2d) +# build/ps2/ps2build.sh install the converted levels, models, +# textures, sounds and images, and +# the attribute edits, into its +# custom/ (build/variant.sh) +# build/ps2/ps2build.sh clean take out what an install wrote +# (build/variant.sh, before the +# modern data: an install's +# attribute files are on top of +# those) +# +# Without NOLFSRC_PS2, `strings` and `install` do nothing, so a build without +# it has none of the PS2 content. Everything an +# install writes is listed in custom/PS2-DATA.txt; build/package.sh makes no +# public archive of a game directory that has it, since that data comes from +# the PS2 disc, which the public archive's install scripts cannot read. +set -e +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" + +disc() { + local f + for f in "$REPO"/rez/isos/ps2/*.iso "$REPO"/rez/isos/ps2/*.ISO; do + [ -f "$f" ] && { echo "$f"; return 0; } + done + echo "error: NOLFSRC_PS2 is set, but there is no PS2 disc image (.iso) in rez/isos/ps2/" >&2 + return 1 +} + +clean() { + local custom="$1/custom" list + list="$custom/PS2-DATA.txt" + [ -f "$list" ] || return 0 + while IFS= read -r f; do + [ -n "$f" ] && rm -f "$custom/$f" + done < "$list" + rm -f "$list" + echo " removed the PS2 levels' files from $1/custom/" +} + +case "${1:-}" in +strings) + out="${2:?usage: ps2build.sh strings }" + rm -f "$out" + [ -n "${NOLFSRC_PS2:-}" ] || { echo " (no PS2 levels: make modern PS2=1 adds them)"; exit 0; } + iso="$(disc)" + python3 "$HERE/ps2strings.py" rc "$iso" "$HERE/ps2-string-map.txt" "$out" + echo " $(basename "$iso") -> $out" + ;; +install) + game="${2:?usage: ps2build.sh install }" + [ -n "${NOLFSRC_PS2:-}" ] || exit 0 + iso="$(disc)" + python3 "$HERE/ps2install.py" --ps2 "$iso" --game "$game" + python3 "$HERE/ps2data.py" "$game" | tee /dev/stderr | + sed -n 's|^ *custom/\([^ ]*\) <- .*|\1|p' >> "$game/custom/PS2-DATA.txt" + ;; +clean) + clean "${2:?usage: ps2build.sh clean }" + ;; +*) + sed -n '2,24p' "$0" >&2 + exit 2 + ;; +esac diff --git a/build/ps2/ps2data.py b/build/ps2/ps2data.py new file mode 100644 index 00000000..ff5029f4 --- /dev/null +++ b/build/ps2/ps2data.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""The PS2 "Nine years ago" levels' attribute edits (MODERN.md, P8). + + ps2data.py + +The recipes in build/ps2/data/ (unified diffs, as src/modern/data/'s) only +make sense with the converted PS2 content installed (build/ps2/ps2install.py), +so they are applied here rather than by tools/moddata.py: each one on top of +the file as the modern build leaves it -- the retail file the game loads, +with src/modern/data/'s own recipe for it, if any, applied first -- and +written into the game directory's custom/. Running it again gives the same +result. A `make modern` rewrites those files without the PS2 edits; run this +again afterwards. +""" +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, os.path.join(REPO, "tools")) +import moddata # noqa: E402 + +DATA = os.path.join(HERE, "data") +MODERN = os.path.join(REPO, "src", "modern", "data") + + +def rez_stack(gamedir): + """moddata.rez_stack, but matching the archive names whatever their + case: the retail launcher writes nolfcmds.txt with NOLF.rez, NOLF2.rez, + which on Linux are not the files nolf.rez, nolf2.rez.""" + names = {n.lower(): n for n in os.listdir(gamedir)} + cmds = open(os.path.join(gamedir, "nolfcmds.txt"), errors="replace").read() + stack = [] + for name in moddata.re.findall(r"-rez\s+(\S+)", cmds): + real = names.get(name.lower()) + if real and real.lower().endswith(".rez") and os.path.isfile(os.path.join(gamedir, real)): + stack.append(os.path.join(gamedir, real)) + return stack + + +moddata.rez_stack = rez_stack + + +def patches(): + for root, _, fs in os.walk(DATA): + for f in sorted(fs): + if f.endswith(".patch"): + p = os.path.join(root, f) + yield os.path.relpath(p, DATA)[:-len(".patch")].replace("\\", "/"), p + + +def custom_path(custom, path): + """custom/, matching existing directory and file names whatever + their case (Wine finds only one of two names that differ in case).""" + cur = custom + parts = path.split("/") + for i, part in enumerate(parts): + hit = None + if os.path.isdir(cur): + for n in os.listdir(cur): + if n.lower() == part.lower(): + hit = n + break + cur = os.path.join(cur, hit or part) + return cur + + +def apply(gamedir): + data = moddata.Data(gamedir) + custom = os.path.join(gamedir, "custom") + for path, recipe in patches(): + raw, real = data.read(path) + if raw is None: + sys.exit(f"ps2data: {path}: not in the game's rez archives") + text = raw.decode("latin-1") + base = os.path.join(MODERN, path + ".patch") + if os.path.exists(base): + text = moddata.apply_patch(text, open(base, encoding="latin-1").read(), path) + text = moddata.apply_patch(text, open(recipe, encoding="latin-1").read(), path) + dest = custom_path(custom, real) + os.makedirs(os.path.dirname(dest), exist_ok=True) + if os.path.lexists(dest): + os.remove(dest) # never write through a hard link + with open(dest, "wb") as f: + f.write(text.encode("latin-1")) + print(f" {os.path.relpath(dest, gamedir)} <- {os.path.relpath(recipe, REPO)}") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit(__doc__) + apply(sys.argv[1]) diff --git a/build/ps2/ps2install.py b/build/ps2/ps2install.py new file mode 100644 index 00000000..0cdf41c3 --- /dev/null +++ b/build/ps2/ps2install.py @@ -0,0 +1,757 @@ +#!/usr/bin/env python3 +"""Install the PS2 "Nine Years Ago" levels (Worlds\\thiefmission1-3) into a PC +NOLF game directory: every converted file they need, from the PS2 disc. + + python3 -I build/ps2/ps2install.py --ps2 --game [--out DIR] + python3 -I build/ps2/ps2install.py --ps2 ... --game ... --check [-v] + +--ps2 the PS2 disc image (ISO9660; LITH_PS2.REZ is read straight out of it) + or an extracted LITH_PS2.REZ. US and European discs both work (the + European one: English, directory table 0). +--game the PC game directory: its nolf*.rez are the retail archives the + converters measure against (textures, the PC child models of + hero_thief), and what --check resolves references in. +--out where the files go (default /custom). Directory names are + upper case, or the case of a directory already there (Wine sees one + directory per case-insensitive name). Every file is written to a + temporary name and renamed over the old one, never rewritten in place + (game directories are often hard-linked copies). +--check write nothing; list every file the levels and the attribute patches + (build/ps2/data/ATTRIBUTES/*.patch) reference and where it resolves + (custom dir or retail .rez, case-insensitive). An install ends with + the same check. -v lists the resolved ones too. + +Written (counts are for the US disc): + WORLDS/thiefmission1-3.dat ps2topc (PS2 lightmaps, scale 1.0), the PS2 + string IDs remapped (ps2strings), the door sound + typo Snd\\Doors\\0wood_c.wav fixed + TEX/... the world textures the PC archives lack + models ps2model (MODELS below) + skins PS2-only .dtx the levels, the patches, the + model styles and the code reference (ps2dtx) + sounds the 81 WAVs of sound-map.txt (ps2sound) + INTERFACE/PHOTOS/...pcx weapon/ammo icons (32x32 -> 48x48) and the + loading-screen photo (the 180x148 card of the + PS2's 256x256 texture), 8-bit PCX +""" +import argparse, os, re, struct, sys, time + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(REPO, 'tools')) +import ps2rez # noqa: E402 +import ps2world # noqa: E402 +import ps2topc # noqa: E402 +import ps2dtx # noqa: E402 +import ps2model # noqa: E402 +import ps2sound # noqa: E402 +import ps2strings # noqa: E402 +import abcmodel # noqa: E402 +from rezextract import Rez # noqa: E402 + +WORLDS = ['thiefmission1', 'thiefmission2', 'thiefmission3'] + +# (PS2 model, installed name as the game references it, PC child models) +MODELS = [ + ('guns/models_pv/ps2lockpick_pv.ltb', r'Guns\Models_PV\PS2lockpick_pv.abc', []), + ('guns/models_pv/ps2safecracker_pv.ltb', r'Guns\Models_PV\PS2SafeCracker_pv.abc', []), + ('guns/models_hh/ps2lockpick_hh.ltb', r'Guns\Models_HH\PS2lockpick_hh.abc', []), + ('guns/models_hh/ps2safecracker_hh.ltb', r'Guns\Models_HH\PS2SafeCracker_hh.abc', []), + ('interface/models/hud/moneybag.ltb', r'Interface\Models\MoneyBag.abc', []), + ('interface/models/hud/purses_01.ltb', r'Interface\Models\Purses_01.abc', []), + ('interface/models/hud/change_01.ltb', r'Interface\Models\Change_01.abc', []), + ('props/models/moneybag.ltb', r'Props\Models\MoneyBag.abc', []), + ('attachments/models/britishpolicehat.ltb', r'Attachments\Models\BritishPoliceHat.abc', []), + # ModelButes: chars\models\_