diff --git a/docs/bring-up/p105-lwvm-write.md b/docs/bring-up/p105-lwvm-write.md new file mode 100644 index 00000000..22be10f8 --- /dev/null +++ b/docs/bring-up/p105-lwvm-write.md @@ -0,0 +1,42 @@ +# P105: writing a Linux entry into the LwVM table + +State: code only. No device has run it, and nothing here claims that it is +safe for iOS. + +## What orchard does + +When the iOS disk's LwVM map has unallocated chunks stored back to back +(at least the minimum root size, and room left in LwVM's virtual space and +descriptor table), `orchard install` offers "create a Linux partition in free +space". The person has to type `create partition`. Then: + +1. The host reads both header copies (offset 0 and the end of the first + chunk), which must agree and match what was inspected. +2. The host computes the new header (`src/orchard/lwvm.py`, + `allocate_free`): one Linux entry (`0fc63daf-…`, name `Linux`, no + attributes) appended after the others, its virtual range after every + other partition's, the free chunks mapped to it in order, the CRC + recomputed, and the result parsed again. The allocation follows Shayne + Legault's `allocate` in #94; nothing else in the header changes. +3. The add-on writes the copy at the end of the first chunk, reads it back, + then the primary, reads it back (`dd ... conv=fsync`), and the host parses + both copies again. +4. The kernel re-reads the table and must add the partition with the planned + number, start and size. Only then is the root written into it. + +If the install stops between the two writes, the copies disagree: the +kernel keeps the old primary, and the installer refuses the disk until +someone looks. + +## Why it has not run on a device + +- Today no unallocated LwVM space exists: Shayne's resize tool shrinks the + Data filesystem but does not yet shrink Data's LwVM entry, so this option + never appears. +- Whether iOS 9 accepts an LwVM entry it did not create is unknown; Shayne's + notes mention a key-store step on the iOS side. + +The first device test of this write happens only after Shayne's resize tool +has shown that LwVM writes are safe for iOS (iOS still boots, Data still +unlocks). Until then the code is tested only with synthetic headers +(`tests/test_lwvm_free.py`). diff --git a/src/orchard/install_direct.py b/src/orchard/install_direct.py index 36f2c2b8..f9b362fc 100644 --- a/src/orchard/install_direct.py +++ b/src/orchard/install_direct.py @@ -323,14 +323,9 @@ def inspect_lwvm(connection, disk): partitions, which the kernel skips, are not offered, and a partition whose place differs between the kernel and the header stops the install. """ - name = Path(disk).name - sector = int(shell(connection, f'cat /sys/class/block/{name}/queue/logical_block_size')) - media = int(shell(connection, f'cat /sys/class/block/{name}/size')) * lwvm.SECTOR - chunk = lwvm.chunk_size(media) + sector, media, chunk, primary, backup = lwvm_headers(connection, disk) try: - snapshot = lwvm.parse(read_disk(connection, disk, 0, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES), - read_disk(connection, disk, chunk - lwvm.HEADER_BYTES, lwvm.HEADER_BYTES, - lwvm.HEADER_BYTES), media) + snapshot = lwvm.parse(primary, backup, media) except lwvm.Error as error: raise D.InstallError('lwvm', f'{disk}: {error}') from error found = kernel_partitions(connection, disk) @@ -349,8 +344,22 @@ def inspect_lwvm(connection, disk): row.update(probe(shell(connection, f'dd if={D.node(disk, part["number"])} bs=4096 count=1 2>/dev/null || true'))) partitions.append(row) info = {'path': disk, 'sector': sector, 'table': 'lwvm', 'table_sha256': snapshot['header_sha256'], - 'media_bytes': media, 'chunk_bytes': chunk} - return {'disk': info, 'partitions': sorted(partitions, key=lambda p: p['start']), 'free': []} + 'media_bytes': media, 'chunk_bytes': chunk, 'entries': len(snapshot['partitions'])} + # Unallocated chunks back to back, where a new Linux entry can go (lwvm.free_space). + free = [{'start': first * chunk // sector, 'blocks': count * chunk // sector} + for first, count in lwvm.free_space(primary, snapshot)] + return {'disk': info, 'partitions': sorted(partitions, key=lambda p: p['start']), + 'free': sorted(free, key=lambda f: f['start'])} + + +def lwvm_headers(connection, disk): + """(sector, media bytes, chunk bytes, primary, backup): both copies of the LwVM header as the disk has them.""" + name = Path(disk).name + sector = int(shell(connection, f'cat /sys/class/block/{name}/queue/logical_block_size')) + media = int(shell(connection, f'cat /sys/class/block/{name}/size')) * lwvm.SECTOR + chunk = lwvm.chunk_size(media) + return (sector, media, chunk, read_disk(connection, disk, 0, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES), + read_disk(connection, disk, chunk - lwvm.HEADER_BYTES, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES)) def lwvm_target(connection, plan, disk): @@ -362,6 +371,8 @@ def lwvm_target(connection, plan, disk): if layout['disk']['table_sha256'] != plan['disk']['table_sha256'] or layout['disk']['sector'] != plan['disk']['sector']: raise D.InstallError('layout-changed', 'The LwVM table changed since it was inspected; inspect again') root = plan['root'] + if root.get('create'): + return lwvm_create(connection, plan, disk) match = [p for p in layout['partitions'] if p['number'] == root['number']] if (not match or (match[0]['partuuid'], match[0]['start'], match[0]['blocks']) != (root['partuuid'], root['start'], root['blocks']) or @@ -370,6 +381,51 @@ def lwvm_target(connection, plan, disk): return D.node(disk, root['number']), layout['partitions'] +def lwvm_create(connection, plan, disk): + """Write the plan's new Linux entry into both LwVM header copies; its node once the kernel has it. + + The host computes the header (lwvm.allocate_free). The copy at the end of the first + chunk is written first, then the primary, each read back; then both are parsed again + and the kernel re-reads the table, which must add the partition where the plan has it. + Interrupted between the two writes, the copies disagree: the kernel keeps reading the + old primary, and the installer refuses the disk until someone looks (both are logged). + No hardware has run this yet (docs/bring-up/p105-lwvm-write.md). + """ + root = plan['root'] + sector, media, chunk, primary, backup = lwvm_headers(connection, disk) + if lwvm.digest(primary) != plan['disk']['table_sha256']: + raise D.InstallError('layout-changed', 'The LwVM table changed since it was inspected; inspect again') + if (root['start'] * sector) % chunk or (root['blocks'] * sector) % chunk: + raise D.InstallError('plan-mismatch', 'The new partition does not start and end on LwVM chunks') + first, chunks = root['start'] * sector // chunk, root['blocks'] * sector // chunk + try: + header, entry = lwvm.allocate_free(primary, backup, root['partuuid'], first, chunks) + except lwvm.Error as error: + raise D.InstallError('lwvm', f'{disk}: {error}') from error + if entry['number'] != root['number'] or entry['extents'] != [{'logical': 0, 'physical': first * chunk, + 'bytes': chunks * chunk}]: + raise D.InstallError('plan-mismatch', 'The new LwVM entry is not as planned') + for name, offset in (('copy', chunk - lwvm.HEADER_BYTES), ('primary', 0)): + write_at(connection, disk, lwvm.SECTOR, offset // lwvm.SECTOR, header) + if read_disk(connection, disk, offset, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES) != header: + raise D.InstallError('lwvm-write', f'The LwVM header {name} reads back differently; ' + 'the table is in an uncertain state, stop and inspect it') + _, _, _, primary, backup = lwvm_headers(connection, disk) + try: + if lwvm.parse(primary, backup, media)['header_sha256'] != lwvm.digest(header): + raise lwvm.Error('the headers on disk are not the ones written') + except lwvm.Error as error: + raise D.InstallError('lwvm-write', f'{disk}: {error}') from error + node = D.node(disk, root['number']) + # A re-read right after the write can be refused while the disk settles, so keep asking. + shell(connection, f'for n in $(seq 150); do blockdev --rereadpt {disk} 2>/dev/null; [ -e {node} ] && exit 0; ' + 'sleep 0.1; done; echo "The new partition did not appear" >&2; exit 1', timeout=30) + if kernel_partitions(connection, disk).get(root['number']) != (root['start'] * sector // lwvm.SECTOR, + root['blocks'] * sector // lwvm.SECTOR): + raise D.InstallError('table-verify', 'The kernel does not see the new partition where it was planned') + return node, inspect_lwvm(connection, disk)['partitions'] + + def entry_bytes(kind, partuuid, start, blocks, name): return (uuid.UUID(kind).bytes_le + uuid.UUID(partuuid).bytes_le + struct.pack(' List[Option]: def lwvm_options(layout) -> List[Option]: - """The Linux partitions of an LwVM disk that a root fits in; LwVM attributes (encryption) rule one out.""" + """Free LwVM space and the Linux partitions of an LwVM disk that a root fits in; LwVM attributes + (encryption) rule a partition out.""" block = layout['disk']['sector'] result = [] + if layout['disk'].get('entries', 8) < 8: + for free in layout['free']: + if free['blocks'] * block >= MIN_ROOT_BYTES: + result.append(Option('free', free['start'], free['blocks'], + f'create a Linux partition in free space, {free["blocks"] * block / GIB:.1f} GiB')) for part in layout['partitions']: if part['type'] != LINUX_DATA or part.get('attributes') or part['blocks'] * block < MIN_ROOT_BYTES: continue @@ -163,15 +169,22 @@ def plan(layout, option: Option, image: Dict, entry: Dict, layer: Dict, firmware def lwvm_plan(layout, option, image, entry, layer, firmware, account): - """The plan for an LwVM disk: the root fills the chosen Linux partition, which keeps its place and identity. + """The plan for an LwVM disk: the root fills the chosen Linux partition, which keeps its place and + identity, or a new Linux entry over a free run (root.create). - No partition is created or removed and there is no boot partition: `boot` - is None, and the boot entry goes to the root's own /boot. + No partition is removed and there is no boot partition: `boot` is None, + and the boot entry goes to the root's own /boot. """ block = layout['disk']['sector'] part = option.partition - if option.kind != 'replace' or part is None or part['type'] != LINUX_DATA or part.get('attributes'): + if option.kind == 'free': + # A new entry over the free run as it is: chunks are its unit, so no alignment changes it. + part = {'number': layout['disk']['entries'] + 1, 'start': option.start, 'blocks': option.blocks, + 'partuuid': str(uuid.uuid4()), 'create': True} + elif option.kind != 'replace' or part is None or part['type'] != LINUX_DATA or part.get('attributes'): raise LayoutError('On an LwVM disk only an existing, unencrypted Linux partition can be replaced') + if part['blocks'] * block < MIN_ROOT_BYTES: + raise LayoutError('Target is too small for a root partition') if part['blocks'] * block < (image['expanded_bytes'] + layer['expanded_bytes']) * 1.25: raise LayoutError('Target is too small for this system image') return { @@ -181,8 +194,8 @@ def lwvm_plan(layout, option, image, entry, layer, firmware, account): 'remove': None, 'boot': None, 'root': {'number': part['number'], 'start': part['start'], 'blocks': part['blocks'], - 'partuuid': part['partuuid'], 'fs_uuid': str(uuid.uuid4()), 'fs': 'ext4', - 'label': entry['id'][:16]}, + 'partuuid': part['partuuid'], 'create': bool(part.get('create')), 'fs_uuid': str(uuid.uuid4()), + 'fs': 'ext4', 'label': entry['id'][:16]}, 'image': image, 'layer': layer, 'entry': entry, diff --git a/src/orchard/lwvm.py b/src/orchard/lwvm.py index 524082c1..755d2bab 100644 --- a/src/orchard/lwvm.py +++ b/src/orchard/lwvm.py @@ -106,3 +106,73 @@ def partition(snapshot, identity): if len(found) != 1: raise Error('LwVM partition identity was not found') return found[0] + + +FREE_CHUNK = 0xf111 + + +def free_runs(header, snapshot): + """[(first physical chunk, chunks)] of unallocated chunks stored back to back, longest first. + + Only a run can hold a Linux partition the kernel's LwVM reader adds: it + takes a partition whose chunks are in order and contiguous. + """ + words = struct.unpack_from('<1024H', header, 0x800) + runs, start = [], None + for physical in range(1, snapshot['media_bytes'] // snapshot['chunk_bytes'] + 1): + free = physical < snapshot['media_bytes'] // snapshot['chunk_bytes'] and words[physical] == FREE_CHUNK + if free and start is None: + start = physical + elif not free and start is not None: + runs.append((start, physical - start)) + start = None + return sorted(runs, key=lambda run: (-run[1], run[0])) + + +def virtual_room(snapshot): + """(first byte, bytes) of the LwVM virtual space after every partition, where a new entry's range goes.""" + begin = max([HEADER_BYTES, *(p['end'] for p in snapshot['partitions'])]) + begin = -(-begin // HEADER_BYTES) * HEADER_BYTES + return begin, snapshot['media_bytes'] - begin + + +def free_space(header, snapshot): + """[(first chunk, chunks)] a new Linux entry could take: free runs, cut to the virtual room left.""" + if len(snapshot['partitions']) >= 8: + return [] + _, room = virtual_room(snapshot) + most = room // snapshot['chunk_bytes'] + return [(first, min(count, most)) for first, count in free_runs(header, snapshot) if min(count, most) > 0] + + +def allocate_free(primary, backup, linux_uuid, first, chunks): + """A new header with a Linux entry over the free chunks FIRST..FIRST+CHUNKS-1, and that entry. + + Nothing else changes: the entry is appended, its virtual range follows + every other partition's, and the chunks are mapped to it in order. The + result is parsed again before it is returned. + """ + before = parse(primary, backup) + count = len(before['partitions']) + if count == 8 or any(primary[0x200 + count * 128:0x280 + count * 128]): + raise Error('No empty LwVM descriptor is available') + if not any(start <= first and first + chunks <= start + length for start, length in free_space(primary, before)): + raise Error('The chosen chunks are not free LwVM space') + identity = uuid.UUID(linux_uuid) + if not identity.int or str(identity) in {p['partuuid'] for p in before['partitions']}: + raise Error('Linux partition identity is zero or repeated') + begin, _ = virtual_room(before) + end = begin + chunks * before['chunk_bytes'] + header = bytearray(primary) + struct.pack_into('