Install into free LwVM space on 32-bit boards

docs/bring-up/p105-lwvm-write.md

@@ -0,0 +1,42 @@
+# P105: writing a Linux entry into the LwVM table
+
+State: code only. No device has run it, and nothing here claims that it is
+safe for iOS.
+
+## What orchard does
+
+When the iOS disk's LwVM map has unallocated chunks stored back to back
+(at least the minimum root size, and room left in LwVM's virtual space and
+descriptor table), `orchard install` offers "create a Linux partition in free
+space". The person has to type `create partition`. Then:
+
+1. The host reads both header copies (offset 0 and the end of the first
+   chunk), which must agree and match what was inspected.
+2. The host computes the new header (`src/orchard/lwvm.py`,
+   `allocate_free`): one Linux entry (`0fc63daf-…`, name `Linux`, no
+   attributes) appended after the others, its virtual range after every
+   other partition's, the free chunks mapped to it in order, the CRC
+   recomputed, and the result parsed again. The allocation follows Shayne
+   Legault's `allocate` in #94; nothing else in the header changes.
+3. The add-on writes the copy at the end of the first chunk, reads it back,
+   then the primary, reads it back (`dd ... conv=fsync`), and the host parses
+   both copies again.
+4. The kernel re-reads the table and must add the partition with the planned
+   number, start and size. Only then is the root written into it.
+
+If the install stops between the two writes, the copies disagree: the
+kernel keeps the old primary, and the installer refuses the disk until
+someone looks.
+
+## Why it has not run on a device
+
+- Today no unallocated LwVM space exists: Shayne's resize tool shrinks the
+  Data filesystem but does not yet shrink Data's LwVM entry, so this option
+  never appears.
+- Whether iOS 9 accepts an LwVM entry it did not create is unknown; Shayne's
+  notes mention a key-store step on the iOS side.
+
+The first device test of this write happens only after Shayne's resize tool
+has shown that LwVM writes are safe for iOS (iOS still boots, Data still
+unlocks). Until then the code is tested only with synthetic headers
+(`tests/test_lwvm_free.py`).

src/orchard/install_direct.py

@@ -323,14 +323,9 @@ def inspect_lwvm(connection, disk):
     partitions, which the kernel skips, are not offered, and a partition whose place
     differs between the kernel and the header stops the install.
     """
-    name = Path(disk).name
-    sector = int(shell(connection, f'cat /sys/class/block/{name}/queue/logical_block_size'))
-    media = int(shell(connection, f'cat /sys/class/block/{name}/size')) * lwvm.SECTOR
-    chunk = lwvm.chunk_size(media)
+    sector, media, chunk, primary, backup = lwvm_headers(connection, disk)
     try:
-        snapshot = lwvm.parse(read_disk(connection, disk, 0, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES),
-                              read_disk(connection, disk, chunk - lwvm.HEADER_BYTES, lwvm.HEADER_BYTES,
-                                        lwvm.HEADER_BYTES), media)
+        snapshot = lwvm.parse(primary, backup, media)
     except lwvm.Error as error:
         raise D.InstallError('lwvm', f'{disk}: {error}') from error
     found = kernel_partitions(connection, disk)
@@ -349,8 +344,22 @@ def inspect_lwvm(connection, disk):
         row.update(probe(shell(connection, f'dd if={D.node(disk, part["number"])} bs=4096 count=1 2>/dev/null || true')))
         partitions.append(row)
     info = {'path': disk, 'sector': sector, 'table': 'lwvm', 'table_sha256': snapshot['header_sha256'],
-            'media_bytes': media, 'chunk_bytes': chunk}
-    return {'disk': info, 'partitions': sorted(partitions, key=lambda p: p['start']), 'free': []}
+            'media_bytes': media, 'chunk_bytes': chunk, 'entries': len(snapshot['partitions'])}
+    # Unallocated chunks back to back, where a new Linux entry can go (lwvm.free_space).
+    free = [{'start': first * chunk // sector, 'blocks': count * chunk // sector}
+            for first, count in lwvm.free_space(primary, snapshot)]
+    return {'disk': info, 'partitions': sorted(partitions, key=lambda p: p['start']),
+            'free': sorted(free, key=lambda f: f['start'])}
+
+
+def lwvm_headers(connection, disk):
+    """(sector, media bytes, chunk bytes, primary, backup): both copies of the LwVM header as the disk has them."""
+    name = Path(disk).name
+    sector = int(shell(connection, f'cat /sys/class/block/{name}/queue/logical_block_size'))
+    media = int(shell(connection, f'cat /sys/class/block/{name}/size')) * lwvm.SECTOR
+    chunk = lwvm.chunk_size(media)
+    return (sector, media, chunk, read_disk(connection, disk, 0, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES),
+            read_disk(connection, disk, chunk - lwvm.HEADER_BYTES, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES))
 
 
 def lwvm_target(connection, plan, disk):
@@ -362,6 +371,8 @@ def lwvm_target(connection, plan, disk):
     if layout['disk']['table_sha256'] != plan['disk']['table_sha256'] or layout['disk']['sector'] != plan['disk']['sector']:
         raise D.InstallError('layout-changed', 'The LwVM table changed since it was inspected; inspect again')
     root = plan['root']
+    if root.get('create'):
+        return lwvm_create(connection, plan, disk)
     match = [p for p in layout['partitions'] if p['number'] == root['number']]
     if (not match or (match[0]['partuuid'], match[0]['start'], match[0]['blocks']) !=
             (root['partuuid'], root['start'], root['blocks']) or
@@ -370,6 +381,51 @@ def lwvm_target(connection, plan, disk):
     return D.node(disk, root['number']), layout['partitions']
 
 
+def lwvm_create(connection, plan, disk):
+    """Write the plan's new Linux entry into both LwVM header copies; its node once the kernel has it.
+
+    The host computes the header (lwvm.allocate_free). The copy at the end of the first
+    chunk is written first, then the primary, each read back; then both are parsed again
+    and the kernel re-reads the table, which must add the partition where the plan has it.
+    Interrupted between the two writes, the copies disagree: the kernel keeps reading the
+    old primary, and the installer refuses the disk until someone looks (both are logged).
+    No hardware has run this yet (docs/bring-up/p105-lwvm-write.md).
+    """
+    root = plan['root']
+    sector, media, chunk, primary, backup = lwvm_headers(connection, disk)
+    if lwvm.digest(primary) != plan['disk']['table_sha256']:
+        raise D.InstallError('layout-changed', 'The LwVM table changed since it was inspected; inspect again')
+    if (root['start'] * sector) % chunk or (root['blocks'] * sector) % chunk:
+        raise D.InstallError('plan-mismatch', 'The new partition does not start and end on LwVM chunks')
+    first, chunks = root['start'] * sector // chunk, root['blocks'] * sector // chunk
+    try:
+        header, entry = lwvm.allocate_free(primary, backup, root['partuuid'], first, chunks)
+    except lwvm.Error as error:
+        raise D.InstallError('lwvm', f'{disk}: {error}') from error
+    if entry['number'] != root['number'] or entry['extents'] != [{'logical': 0, 'physical': first * chunk,
+                                                                  'bytes': chunks * chunk}]:
+        raise D.InstallError('plan-mismatch', 'The new LwVM entry is not as planned')
+    for name, offset in (('copy', chunk - lwvm.HEADER_BYTES), ('primary', 0)):
+        write_at(connection, disk, lwvm.SECTOR, offset // lwvm.SECTOR, header)
+        if read_disk(connection, disk, offset, lwvm.HEADER_BYTES, lwvm.HEADER_BYTES) != header:
+            raise D.InstallError('lwvm-write', f'The LwVM header {name} reads back differently; '
+                                               'the table is in an uncertain state, stop and inspect it')
+    _, _, _, primary, backup = lwvm_headers(connection, disk)
+    try:
+        if lwvm.parse(primary, backup, media)['header_sha256'] != lwvm.digest(header):
+            raise lwvm.Error('the headers on disk are not the ones written')
+    except lwvm.Error as error:
+        raise D.InstallError('lwvm-write', f'{disk}: {error}') from error
+    node = D.node(disk, root['number'])
+    # A re-read right after the write can be refused while the disk settles, so keep asking.
+    shell(connection, f'for n in $(seq 150); do blockdev --rereadpt {disk} 2>/dev/null; [ -e {node} ] && exit 0; '
+                      'sleep 0.1; done; echo "The new partition did not appear" >&2; exit 1', timeout=30)
+    if kernel_partitions(connection, disk).get(root['number']) != (root['start'] * sector // lwvm.SECTOR,
+                                                                     root['blocks'] * sector // lwvm.SECTOR):
+        raise D.InstallError('table-verify', 'The kernel does not see the new partition where it was planned')
+    return node, inspect_lwvm(connection, disk)['partitions']
+
+
 def entry_bytes(kind, partuuid, start, blocks, name):
     return (uuid.UUID(kind).bytes_le + uuid.UUID(partuuid).bytes_le +
             struct.pack('<QQQ', start, start + blocks - 1, 0) + name.encode('utf-16-le')[:72].ljust(72, b'\0'))

src/orchard/install_layout.py

@@ -9,9 +9,9 @@
 - The disk has at most one XBOOTLDR boot partition; it is created inside the
   target when missing, and an existing one is reused.
 - On an LwVM disk (32-bit boards) the target is an existing Linux partition,
-  which is replaced in place, and there is no boot partition: the boot entry
-  stays in the root's /boot. The installer never changes the LwVM map; making
-  room for Linux is iOS's resize step.
+  replaced in place, or unallocated LwVM space, where a new Linux entry is
+  written into both header copies. There is no boot partition: the boot entry
+  stays in the root's /boot. Making unallocated space is iOS's resize step.
 """
 from dataclasses import dataclass
 from typing import Dict, List, Optional
@@ -98,9 +98,15 @@ def options(layout) -> List[Option]:
 
 
 def lwvm_options(layout) -> List[Option]:
-    """The Linux partitions of an LwVM disk that a root fits in; LwVM attributes (encryption) rule one out."""
+    """Free LwVM space and the Linux partitions of an LwVM disk that a root fits in; LwVM attributes
+    (encryption) rule a partition out."""
     block = layout['disk']['sector']
     result = []
+    if layout['disk'].get('entries', 8) < 8:
+        for free in layout['free']:
+            if free['blocks'] * block >= MIN_ROOT_BYTES:
+                result.append(Option('free', free['start'], free['blocks'],
+                                     f'create a Linux partition in free space, {free["blocks"] * block / GIB:.1f} GiB'))
     for part in layout['partitions']:
         if part['type'] != LINUX_DATA or part.get('attributes') or part['blocks'] * block < MIN_ROOT_BYTES:
             continue
@@ -163,15 +169,22 @@ def plan(layout, option: Option, image: Dict, entry: Dict, layer: Dict, firmware
 
 
 def lwvm_plan(layout, option, image, entry, layer, firmware, account):
-    """The plan for an LwVM disk: the root fills the chosen Linux partition, which keeps its place and identity.
+    """The plan for an LwVM disk: the root fills the chosen Linux partition, which keeps its place and
+    identity, or a new Linux entry over a free run (root.create).
 
-    No partition is created or removed and there is no boot partition: `boot`
-    is None, and the boot entry goes to the root's own /boot.
+    No partition is removed and there is no boot partition: `boot` is None,
+    and the boot entry goes to the root's own /boot.
     """
     block = layout['disk']['sector']
     part = option.partition
-    if option.kind != 'replace' or part is None or part['type'] != LINUX_DATA or part.get('attributes'):
+    if option.kind == 'free':
+        # A new entry over the free run as it is: chunks are its unit, so no alignment changes it.
+        part = {'number': layout['disk']['entries'] + 1, 'start': option.start, 'blocks': option.blocks,
+                'partuuid': str(uuid.uuid4()), 'create': True}
+    elif option.kind != 'replace' or part is None or part['type'] != LINUX_DATA or part.get('attributes'):
         raise LayoutError('On an LwVM disk only an existing, unencrypted Linux partition can be replaced')
+    if part['blocks'] * block < MIN_ROOT_BYTES:
+        raise LayoutError('Target is too small for a root partition')
     if part['blocks'] * block < (image['expanded_bytes'] + layer['expanded_bytes']) * 1.25:
         raise LayoutError('Target is too small for this system image')
     return {
@@ -181,8 +194,8 @@ def lwvm_plan(layout, option, image, entry, layer, firmware, account):
         'remove': None,
         'boot': None,
         'root': {'number': part['number'], 'start': part['start'], 'blocks': part['blocks'],
-                 'partuuid': part['partuuid'], 'fs_uuid': str(uuid.uuid4()), 'fs': 'ext4',
-                 'label': entry['id'][:16]},
+                 'partuuid': part['partuuid'], 'create': bool(part.get('create')), 'fs_uuid': str(uuid.uuid4()),
+                 'fs': 'ext4', 'label': entry['id'][:16]},
         'image': image,
         'layer': layer,
         'entry': entry,

src/orchard/lwvm.py

@@ -106,3 +106,73 @@ def partition(snapshot, identity):
     if len(found) != 1:
         raise Error('LwVM partition identity was not found')
     return found[0]
+
+
+FREE_CHUNK = 0xf111
+
+
+def free_runs(header, snapshot):
+    """[(first physical chunk, chunks)] of unallocated chunks stored back to back, longest first.
+
+    Only a run can hold a Linux partition the kernel's LwVM reader adds: it
+    takes a partition whose chunks are in order and contiguous.
+    """
+    words = struct.unpack_from('<1024H', header, 0x800)
+    runs, start = [], None
+    for physical in range(1, snapshot['media_bytes'] // snapshot['chunk_bytes'] + 1):
+        free = physical < snapshot['media_bytes'] // snapshot['chunk_bytes'] and words[physical] == FREE_CHUNK
+        if free and start is None:
+            start = physical
+        elif not free and start is not None:
+            runs.append((start, physical - start))
+            start = None
+    return sorted(runs, key=lambda run: (-run[1], run[0]))
+
+
+def virtual_room(snapshot):
+    """(first byte, bytes) of the LwVM virtual space after every partition, where a new entry's range goes."""
+    begin = max([HEADER_BYTES, *(p['end'] for p in snapshot['partitions'])])
+    begin = -(-begin // HEADER_BYTES) * HEADER_BYTES
+    return begin, snapshot['media_bytes'] - begin
+
+
+def free_space(header, snapshot):
+    """[(first chunk, chunks)] a new Linux entry could take: free runs, cut to the virtual room left."""
+    if len(snapshot['partitions']) >= 8:
+        return []
+    _, room = virtual_room(snapshot)
+    most = room // snapshot['chunk_bytes']
+    return [(first, min(count, most)) for first, count in free_runs(header, snapshot) if min(count, most) > 0]
+
+
+def allocate_free(primary, backup, linux_uuid, first, chunks):
+    """A new header with a Linux entry over the free chunks FIRST..FIRST+CHUNKS-1, and that entry.
+
+    Nothing else changes: the entry is appended, its virtual range follows
+    every other partition's, and the chunks are mapped to it in order. The
+    result is parsed again before it is returned.
+    """
+    before = parse(primary, backup)
+    count = len(before['partitions'])
+    if count == 8 or any(primary[0x200 + count * 128:0x280 + count * 128]):
+        raise Error('No empty LwVM descriptor is available')
+    if not any(start <= first and first + chunks <= start + length for start, length in free_space(primary, before)):
+        raise Error('The chosen chunks are not free LwVM space')
+    identity = uuid.UUID(linux_uuid)
+    if not identity.int or str(identity) in {p['partuuid'] for p in before['partitions']}:
+        raise Error('Linux partition identity is zero or repeated')
+    begin, _ = virtual_room(before)
+    end = begin + chunks * before['chunk_bytes']
+    header = bytearray(primary)
+    struct.pack_into('<I', header, 0x28, count + 1)
+    descriptor = bytearray(128)
+    descriptor[:16] = uuid.UUID(LINUX_TYPE).bytes
+    descriptor[16:32] = identity.bytes
+    struct.pack_into('<QQQ', descriptor, 32, begin, end, 0)
+    descriptor[56:66] = 'Linux'.encode('utf-16-le')
+    header[0x200 + count * 128:0x280 + count * 128] = descriptor
+    for logical in range(chunks):
+        struct.pack_into('<H', header, 0x800 + (first + logical) * 2, count << 12 | logical)
+    struct.pack_into('<I', header, 0x2c, checksum(header))
+    after = parse(bytes(header), bytes(header), before['media_bytes'])
+    return bytes(header), partition(after, str(identity))

src/orchard/service.py

@@ -113,8 +113,9 @@ def choose_target(layout, clock, *, name='Linux', input_fn=input, output=print):
     """Pick one install target from the device's layout; replacing a partition is typed out."""
     options = install_layout.options(layout)
     if not options and install_layout.lwvm(layout):
-        raise ValueError(f'No unencrypted Linux partition of at least {install_layout.MIN_ROOT_BYTES / 1024**3:.0f} GiB. '
-                         'The installer never resizes iOS; a Linux partition has to exist already.')
+        raise ValueError(f'No free LwVM space or unencrypted Linux partition of at least '
+                         f'{install_layout.MIN_ROOT_BYTES / 1024**3:.0f} GiB. '
+                         'The installer never resizes iOS; the space has to exist already.')
     if not options:
         needed = (install_layout.MIN_ROOT_BYTES + install_layout.BOOT_BYTES) / 1024**3
         raise ValueError(f'No free space or Linux partition of at least {needed:.0f} GiB. '
@@ -127,6 +128,11 @@ def choose_target(layout, clock, *, name='Linux', input_fn=input, output=print):
     if not answer.isdecimal() or not 1 <= int(answer) <= len(options):
         raise ValueError('No valid target was chosen; nothing was changed')
     option = options[int(answer) - 1]
+    if option.kind == 'free' and install_layout.lwvm(layout):
+        output('This writes a new Linux entry into the iOS partition table (LwVM). Whether iOS accepts '
+               'an entry it did not create has not been tested yet.')
+        if ask(clock, 'Type "create partition" to confirm: ', input_fn) != 'create partition':
+            raise ValueError('Not confirmed; nothing was changed')
     if option.kind == 'replace':
         phrase = f'erase {option.partition["number"]}'
         if option.partition['type'] == install_layout.LINUX_DATA and not install_layout.lwvm(layout):