Release without a recovery: the root as a block image, orchard's files linked

.github/workflows/build.yml

@@ -147,6 +147,9 @@ jobs:
     needs: [test, orchard, artwork, base]
     env:
       HARDWARE_TAG: ${{ needs.orchard.outputs.tag }}
+      # The root image keeps its owners: as root, directly on this runner.
+      ORCHARD_BUILD_NATIVE: '1'
+      ORCHARD_BUILD_ENV_ID: ubuntu-24.04-arm-root-image
     runs-on: ubuntu-24.04-arm
     permissions:
       contents: read
@@ -211,8 +214,13 @@ jobs:
           # What orchard supplies, from its bundles as they are: nothing is fetched or built.
           python -m orchard.inputs --output out/orchard-inputs.json "${kernels[@]}" \
             --m1n1 "$tools/quiet/m1n1.bin" --pongo "$tools/quiet/pongo.bin" --tools "$tools/boot"
+          # A published orchard build's files are linked from the release; a run's own are hosted with it.
+          source=(--orchard-release "$HARDWARE_TAG")
+          if [[ -n "${HARDWARE_ARTIFACT_DIR:-}" && -d "${HARDWARE_ARTIFACT_DIR:-}" ]]; then
+            source=(--orchard-inputs out/orchard-inputs.json)
+          fi
           python build.py --base out/base/base.tar.gz --base-receipt out/base/base.tar.gz.json \
-            --inputs out/inputs/inputs.json --orchard-inputs out/orchard-inputs.json --orchard .orchard \
+            --inputs out/inputs/inputs.json "${source[@]}" --orchard .orchard \
             --version "$VERSION" --output out/release
           python audit.py out/release --orchard .orchard
           python -m orchard.manifests --verify-local out/release/index.toml

audit.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-"""Audit actual root/recovery payloads without extracting links on the host."""
+"""Audit the actual root and platform layers without extracting links on the host."""
 import argparse
-import gzip
 import hashlib
 import json
 from pathlib import Path, PurePosixPath
-import stat
 import struct
 import tarfile
 import tomllib
@@ -15,31 +13,6 @@
 HARDWARE = None  # the orchard snapshot; main() sets it, else orchard_root() finds it
 
 
-def newc(data):
-    result = {}
-    offset = 0
-    while True:
-        header = data[offset:offset + 110]
-        if len(header) != 110 or header[:6] != b'070701':
-            raise ValueError('Invalid newc header')
-        fields = [int(header[i:i + 8], 16) for i in range(6, 110, 8)]
-        mode, size, namesize = fields[1], fields[6], fields[11]
-        start = offset + 110
-        name = data[start:start + namesize - 1].decode()
-        if namesize < 1 or data[start + namesize - 1] != 0:
-            raise ValueError('Invalid newc name')
-        start = (start + namesize + 3) & ~3
-        payload = data[start:start + size]
-        if len(payload) != size:
-            raise ValueError('Truncated newc member')
-        offset = (start + size + 3) & ~3
-        if name == 'TRAILER!!!':
-            return result
-        if name.startswith('/') or '..' in PurePosixPath(name).parts or name in result:
-            raise ValueError('Unsafe/duplicate newc member')
-        result[name] = (payload, mode)
-
-
 def check_elf_pages(data):
     """The real J71b kernel uses 16 KiB pages; the container kernel may not."""
     if data[:6] != b'\x7fELF\x02\x01':
@@ -76,51 +49,17 @@ def audit_layer(path, platform, row):
     return files, description
 
 
-def audit_recovery(image, release, kernel_release, layer):
-    framing = json.loads(Path(str(image) + '.json').read_text())
-    assert sha(image) == framing['image_sha256']
-    components, dtbs = {}, 0
-    with image.open('rb') as stream:
-        for row in framing['components']:
-            stream.seek(row['offset'])
-            payload = stream.read(row['bytes'])
-            assert len(payload) == row['bytes'] and hashlib.sha256(payload).hexdigest() == row['sha256']
-            dtbs += row['name'] == 'dtb'
-            components[row['name']] = payload
-    profile = (HARDWARE or orchard_root()) / 'boot/initramfs/recovery/boot.json'  # the snapshot's tested profile
-    assert components['bootargs'] == json.loads(profile.read_text())['bootargs'].encode(), 'Recovery must preserve the tested standard boot profile'
-    recovery = newc(gzip.decompress(components['initramfs']))
-    config = json.loads(recovery['etc/orchard/release.json'][0])
-    assert (config['release'], config['kernel_release']) == (release, kernel_release)
-    # The recovery runs the kernel the installed entry boots: the same modules and regulatory data.
-    shared = 0
-    for name, data in layer.items():
-        if name.startswith(('usr/lib/modules/', 'usr/lib/firmware/')) and not name.endswith(('/vmlinuz', '/initrd')) \
-                and '/dtb/' not in name:
-            assert recovery[name][0] == data, name
-            shared += 1
-    for data, mode in recovery.values():
-        if stat.S_ISREG(mode):
-            check_elf_pages(data)
-    for name in ('usr/bin/python3', 'usr/bin/orchard-device', 'init'):
-        assert name in recovery
-    for name in ('blkid', 'sfdisk', 'mke2fs'):
-        data, mode = recovery['usr/sbin/' + name]
-        assert stat.S_ISREG(mode) and mode & 0o111 == 0o111 and data.startswith(b'\x7fELF')
-    assert recovery['etc/mke2fs.conf'][0]
-    for name in ('device.py', 'identity.py', 'adt.py', 'storage-fence'):
-        assert 'usr/libexec/orchard/' + name in recovery
-    assert not any(name.startswith('root/.ssh') or name.startswith('run/recovery/') for name in recovery)
-    return {'members': len(recovery), 'dtbs': dtbs, 'shared_with_layer': shared}
-
-
 def audit(directory):
     manifest = tomllib.loads((directory / 'release.toml').read_text())
     assert manifest['schema'] == 2 and manifest['distro'] == 'minimal-linux'
     record = json.loads((directory / 'rootfs.json').read_text())
     rootfile = directory / 'rootfs.tar.gz'
     assert rootfile.stat().st_size == record['bytes'] and sha(rootfile) == record['sha256']
-    assert manifest['root']['sha256'] == record['sha256']
+    # The checks read the tree the image was made from; orchard.image.root_image made it, and checks it.
+    for row in (manifest['root'], manifest['root']['map'], manifest['logo'], manifest['pongo'], manifest['m1n1'],
+                manifest['install_tools']):
+        path = directory / row['file']
+        assert path.stat().st_size == row['bytes'] and sha(path) == row['sha256'], row['file']
     entries = {}
     size = elf_loads = 0
     with tarfile.open(rootfile, 'r|gz') as archive:
@@ -160,21 +99,17 @@ def audit(directory):
     assert [row[0] for row in users if 1000 <= int(row[2]) < 60000] == ['tablet']
     platforms = {}
     for name, files in sorted(manifest['platforms'].items()):
-        for role in ('boot', 'layer'):
-            path = directory / files[role]['file']
-            assert path.stat().st_size == files[role]['bytes'] and sha(path) == files[role]['sha256']
+        path = directory / files['layer']['file']
+        assert path.stat().st_size == files['layer']['bytes'] and sha(path) == files['layer']['sha256']
         layer, description = audit_layer(directory / files['layer']['file'], name, files['layer'])
         assert sorted(description['boards']) == sorted(files['boards'])
         # The installer refuses a layer file the root already has.
         assert not (layer.keys() & {n for n, (data, _, _) in entries.items() if data is not None})
-        result = audit_recovery(directory / files['boot']['file'], record['release'],
-                                files['layer']['kernel_release'], layer)
-        assert result['dtbs'] == len(set(description['boards'].values()))
-        platforms[name] = {'boards': sorted(description['boards']), 'recovery': result}
+        platforms[name] = {'boards': sorted(description['boards'])}
     return {'root_sha256': record['sha256'], 'root_bytes': record['bytes'],
             'expanded_bytes': size, 'root_members': len(entries), 'platforms': platforms,
             'elf_16k_compatible_load_segments': elf_loads,
-            'standard_recovery_boot_profile': True, 'signed_regulatory_database_present': True,
+            'signed_regulatory_database_present': True,
             'init': 'BusyBox PID 1 + OpenRC',
             'libc': 'musl', 'compression': 'gzip', 'systemd': False,
             'physical_device_operations_performed': False, 'hardware_validated': False}

build.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""One Alpine/OpenRC release for every platform: the root, per platform a recovery and orchard's layer.
+"""One Alpine/OpenRC release for every platform: the root as a block image, its logo, and orchard's files.
 
 The root holds no kernel; orchard's platform layer brings kernel, initrd, DTBs
 and modules at install, and orchard writes the boot entry.
@@ -16,6 +16,7 @@
 import sys
 import tarfile
 import tempfile
+import tomllib
 
 from snapshot import command as orchard_command, root as orchard_root, sha
 
@@ -34,18 +35,6 @@ def verify_inputs(path):
     return result, spec
 
 
-def kernels(resolved, explicit=()):
-    """{platform: staged kernel bundle} of orchard.inputs' RESOLVED platforms; --kernel-dir overrides one."""
-    known = resolved['platforms']
-    given = {}
-    for value in explicit:
-        name, separator, path = value.partition('=')
-        if not separator or name not in known:
-            raise ValueError(f'Use --kernel-dir PLATFORM=DIRECTORY with a platform of: {", ".join(known)}')
-        given[name] = Path(path)
-    return {name: given.get(name) or Path(row['kernel']) for name, row in known.items()}
-
-
 def overlay():
     result = {}
     for path in (PROJECT / 'system/overlay').rglob('*'):
@@ -167,8 +156,11 @@ def receipt(path):
     return {'file': path.name, 'bytes': path.stat().st_size, 'sha256': sha(path)}
 
 
-def manifests(output, version, root, platforms, orchard):
-    """release.toml (orchard's schema 2, every platform) and the index that pins it."""
+def manifests(output, version, root, offered, orchard):
+    """release.toml (orchard's schema 2, every platform) and the index that pins it.
+
+    ROOT is orchard.image.root_image's table, OFFERED orchard's platforms.toml beside it.
+    """
     document = ('schema = 2\nkind = "orchard.install"\ndistro = "minimal-linux"\n'
                 'name = "Minimal Alpine"\nversion = ' + json.dumps(version) + '\n'
                 'boot_method = "checkm8-pongo"\nmin_orchard = "0.2.0"\nprotocol = 1\n')
@@ -178,73 +170,54 @@ def manifests(output, version, root, platforms, orchard):
     document += table('install', {'default_root': '/minimal', 'ready': '/run/orchard-ready'})
     document += table('install.account', {'user': 'tablet', 'password': True, 'pin': False,
                                           'passwordless_sudo': False})
-    row = receipt(output / 'rootfs.tar.gz')
-    row.update({key: root[key] for key in ('compression', 'expanded_bytes', 'members')})
-    document += table('root', row) + table('pongo', receipt(output / 'pongo.bin'))
-    for name, files in sorted(platforms.items()):
-        document += f'\n[platforms.{name}]\nboards = {json.dumps(files["boards"])}\n'
-        document += table(f'platforms.{name}.boot', files['boot']) + table(f'platforms.{name}.layer', files['layer'])
+    document += table('root', {key: value for key, value in root.items() if key != 'map'})
+    document += table('root.map', root['map'])
+    document += table('logo', receipt(output / 'logo.bin'))  # the Alpine mark, drawn by orchard's m1n1
+    document += table('pongo', offered['loaders']['pongo']) + table('m1n1', offered['loaders']['m1n1'])
+    document += table('install_tools', offered['install_tools'])
+    for name, layer in sorted(offered['platforms'].items()):
+        document += f'\n[platforms.{name}]\nboards = {json.dumps(layer["boards"])}\n'
+        document += table(f'platforms.{name}.layer', {key: value for key, value in layer.items() if key != 'boards'})
     path = output / 'release.toml'
     path.write_text(document)
     (output / 'index.toml').write_text('schema = 2\nkind = "orchard.index"\ndistro = "minimal-linux"\n'
                                        + table('release', receipt(path)))
     return path
 
 
-def recovery(orchard, output, base, base_receipt, components, platform_name, bundle, boards):
-    """The platform's RAM recovery, which starts every board of it."""
-    dtbs = [bundle / 'dtbs' / (dtb + '.dtb') for dtb in sorted(set(boards.values()))]
-    image = output / f'recovery-{platform_name}.bin'
-    command = [*(f'--dtb={path}' for path in dtbs),
-               '--base', str(base), '--base-receipt', str(base_receipt),
-               '--kernel', str(bundle / 'Image.gz'), '--modules-dir', str(bundle / 'lib/modules')]
-    for name in ('loader', 'busybox', 'dropbear', 'dropbearkey', 'wpa_supplicant', 'wpa_cli', 'iw'):
-        command += ['--' + name.replace('_', '-'), str(components[name])]
-    command += ['--logo', str(components['logo'])]  # the Alpine mark, drawn by orchard's m1n1
-    command += ['--root-receipt', str(output / 'rootfs.json'), '--output', str(image)]
-    orchard_command(orchard, 'image.recovery', *command)
-    return image
-
-
 def main():
     parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
-    for name in ('base', 'base-receipt', 'inputs', 'orchard-inputs', 'output'):
+    for name in ('base', 'base-receipt', 'inputs', 'output'):
         parser.add_argument('--' + name, type=Path, required=True)
     parser.add_argument('--version', required=True)
     parser.add_argument('--orchard', type=Path, help='orchard snapshot (default: ORCHARD_CHECKOUT, then .orchard)')
-    parser.add_argument('--kernel-dir', action='append', default=[], metavar='PLATFORM=DIRECTORY',
-                        help="a platform's staged kernel bundle instead of the one orchard.inputs names")
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument('--orchard-release', metavar='TAG', help="take orchard's files finished from this build")
+    source.add_argument('--orchard-inputs', type=Path, help="orchard's inputs for this build (orchard.inputs)")
     args = parser.parse_args()
     if args.output.exists() or args.output.is_symlink():
         parser.error('Choose a new output directory')
     orchard = orchard_root(args.orchard)
     components, inputs = verify_inputs(args.inputs)
-    resolved = json.loads(args.orchard_inputs.read_text())
-    try:
-        bundles = kernels(resolved, args.kernel_dir)
-    except ValueError as error:
-        parser.error(str(error))
+    resolved = json.loads(args.orchard_inputs.read_text()) if args.orchard_inputs else None
+    revision = resolved['orchard'] if resolved else None
     args.output.parent.mkdir(parents=True, exist_ok=True)
     with tempfile.TemporaryDirectory(prefix='.minimal-build-', dir=args.output.parent) as temp:
         output = Path(temp) / 'release'
         output.mkdir()
-        root = root_archive(args.base, args.base_receipt, components, output, args.version, orchard)
-        platforms = {}
-        for name, bundle in bundles.items():
-            layer_receipt = Path(temp) / f'layer-{name}.json'
-            orchard_command(orchard, 'image.platform', name, '--output', output, '--kernel-dir', bundle,
-                            '--busybox', components['busybox'], '--receipt', layer_receipt, stdout=subprocess.DEVNULL)
-            layer = json.loads(layer_receipt.read_text())
-            image = recovery(orchard, output, args.base, args.base_receipt, components, name, bundle,
-                             resolved['platforms'][name]['boards'])
-            platforms[name] = {'boards': layer.pop('boards'), 'layer': layer, 'boot': receipt(image)}
-        shutil.copyfile(components['pongo'], output / 'pongo.bin')
-        revision = resolved['orchard']
-        manifests(output, args.version, root, platforms, revision)
+        root_archive(args.base, args.base_receipt, components, output, args.version, orchard)
+        source = ('--release', args.orchard_release) if args.orchard_release else ('--inputs', args.orchard_inputs)
+        orchard_command(orchard, 'image.platforms_index', output, *source, stdout=subprocess.DEVNULL)
+        offered = tomllib.loads((output / 'platforms.toml').read_text())
+        revision = revision or {'commit': offered['commit'], 'modified': False}
+        orchard_command(orchard, 'image.root_image', output / 'rootfs.tar.gz', '--output', output)
+        root = json.loads((output / 'root-image.json').read_text())
+        shutil.copyfile(components['logo'], output / 'logo.bin')
+        manifests(output, args.version, root, offered, revision)
         public_inputs = {**inputs, 'files': {name: {k: v for k, v in row.items() if k != 'path'}
                                              for name, row in inputs['files'].items()}}
         (output / 'build-receipt.json').write_text(json.dumps({'distro': 'minimal-linux',
-            'version': args.version, 'orchard': revision, 'platforms': platforms,
+            'version': args.version, 'orchard': revision, 'platforms': offered['platforms'],
             'base': json.loads(args.base_receipt.read_text()), 'inputs': public_inputs,
             'physical_device_operations_performed': False, 'hardware_validated': False}, indent=2) + '\n')
         shutil.rmtree(output / '.work', ignore_errors=True)