From 1e7534b7a467b4487aa922b273bb9deec0271aad Mon Sep 17 00:00:00 2001 From: Stuart Fraser <60394096+stuart902@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:21:18 -0300 Subject: [PATCH 1/2] Reuse device-cached root images with local extraction progress --- README.md | 12 +++ docs/device-protocol.md | 47 ++++++++- src/orchard_cli/progress.py | 67 +++++++++++-- src/orchard_cli/protocol.py | 4 +- src/orchard_cli/service.py | 52 ++++++++-- tests/test_cached_progress.py | 182 ++++++++++++++++++++++++++++++++++ tests/test_host_workflow.py | 23 ++++- tests/test_root_cache.py | 164 ++++++++++++++++++++++++++++++ 8 files changed, 527 insertions(+), 24 deletions(-) create mode 100644 tests/test_cached_progress.py create mode 100644 tests/test_root_cache.py diff --git a/README.md b/README.md index b982f5f..a9df925 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,18 @@ orchard install omarchy-a The registered name stores a URL. Each resolution fetches the current index; verified images are cached by SHA-256. Publishing a new distro index updates future installations, not an already installed operating system. +### Reinstalling the same image + +With a recovery image that supports root-image caching, Orchard retains a verified +public root archive on the selected Linux-data partition, outside the installed +system directory. Reinstalling the same image uses that cache instead of sending +the archive over USB again. Changed, missing or corrupt images are transferred +in full. The progress output identifies cache verification and local extraction. + +No extra partition is needed. Cache reuse still performs local verification and +extraction; it is not an in-place upgrade or a backup of the replaced directory. +Older recovery images continue to use the normal streaming installation path. + ## Launch an installed system Connect the device in DFU mode, then run: diff --git a/docs/device-protocol.md b/docs/device-protocol.md index 4f62117..f3339d8 100644 --- a/docs/device-protocol.md +++ b/docs/device-protocol.md @@ -13,12 +13,12 @@ Images expose `orchard-device` and image-owned generic `ready`. Omarchy-specific Initial USB serial invokes the versioned `orchard-device bootstrap BASE64_JSON` operation with a public key. The runtime owns authorization, daemon/PID/key-file state and startup; it returns the host key and endpoint. The host only pins that response and waits for authenticated connectivity. No device filesystem or service-management shell snippets belong in the host. -Authenticated USB-network SSH invokes `orchard-device OPERATION BASE64_JSON`. Requests include `protocol: 1`. `prepare` and `install` carry their byte stream on stdin. Other operations do not consume payload data. +Authenticated USB-network SSH invokes `orchard-device OPERATION BASE64_JSON`. Requests include `protocol: 1`. `prepare` and streamed `install` calls carry their byte stream on stdin. A capability-negotiated cached `install` has no stdin payload. Other operations do not consume payload data. Stdout is newline-delimited JSON: - `phase`: named device activity, no fabricated percentage. -- `progress`: a bounded descriptive message. +- `progress`: a bounded descriptive message, or structured cached-extraction byte progress. - `result`: one completed operation result. - `error`: actionable error, with `retry: false`. @@ -33,8 +33,9 @@ All records include `protocol: 1`. Lost connections during mutations mean unknow | `prepare` | release, bytes, sha256 + mixin stdin | Verify fresh-ADT binding/data allowlist; apply in RAM, prepare hardware once, and clean owned leftover staging before target selection. | | `targets` | none | Boot ID, device-issued selection token, capacity estimate and GPT Linux-data partition/signature descriptions; free space for verified ext4. | | `format-target` | token, target ID, boot ID, operation ID, explicit confirmation | Once per recovery boot after preparation: revalidate a Linux-data target without recognized signatures, create ext4 without force/discard, verify UUID and restore fencing. Return a matching receipt; invalidate the old selection. | -| `destination` | token, target ID, relative directory | Read-only inspection; return existence, destination identity, free/reclaimable bytes, estimated needed bytes and inode sufficiency. No ownership is inferred from installation markers. | -| `install` | selected target/directory, destination_identity, explicit replace boolean, bytes, sha256 + root stdin | Revalidate selection; stream into fresh staging, verify, apply mixin and promote. | +| `destination` | token, target ID, relative directory, optional `cache_root: true` | Read-only inspection; return existence, destination identity, free/reclaimable bytes, estimated needed bytes and inode sufficiency. Cache-aware estimates also return separately reclaimable owned cache bytes. No ownership is inferred from installation markers. | +| `root-cache` | selected target/directory, bytes, sha256 | With `root-image-cache-v1` capability: read-only verification of the exact public root archive. Returns `cached` (boolean), `bytes` and `sha256`. | +| `install` | selected target/directory, destination_identity, explicit replace boolean, bytes, sha256; optional `root_cache: store` or `reuse` | Revalidate selection; extract into fresh staging, verify, apply mixin and promote. Legacy/`store` calls send root stdin; `reuse` calls send no payload. | | `handoff` | none | Recheck the completed installation, preserve diagnostics and ask recovery PID 1 to switch root. | | `diagnostics` | none | Phase/failure state and bounded kernel/system logs. | | `abort` | none | Device-owned incomplete-staging cleanup, unmount/fence storage, retain used-phase guards and require a fresh recovery boot. | @@ -42,11 +43,47 @@ All records include `protocol: 1`. Lost connections during mutations mean unknow Host code must not infer device paths or encode module loads, filesystem offsets, mounts, permissions or PID 1 commands. Target IDs are displayable identifiers, not host-authored storage paths. +## Device root-image cache + +Only use cache operations when the installer advertises `root-image-cache-v1` +in `status.capabilities`. Otherwise the host retains the existing streaming +workflow. Old hosts may omit all cache fields when using a new runtime. + +The host requests cache-aware capacity estimates before confirmation, then +probes the selected partition using the exact release root hash and size. It +checks all probe response fields before choosing `reuse` or `store`. It never +sends device cache paths. A verified hit skips the USB root payload, but not +local integrity checking or extraction. A miss transfers the whole image; +there is no delta-update protocol. + +The runtime owns the persistent cache outside the installed root and rechecks +cached bytes before replacement. Store mode retains only a verified public +archive; private firmware/mixins remain separate. Missing or corrupt completed +entries are cache misses, while unsafe storage paths fail closed. Incomplete +entries are never cache hits. Capacity checks include the compressed archive, +and cleanup is limited to owned cache entries. Replacing the installation +directory does not remove this cache. + +Progress identifies cache verification, uploading and local extraction. During +cached extraction, optional `progress` records carry `phase: cached-extraction`, +integer `bytes` and `total` for compressed bytes consumed, and boolean `complete`. +An initial zero starts the meter; intermediate counts are monotonic and below +the fixed total. Only a successful extractor completion emits the total with +`complete: true`. This does not mean installation finalization or boot has +completed. The host labels the rate as cached extraction, not USB transfer, +limits terminal refreshes and leaves failed/unfinished operations below 100%. +Old phase-only runtimes remain supported; malformed or unknown events stay in +diagnostics rather than producing a misleading meter. + +A failed or disconnected install is never automatically retried with a stream, +even after a successful cache probe. Cache support is not an installation backup +or a change to replacement confirmation semantics. + ## Streaming safety - Root size, digest, compression and expansion limit are baked into recovery from the public release receipt. - Package-defined privileged entries have a public metadata manifest. Producers validate contents before restoring special modes from pacman MTREE; runtime verifies content, UID/GID and modes before activation. Missing `sudo` setuid/root ownership is a finalization error, not a broken system to promote. -- Decode Zstandard and extract concurrently with incoming bytes; no temporary compressed archive. +- Decode Zstandard and extract concurrently with incoming bytes. Negotiated `store` mode also retains a temporary compressed archive and promotes it only after validation; legacy streaming retains no archive. `reuse` extracts a verified local archive instead. - All file writes use no-follow directory descriptors; links are deferred. Traversal, duplicate paths, special files, reserved metadata and escaping hardlinks are rejected. - Verify compressed-byte count/digest and decoder completion before creating links or promoting. Preserve file ownership, modes and timestamps. - Replacement requires explicit approval and matching destination identity. Delete the selected directory before extraction; never create an automatic backup. Reject symlinks, non-directories and nested mounts. Estimate required bytes as expanded payload × 1.2 + 1 GiB and check free inodes (archive member count + 1024; legacy fallback 65536 + 1024). Recheck actual capacity after deletion. Failures must warn that replacement deletion cannot be undone. diff --git a/src/orchard_cli/progress.py b/src/orchard_cli/progress.py index e3263b3..cd069f3 100644 --- a/src/orchard_cli/progress.py +++ b/src/orchard_cli/progress.py @@ -50,6 +50,8 @@ def __init__(self, output=None, *, verbose=False): self.rows = {} self.rendered = 0 self.log_path = None + self.device_phase = None + self.device_progress = None def task(self, name): return TaskProgress(self, name) @@ -66,7 +68,7 @@ def phase(self, message): self.end_line() print(message, file=self.output, flush=True) - def diagnostic(self, message): + def diagnostic(self, message, *, display=True): with self.lock: if self.log_path is None: from .cache import cache_root @@ -76,7 +78,7 @@ def diagnostic(self, message): os.close(fd) with open(self.log_path, 'a') as stream: stream.write(str(message) + '\n') - if self.verbose: + if self.verbose and display: self.phase(str(message)) def transfer(self, sent, total): @@ -102,8 +104,58 @@ def reset_transfer(self): self.end_line() self.started, self.last, self.initial = time.monotonic(), None, None + def reset_device_progress(self): + """End a device meter without inventing completion, including on RPC failure.""" + with self.lock: + if self.device_progress is not None: + self.end_line() + self.device_phase = self.device_progress = None + + def cached_extraction(self, record): + """Render validated compressed-byte consumption, not USB transfer speed.""" + if self.device_phase != 'cached-extraction': + return False + count, total, complete = record.get('bytes'), record.get('total'), record.get('complete') + if (type(count) is not int or type(total) is not int or type(complete) is not bool or + not 0 < total <= 2**63 - 1 or not 0 <= count <= total or + (complete and count != total) or (not complete and count == total)): + return False + now = time.monotonic() + with self.lock: + state = self.device_progress + if state is None: + if count != 0 or complete: + return False + state = self.device_progress = {'bytes': 0, 'total': total, 'started': now, + 'last': None, 'complete': False} + elif (state['total'] != total or count < state['bytes'] or state['complete']): + return False + state['bytes'], state['complete'] = count, complete + interval = 0.2 if self.interactive else 10 + if not complete and state['last'] is not None and now - state['last'] < interval: + return True + state['last'] = now + message = 'Cached extraction ' + transfer_text(count, total, state['started'], 0, now, + complete=complete) + if self.interactive: + print('\r\033[2K' + message, end='', file=self.output, flush=True) + self.active_line = True + if complete: + self.end_line() + else: + print(message, file=self.output, flush=True) + return True + def event(self, record): - # Internal hardware events are diagnostics, not user-facing phases. + if record.get('event') == 'phase': + self.reset_device_progress() + self.device_phase = record.get('phase') + if (record.get('event') == 'progress' and record.get('phase') == 'cached-extraction' and + self.cached_extraction(record)): + # Keep structured diagnostics without interrupting the live meter in verbose mode. + self.diagnostic(str(record), display=False) + return + # Old and unknown hardware events retain their diagnostic-only behavior. self.diagnostic(str(record)) def row(self, name, message, *, complete=False): @@ -120,11 +172,12 @@ def row(self, name, message, *, complete=False): print(f' {name:<20.20} {message}', file=self.output, flush=True) -def transfer_text(sent, total, started, initial, now): +def transfer_text(sent, total, started, initial, now, *, complete=None): unit, divisor = ('KiB', 1024) if total < 1048576 else ('MiB', 1048576) rate = max(0, sent - initial) / max(now - started, 0.001) eta = f'{(total - sent) / rate:.0f}s' if rate and sent < total else '—' - return (f'{sent * 100 / max(total, 1):3.0f}% ' + percent = sent * 100 / max(total, 1) if complete is None else (100 if complete else sent * 100 // total) + return (f'{percent:3.0f}% ' f'{sent / divisor:5.0f} / {total / divisor:5.0f} {unit} ' f'{rate / 1048576:5.1f} MiB/s {eta:>5}') @@ -156,5 +209,5 @@ def transfer(self, sent, total): self.parent.row(self.name, transfer_text(sent, total, self.started, self.initial, now), complete=sent == total or not self.parent.interactive) - def diagnostic(self, message): - self.parent.diagnostic(message) + def diagnostic(self, message, *, display=True): + self.parent.diagnostic(message, display=display) diff --git a/src/orchard_cli/protocol.py b/src/orchard_cli/protocol.py index abeae10..eba23a9 100644 --- a/src/orchard_cli/protocol.py +++ b/src/orchard_cli/protocol.py @@ -21,7 +21,7 @@ def __init__(self, connection, progress): self.ssh, self.progress = connection.argv, progress def call(self, operation, request=None, *, source=None, size=0, timeout=120): - if operation not in ('status', 'prepare', 'targets', 'destination', 'install', 'handoff', 'diagnostics', 'abort', 'bootstrap', 'revoke-ssh', 'format-target', 'installations', 'launch'): + if operation not in ('status', 'prepare', 'targets', 'destination', 'root-cache', 'install', 'handoff', 'diagnostics', 'abort', 'bootstrap', 'revoke-ssh', 'format-target', 'installations', 'launch'): raise ValueError('Unsupported phase operation') try: self.connection.check() @@ -29,6 +29,7 @@ def call(self, operation, request=None, *, source=None, size=0, timeout=120): raise DeviceError(str(error)) from error payload = base64.urlsafe_b64encode(json.dumps({'protocol': PROTOCOL, **(request or {})}).encode()).decode() argv = self.ssh + ['/usr/bin/orchard-device ' + operation + ' ' + payload] + self.progress.reset_device_progress() process = subprocess.Popen(argv, stdin=subprocess.PIPE if source else subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE) records, errors, stderr = [], [], [] @@ -89,6 +90,7 @@ def collect_errors(): if process.poll() is None: process.kill() process.wait() + self.progress.reset_device_progress() for stream in (process.stdin, process.stdout, process.stderr): if stream: stream.close() diff --git a/src/orchard_cli/service.py b/src/orchard_cli/service.py index 0e832ce..4222b49 100644 --- a/src/orchard_cli/service.py +++ b/src/orchard_cli/service.py @@ -15,6 +15,9 @@ from .devices import choose +ROOT_CACHE_CAPABILITY = 'root-image-cache-v1' + + def ask(clock, prompt, input_fn=input): with clock.exclude('input'): return input_fn(prompt).strip() @@ -58,7 +61,8 @@ def initialize_partition(client, clock, observation, row, report_path, *, input_ return refreshed, matches[0] -def select_destination(client, clock, *, input_fn=input, output=print, directory='linux', format_report=None): +def select_destination(client, clock, *, input_fn=input, output=print, directory='linux', format_report=None, + cache_root=False): observation = client.call('targets') rows = observation['partitions'] if not rows: @@ -83,7 +87,11 @@ def select_destination(client, clock, *, input_fn=input, output=print, directory raise ValueError('Selected partition has an unsupported signature; nothing was formatted') while True: request = {'selection_token': observation['selection_token'], 'target': row['id'], 'directory': directory} - existing = client.call('destination', request) + if cache_root: + request['cache_root'] = True + # Cache-aware inspection may hash a large archive on the device. + existing = (client.call('destination', request, timeout=1200) if cache_root + else client.call('destination', request)) replace = existing['exists'] if replace: output(f'/{directory} already exists.') @@ -96,11 +104,17 @@ def select_destination(client, clock, *, input_fn=input, output=print, directory continue if action != '1': raise ValueError('Installation cancelled; nothing was deleted') - available = existing['free_bytes'] + (existing['reclaimable_bytes'] if replace else 0) + cache_reclaimable = existing.get('cache_reclaimable_bytes', 0) if cache_root else 0 + if type(cache_reclaimable) is not int or cache_reclaimable < 0: + raise ValueError('Invalid device cache capacity estimate; nothing was deleted') + available = (existing['free_bytes'] + (existing['reclaimable_bytes'] if replace else 0) + + cache_reclaimable) needed = existing['needed_bytes'] output('Installation plan:') output(f' Destination: /{directory}') label = 'Available after replacement' if replace else 'Available' + if cache_reclaimable: + label += ' and cache cleanup' if replace else ' after cache cleanup' output(f' {label}: {available / 1024**3:.1f} GiB') output(f' Estimated needed, including margin: {needed / 1024**3:.1f} GiB') if available < needed or not existing['inodes_sufficient']: @@ -108,7 +122,7 @@ def select_destination(client, clock, *, input_fn=input, output=print, directory 'Free space or choose another partition or directory.') if replace: phrase = 'replace ' + directory - output('No backup will be created. Other directories will not be touched.') + output('No backup will be created. Other installation directories will not be touched.') output('If installation fails, the deleted contents cannot be restored.') else: phrase = 'install ' + directory @@ -118,6 +132,27 @@ def select_destination(client, clock, *, input_fn=input, output=print, directory return {**request, 'replace': replace, 'destination_identity': existing['identity']} +def install_root(client, destination, receipt, path, progress, *, cache_enabled=False): + """Negotiate public root reuse; device paths and validation remain device-owned.""" + request = {**destination, **receipt} + if cache_enabled: + progress.phase('Checking device root-image cache') + cached = client.call('root-cache', request, timeout=1200) + if (not isinstance(cached, dict) or type(cached.get('cached')) is not bool + or cached.get('sha256') != receipt['sha256'] + or type(cached.get('bytes')) is not int or cached['bytes'] != receipt['bytes']): + raise ValueError('Device root-cache response does not match the selected image') + request['root_cache'] = 'reuse' if cached['cached'] else 'store' + if cached['cached']: + progress.phase('Using verified cached root image; extracting locally') + # No stdin payload, and no automatic streaming retry if the cache + # changes or installation fails after the read-only probe. + return client.call('install', request, timeout=1200) + progress.phase('Streaming root image and saving it in the device cache') + with path.open('rb') as source: + return client.call('install', request, source=source, size=receipt['bytes'], timeout=1200) + + def save_report(path, report): # Persist commitment before handoff; a reporting/transport failure must not # erase evidence that the root was successfully installed. @@ -259,12 +294,13 @@ def install(manifest=None, *, input_fn=input, progress=None): client.call('prepare', {'release': record['release'], **mixin_record}, source=source, size=mixin_record['bytes'], timeout=180) root_path = record.get('default_root', '/linux') + capabilities = status.get('capabilities', []) + cache_enabled = isinstance(capabilities, list) and ROOT_CACHE_CAPABILITY in capabilities destination = select_destination(client, clock, input_fn=input_fn, directory=root_path.lstrip('/'), - format_report=private / 'last-format.json') + format_report=private / 'last-format.json', cache_root=cache_enabled) progress.phase('Installing ' + record.get('name', 'Linux') + '…') - with assets['root'].open('rb') as source: - result = client.call('install', {**destination, **record['assets']['root']}, - source=source, size=record['assets']['root']['bytes'], timeout=1200) + result = install_root(client, destination, record['assets']['root'], assets['root'], + progress, cache_enabled=cache_enabled) snapshot = clock.result() save_report(private / 'last-install.json', { **identity, 'installed': True, 'ready': None, 'release': record['release'], diff --git a/tests/test_cached_progress.py b/tests/test_cached_progress.py new file mode 100644 index 0000000..abd5525 --- /dev/null +++ b/tests/test_cached_progress.py @@ -0,0 +1,182 @@ +"""Device-local extraction meters remain accurate, bounded and backwards compatible.""" +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +from orchard_cli.contracts import Connection +from orchard_cli.progress import Progress +from orchard_cli.protocol import Client, DeviceError +from orchard_cli.state import BootState, DeviceState + + +class Terminal(io.StringIO): + def isatty(self): + return True + + +def phase(name='cached-extraction'): + return {'protocol': 1, 'event': 'phase', 'phase': name} + + +def event(count, total=1048576, complete=False): + return {'protocol': 1, 'event': 'progress', 'phase': 'cached-extraction', + 'bytes': count, 'total': total, 'complete': complete} + + +class CachedProgress(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory() + self.addCleanup(self.folder.cleanup) + self.cache = patch('orchard_cli.cache.cache_root', return_value=Path(self.folder.name)) + self.cache.start() + self.addCleanup(self.cache.stop) + + def test_tty_updates_and_verbose_does_not_print_raw_progress(self): + output = Terminal() + progress = Progress(output, verbose=True) + progress.event(phase()) + with patch('orchard_cli.progress.time.monotonic', side_effect=[10, 11, 12]): + for record in (event(0), event(524288), event(1048576, complete=True)): + progress.event(record) + text = output.getvalue() + self.assertEqual(text.count('Cached extraction'), 3) + self.assertIn(' 50%', text) + self.assertIn('100%', text) + self.assertNotIn("'bytes':", text) + self.assertNotIn('USB', text) + self.assertIn("'bytes': 524288", Path(progress.log_path).read_text()) + self.assertFalse(progress.active_line) + + def test_nonterminal_output_is_sparse_but_completion_is_always_visible(self): + output = io.StringIO() + progress = Progress(output) + progress.event(phase()) + with patch('orchard_cli.progress.time.monotonic', side_effect=[0, 1, 11, 12]): + for record in (event(0), event(10), event(20), event(1048576, complete=True)): + progress.event(record) + text = output.getvalue() + self.assertEqual(text.count('Cached extraction'), 3) + self.assertNotIn('\r', text) + self.assertIn('100%', text) + + def test_tty_updates_are_throttled(self): + output = Terminal() + progress = Progress(output) + progress.event(phase()) + with patch('orchard_cli.progress.time.monotonic', side_effect=[0, 0.1, 0.25, 0.26]): + for record in (event(0), event(10), event(20), event(1048576, complete=True)): + progress.event(record) + self.assertEqual(output.getvalue().count('Cached extraction'), 3) + + def test_near_completion_does_not_round_up_to_100(self): + output = Terminal() + progress = Progress(output) + progress.event(phase()) + with patch('orchard_cli.progress.time.monotonic', side_effect=[0, 1]): + progress.event(event(0)) + progress.event(event(1048575)) + self.assertIn(' 99%', output.getvalue()) + self.assertNotIn('100%', output.getvalue()) + progress.reset_device_progress() + self.assertFalse(progress.active_line) + self.assertNotIn('100%', output.getvalue()) + + def test_invalid_records_are_diagnostics_not_meter_updates(self): + output = Terminal() + progress = Progress(output) + progress.event(phase()) + progress.event(event(0)) + records = [event(True), event(-1), event(1, total=True), event(1, total=0), + event(1, total=2**64), event(1.5), event('1'), event(2, total=1), + event(1, complete=True), event(1048576), event(0, complete='yes')] + before = output.getvalue() + for record in records: + progress.event(record) + self.assertEqual(output.getvalue(), before) + self.assertEqual(progress.device_progress['bytes'], 0) + self.assertIn("'bytes': -1", Path(progress.log_path).read_text()) + + def test_backwards_changed_total_and_duplicate_completion_are_ignored(self): + output = Terminal() + progress = Progress(output) + progress.event(phase()) + with patch('orchard_cli.progress.time.monotonic', side_effect=range(20)): + progress.event(event(0)) + progress.event(event(100)) + before = output.getvalue() + progress.event(event(99)) + progress.event(event(101, total=2097152)) + self.assertEqual(output.getvalue(), before) + progress.event(event(1048576, complete=True)) + before = output.getvalue() + progress.event(event(1048576, complete=True)) + progress.event(event(0)) + self.assertEqual(output.getvalue(), before) + + def test_phase_transition_ends_incomplete_line_and_resets(self): + output = Terminal() + progress = Progress(output) + progress.event(phase()) + progress.event(event(0)) + progress.event(phase('finalize')) + self.assertFalse(progress.active_line) + self.assertIsNone(progress.device_progress) + self.assertNotIn('100%', output.getvalue()) + progress.event(phase()) + progress.event(event(0, total=2097152)) + self.assertEqual(progress.device_progress['total'], 2097152) + + def test_old_and_unknown_events_stay_compatible(self): + output = Terminal() + progress = Progress(output) + for record in (phase(), {'protocol': 1, 'event': 'progress', 'message': 'touch'}, + {'protocol': 1, 'event': 'progress', 'phase': 'future', 'bytes': 1}, + phase('finalize')): + progress.event(record) + self.assertEqual(output.getvalue(), '') + self.assertIsNone(progress.device_progress) + + def test_unsolicited_progress_or_missing_initial_zero_is_ignored(self): + output = Terminal() + progress = Progress(output) + progress.event(event(0)) + progress.event(phase()) + progress.event(event(100)) + self.assertEqual(output.getvalue(), '') + self.assertIsNone(progress.device_progress) + + def test_rpc_error_closes_partial_meter_and_next_call_can_restart(self): + output = Terminal() + progress = Progress(output) + scope = BootState(DeviceState('device', 1), '11111111-1111-4111-8111-111111111111') + records = [phase(), event(0), event(100), + {'protocol': 1, 'event': 'error', 'message': 'integrity mismatch'}] + program = 'import sys; print(' + repr('\n'.join(map(json.dumps, records))) + '); sys.exit(1)' + client = Client(Connection([sys.executable, '-c', program], scope, lambda: None), progress) + for _ in range(2): + with self.assertRaisesRegex(DeviceError, 'integrity mismatch'): + client.call('install', timeout=5) + self.assertFalse(progress.active_line) + self.assertIsNone(progress.device_progress) + self.assertEqual(output.getvalue().count('Cached extraction'), 2) + self.assertNotIn('100%', output.getvalue()) + + def test_rpc_success_resets_completed_meter(self): + output = Terminal() + progress = Progress(output) + scope = BootState(DeviceState('device', 1), '11111111-1111-4111-8111-111111111111') + records = [phase(), event(0), event(1048576, complete=True), phase('finalize'), + {'protocol': 1, 'event': 'result', 'operation': 'install', 'result': {'ok': True}}] + program = 'print(' + repr('\n'.join(map(json.dumps, records))) + ')' + client = Client(Connection([sys.executable, '-c', program], scope, lambda: None), progress) + self.assertTrue(client.call('install', timeout=5)['ok']) + self.assertIsNone(progress.device_progress) + self.assertIn('100%', output.getvalue()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_host_workflow.py b/tests/test_host_workflow.py index aa811a6..0e39d86 100644 --- a/tests/test_host_workflow.py +++ b/tests/test_host_workflow.py @@ -45,7 +45,12 @@ def test_guided_run_reaches_desktop_with_one_mixin_and_one_root_stream(self): with self.subTest(filesystem=filesystem): self.guided_run(filesystem) - def guided_run(self, filesystem): + def test_guided_run_with_cache_hit_and_miss(self): + for cached in (True, False): + with self.subTest(cached=cached): + self.guided_run('ext4', cached=cached) + + def guided_run(self, filesystem, cached=None): from contextlib import contextmanager from unittest.mock import Mock with tempfile.TemporaryDirectory() as directory: @@ -76,7 +81,9 @@ class Device: prepared = installed = handed_off = formatted = False def call(self, operation, request=None, **kwargs): if operation == 'status': - return {'release': '1.0', 'boot_id': BOOT, 'ecid':1, 'profile':'device', 'ready': self.handed_off} + return {'release': '1.0', 'boot_id': BOOT, 'ecid':1, 'profile':'device', + 'ready': self.handed_off, + 'capabilities': ['root-image-cache-v1'] if cached is not None else []} if operation == 'prepare': assert not self.prepared and kwargs['source'].read() == b'private-data' self.prepared = True @@ -98,10 +105,20 @@ def call(self, operation, request=None, **kwargs): 'filesystem_uuid': '11111111-1111-4111-8111-111111111111'} if operation == 'destination': assert request['selection_token'] == ('new-token' if self.formatted else 'token') + assert request.get('cache_root', False) == (cached is not None) return {'exists': False, 'identity': None, 'free_bytes': 10 * 1024**3, 'reclaimable_bytes': 0, 'needed_bytes': 2 * 1024**3, 'inodes_sufficient': True} + if operation == 'root-cache': + assert self.prepared and not self.installed and cached is not None + assert 'source' not in kwargs and request['cache_root'] is True + return {'cached': cached, 'sha256': 'root-digest', 'bytes': 11} if operation == 'install': - assert self.prepared and not self.installed and kwargs['source'].read() == b'root-stream' + assert self.prepared and not self.installed + if cached is True: + assert request['root_cache'] == 'reuse' and 'source' not in kwargs + else: + assert kwargs['source'].read() == b'root-stream' + assert request.get('root_cache') == ('store' if cached is False else None) self.installed = True return {'status': 'complete'} if operation == 'handoff': diff --git a/tests/test_root_cache.py b/tests/test_root_cache.py new file mode 100644 index 0000000..0e92e55 --- /dev/null +++ b/tests/test_root_cache.py @@ -0,0 +1,164 @@ +"""Device-cache negotiation never turns an uncertain install into a retry.""" +import hashlib +import io +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock + +from orchard_cli.contracts import Connection +from orchard_cli.progress import Clock, Progress +from orchard_cli.protocol import Client, DeviceError +from orchard_cli.service import install_root, select_destination +from orchard_cli.state import BootState, DeviceState + + +class RootCache(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.path = Path(self.temp.name) / 'root.tar.zst' + self.payload = b'public-root-image' + self.path.write_bytes(self.payload) + self.receipt = {'sha256': hashlib.sha256(self.payload).hexdigest(), 'bytes': len(self.payload)} + self.destination = {'selection_token': 'token', 'target': 'partition', 'directory': 'omarchy', + 'replace': True, 'destination_identity': [1, 2, 3, 4], 'cache_root': True} + self.output = io.StringIO() + self.progress = Progress(self.output) + + def run_install(self, client, enabled=True): + return install_root(client, self.destination, self.receipt, self.path, + self.progress, cache_enabled=enabled) + + def test_hit_does_not_open_or_send_host_archive(self): + self.path.unlink() # Reuse must not even open the local archive here. + client = Mock() + client.call.side_effect = [{**self.receipt, 'cached': True}, {'status': 'complete', 'root_cache': 'reused'}] + result = self.run_install(client) + self.assertEqual(result['root_cache'], 'reused') + self.assertEqual([c.args[0] for c in client.call.call_args_list], ['root-cache', 'install']) + request = client.call.call_args.args[1] + self.assertEqual(request['root_cache'], 'reuse') + self.assertEqual(request['destination_identity'], self.destination['destination_identity']) + self.assertEqual(client.call.call_args.kwargs, {'timeout': 1200}) + self.assertIn('extracting locally', self.output.getvalue()) + + def test_miss_transfers_complete_archive_once(self): + client = Mock() + streams = [] + def call(operation, request, **kwargs): + self.assertEqual(request['sha256'], self.receipt['sha256']) + self.assertEqual(request['bytes'], self.receipt['bytes']) + if operation == 'root-cache': + self.assertNotIn('source', kwargs) + return {**self.receipt, 'cached': False} + self.assertEqual(operation, 'install') + self.assertEqual(request['root_cache'], 'store') + self.assertEqual(kwargs['size'], len(self.payload)) + streams.append(kwargs['source'].read()) + return {'status': 'complete'} + client.call.side_effect = call + self.run_install(client) + self.assertEqual(streams, [self.payload]) + self.assertIn('saving it in the device cache', self.output.getvalue()) + + def test_changed_image_probes_new_receipt_and_uploads_on_miss(self): + self.payload = b'changed-build' + self.path.write_bytes(self.payload) + self.receipt = {'sha256': hashlib.sha256(self.payload).hexdigest(), 'bytes': len(self.payload)} + self.test_miss_transfers_complete_archive_once() + + def test_old_runtime_uses_original_stream_without_cache_operation(self): + client = Mock() + self.destination.pop('cache_root') + def call(operation, request, **kwargs): + self.assertEqual(operation, 'install') + self.assertNotIn('root_cache', request) + self.assertNotIn('cache_root', request) + self.assertEqual(kwargs['source'].read(), self.payload) + return {'status': 'complete'} + client.call.side_effect = call + self.run_install(client, enabled=False) + self.assertEqual(client.call.call_count, 1) + + def test_invalid_probe_cannot_select_reuse_or_trigger_upload(self): + good = {**self.receipt, 'cached': True} + invalid = [None, [], {}, {**good, 'cached': 1}, {**good, 'cached': 'false'}, + {**good, 'sha256': '0' * 64}, {**good, 'bytes': 1}, + {**good, 'bytes': str(self.receipt['bytes'])}, {**good, 'bytes': True}] + for response in invalid: + with self.subTest(response=response): + client = Mock() + client.call.return_value = response + with self.assertRaisesRegex(ValueError, 'does not match'): + self.run_install(client) + self.assertEqual(client.call.call_count, 1) + + def test_probe_error_stops_without_mutation(self): + client = Mock() + client.call.side_effect = DeviceError('unsafe cache path', reported=True) + with self.assertRaisesRegex(DeviceError, 'unsafe cache'): + self.run_install(client) + self.assertEqual(client.call.call_count, 1) + + def test_failed_cached_or_streamed_install_is_never_retried(self): + for hit in (True, False): + for reported in (True, False): + with self.subTest(hit=hit, reported=reported): + client = Mock() + client.call.side_effect = [{**self.receipt, 'cached': hit}, + DeviceError('install failed', reported=reported)] + with self.assertRaisesRegex(DeviceError, 'install failed'): + self.run_install(client) + self.assertEqual(client.call.call_count, 2) + + def test_destination_requests_cache_capacity_before_confirmation(self): + client = Mock() + def call(operation, request=None, **kwargs): + if operation == 'targets': + return {'selection_token': 'token', 'partitions': [ + {'id': 'partition', 'bytes': 20 * 1024**3, 'filesystem': 'ext4', 'label': 'Linux'}]} + self.assertEqual(operation, 'destination') + self.assertIs(request['cache_root'], True) + self.assertEqual(kwargs['timeout'], 1200) + return {'exists': False, 'identity': None, 'free_bytes': 10 * 1024**3, + 'reclaimable_bytes': 0, 'needed_bytes': 3 * 1024**3, 'inodes_sufficient': True} + client.call.side_effect = call + answers = iter(('', 'install omarchy')) + result = select_destination(client, Clock(), directory='omarchy', cache_root=True, + input_fn=lambda _: next(answers), output=lambda _: None) + self.assertIs(result['cache_root'], True) + self.assertEqual([c.args[0] for c in client.call.call_args_list], ['targets', 'destination']) + + def test_protocol_accepts_read_only_probe_without_payload(self): + program = ('import sys,json; assert sys.stdin.buffer.read() == b""; ' + 'print(json.dumps({"protocol":1,"event":"result","operation":"root-cache",' + '"result":{"cached":False,"sha256":"abc","bytes":3}}))') + scope = BootState(DeviceState('device', 1), '11111111-1111-4111-8111-111111111111') + client = Client(Connection([sys.executable, '-c', program], scope, lambda: None), self.progress) + result = client.call('root-cache', {'sha256': 'abc', 'bytes': 3}, timeout=5) + self.assertIs(result['cached'], False) + + def test_cache_reclaimable_capacity_applies_to_new_directory(self): + for reclaimed, success in ((1024**3, True), (0, False), (-1, False), ('lots', False)): + with self.subTest(reclaimed=reclaimed): + client = Mock() + client.call.side_effect = [ + {'selection_token': 'token', 'partitions': [ + {'id': 'partition', 'bytes': 10 * 1024**3, 'filesystem': 'ext4', 'label': 'Linux'}]}, + {'exists': False, 'identity': None, 'free_bytes': 2 * 1024**3, + 'reclaimable_bytes': 0, 'cache_reclaimable_bytes': reclaimed, + 'needed_bytes': 3 * 1024**3, 'inodes_sufficient': True}] + answers = iter(('', 'install linux')) + invoke = lambda: select_destination(client, Clock(), cache_root=True, + input_fn=lambda _: next(answers), output=lambda _: None) + if success: + self.assertFalse(invoke()['replace']) + else: + with self.assertRaisesRegex(ValueError, 'nothing was deleted'): + invoke() + + +if __name__ == '__main__': + unittest.main() From 8b6d615458ef37150deab2c1f51816c2e8cf6593 Mon Sep 17 00:00:00 2001 From: Stuart Fraser <60394096+stuart902@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:48:47 -0300 Subject: [PATCH 2/2] Build pinned development dependencies in pull request CI --- .github/workflows/build.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e9e7dec..76adc64 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,28 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + # Development can require a hardware version before its release exists. + - name: Check out pinned development hardware dependency + if: github.ref_type != 'tag' && github.event_name != 'workflow_dispatch' + uses: actions/checkout@v4 + with: + repository: OrchardLinux/hardware + ref: f3f14c91940e3d9ec83db6980abb2e8857fad439 + path: out/hardware-source + persist-credentials: false + - name: Build pinned development hardware dependency + if: github.ref_type != 'tag' && github.event_name != 'workflow_dispatch' + run: | + python -m pip install build + python -m build --wheel --outdir out/hardware out/hardware-source + python - <<'PY' >> "$GITHUB_ENV" + from pathlib import Path + import tomllib + project = tomllib.loads(Path('out/hardware-source/pyproject.toml').read_text()) + print('HARDWARE_TAG=v' + project['project']['version']) + PY - name: Download pinned released hardware dependency - if: inputs.hardware_run_id == '' + if: (github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && inputs.hardware_run_id == '' env: GH_TOKEN: ${{ github.token }} run: |