diff --git a/deploy/install-release.sh b/deploy/install-release.sh index daf02eacd..ebcc28097 100755 --- a/deploy/install-release.sh +++ b/deploy/install-release.sh @@ -5,16 +5,24 @@ command -v python3 >/dev/null || { echo 'Python 3.9+ is required.' >&2; exit 1; # Keep the caller's stdin available to the bundled installer. exec python3 /dev/fd/3 "$@" 3<<'PY' import argparse +import contextlib +import errno +import fcntl import hashlib +import http.client import json import os import pathlib import platform import re import shutil +import signal +import stat +import subprocess import sys import tarfile import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -28,6 +36,51 @@ class ReleaseError(Exception): pass +class TransferError(ReleaseError): + pass + + +class Progress: + """Byte progress for the standalone downloader, without terminal escapes in logs.""" + def __init__(self, label, total=None): + self.label, self.total = label, total + self.started = time.monotonic() + self.updated = 0 + self.visible = sys.stderr.isatty() and os.environ.get("TERM") != "dumb" + + def update(self, size): + now = time.monotonic() + if not self.visible or now - self.updated < 0.2 and size != self.total: + return + self.updated = now + bar = "" + if self.total: + percent = min(100, size * 100 // self.total) + filled = percent // 5 + bar = "[" + "=" * filled + " " * (20 - filled) + f"] {percent:3d}% " + print(f"\r{self.label}: {bar}{size / 1048576:.1f} MiB ({now - self.started:.0f}s)", + end="\033[K", file=sys.stderr, flush=True) + + def __enter__(self): + return self + + def __exit__(self, *exc): + if self.visible: + with contextlib.suppress(OSError): + print(file=sys.stderr, flush=True) + + +def retry(operation): + for attempt in range(3): + try: + return operation() + except (TransferError, TimeoutError, ConnectionError, http.client.HTTPException): + if attempt == 2: + raise ReleaseError("Download interrupted after 3 attempts. Check the network and rerun the command.") from None + print("==> Download interrupted; retrying...", flush=True) + time.sleep(attempt + 1) + + class DownloadRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, request, fp, code, msg, headers, newurl): if urllib.parse.urlsplit(newurl).scheme != "https": @@ -36,24 +89,43 @@ class DownloadRedirect(urllib.request.HTTPRedirectHandler): return redirected -def open_url(url, binary=False): +def open_url(url, binary=False, offset=0): headers = {"Accept": "application/octet-stream" if binary else "application/vnd.github+json", "User-Agent": "OpenAgentCore-installer", "X-GitHub-Api-Version": "2022-11-28"} + if offset: + headers["Range"] = f"bytes={offset}-" request = urllib.request.Request(url, headers=headers) try: return urllib.request.build_opener(DownloadRedirect()).open(request, timeout=60) except urllib.error.HTTPError as error: + if error.code in (408, 429, 500, 502, 503, 504): + error.close() + raise TransferError("Temporary GitHub download failure") from None + error.close() if error.code in (401, 403, 404): raise ReleaseError("Release unavailable. Check the version and GitHub access limits.") from None raise ReleaseError("GitHub download failed (HTTP " + str(error.code) + "). Retry later.") from None except urllib.error.URLError: - raise ReleaseError("Could not reach GitHub. Check the network and retry.") from None + raise TransferError("Could not reach GitHub") from None def select_release(version): endpoint = "/releases/latest" if version == "latest" else "/releases/tags/" + urllib.parse.quote(version, safe="") - with open_url(API + endpoint) as response: - release = json.load(response) + def metadata(): + with open_url(API + endpoint) as response: + raw = response.read(4 * 1024 * 1024 + 1) + length = getattr(response, "headers", {}).get("Content-Length") + if len(raw) > 4 * 1024 * 1024: + raise ReleaseError("Release metadata is too large") + if length is not None and len(raw) < int(length): + raise TransferError("Incomplete release metadata") + return json.loads(raw) + release = retry(metadata) + if (not isinstance(release, dict) or not isinstance(release.get("tag_name"), str) + or not isinstance(release.get("assets"), list) + or any(not isinstance(asset, dict) or not isinstance(asset.get("name"), str) + for asset in release["assets"])): + raise ReleaseError("Invalid GitHub release metadata") if release.get("draft") or (version == "latest" and release.get("prerelease")): raise ReleaseError("Select a published release; latest excludes prereleases") assets = release.get("assets", []) @@ -67,24 +139,94 @@ def select_release(version): for asset in (bundle, sums[0]): if type(asset.get("id")) is not int or asset["id"] <= 0: raise ReleaseError("Invalid release asset identity") + if type(asset.get("size")) is not int or asset["size"] <= 0: + raise ReleaseError("Invalid release asset size") return release["tag_name"], bundle, sums[0] def download(asset, destination, limit=None): # Resolve latest once, then download only those immutable asset IDs. - with open_url(API + "/releases/assets/" + str(asset["id"]), binary=True) as response: - with destination.open("wb") as output: - size = 0 - for chunk in iter(lambda: response.read(1024 * 1024), b""): - size += len(chunk) - if limit is not None and size > limit: - raise ReleaseError("Release checksum file is too large") - output.write(chunk) + expected = asset["size"] + if limit is not None and expected > limit: + raise ReleaseError("Release checksum file is too large") + require_space(destination.parent, expected) + + def transfer(): + offset = destination.stat().st_size if destination.exists() else 0 + if offset == expected: + return + with open_url(API + "/releases/assets/" + str(asset["id"]), binary=True, offset=offset) as response: + status = getattr(response, "status", 200) + if status == 206: + match = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", response.headers.get("Content-Range", "")) + if not match or tuple(map(int, match.groups())) != (offset, expected - 1, expected): + raise ReleaseError("Invalid release download range") + elif status == 200: + offset = 0 # Servers may ignore Range; replace the partial file. + else: + raise ReleaseError("Unexpected release download response") + with destination.open("ab" if offset else "wb") as output: + size = offset + for chunk in iter(lambda: response.read1(1024 * 1024), b""): + size += len(chunk) + if size > expected: + raise ReleaseError("Release download exceeds its declared size") + output.write(chunk) + progress.update(size) + if size != expected: + raise TransferError("Incomplete release download") + with Progress("Downloading checksum" if limit else "Downloading archive", expected) as progress: + retry(transfer) + + +def require_space(directory, size): + if shutil.disk_usage(directory).free < size: + raise ReleaseError(f"Not enough disk space in {directory}; need {size} bytes available. Free space and rerun.") + + +@contextlib.contextmanager +def ignore_interrupts(): + handlers = {number: signal.signal(number, signal.SIG_IGN) + for number in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + yield + finally: + for number, handler in handlers.items(): + signal.signal(number, handler) + + +@contextlib.contextmanager +def staging(cache): + # A stable lock makes one reserved staging directory safe to discard after a crash. + descriptor = os.open(cache / ".download.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "rb") as lock: + info = os.fstat(lock.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1: + raise ReleaseError("Release download lock must be an owned regular file") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ReleaseError("Another release download is running; wait for it to finish and retry") from None + temporary = cache / ".download" + if temporary.exists() or temporary.is_symlink(): + info = temporary.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ReleaseError("Release staging directory must be private and owned by you") + with ignore_interrupts(): + shutil.rmtree(temporary) + try: + temporary.mkdir(mode=0o700) + yield temporary, lock.fileno() + finally: + with ignore_interrupts(): + if temporary.exists(): + shutil.rmtree(temporary) def extract(archive, destination, stem): seen = set() - with tarfile.open(archive, "r:gz") as source: + unpacked = 0 + with tarfile.open(archive, "r:gz") as source, Progress("Extracting") as progress: for member in source: path = pathlib.PurePosixPath(member.name) if (not member.isfile() or path.is_absolute() or ".." in path.parts @@ -92,10 +234,14 @@ def extract(archive, destination, stem): or path in seen): raise ReleaseError("Release archive contains an unsafe or duplicate path") seen.add(path) + require_space(destination, member.size) target = destination.joinpath(*path.parts) target.parent.mkdir(parents=True, exist_ok=True) with source.extractfile(member) as data, target.open("xb") as output: - shutil.copyfileobj(data, output) + for chunk in iter(lambda: data.read(1024 * 1024), b""): + output.write(chunk) + unpacked += len(chunk) + progress.update(unpacked) target.chmod(member.mode & 0o777) root = destination / stem if not (root / "install.sh").is_file() or not (root / "manifest.json").is_file(): @@ -103,23 +249,59 @@ def extract(archive, destination, stem): return root +def run_installer(root, arguments, lock): + """Wait for the installer to finish cleanup before removing its source bundle.""" + child = None + interrupted = None + + def forward(signum, frame): + nonlocal interrupted + if interrupted is not None: + return # Let the installer finish cleanup despite repeated interrupts. + interrupted = signum + if child is not None: + with contextlib.suppress(ProcessLookupError): + os.killpg(child.pid, signum) + + handlers = {number: signal.signal(number, forward) + for number in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + # The child retains the lock if this downloader is killed without cleanup. + child = subprocess.Popen(["bash", str(root / "install.sh"), *arguments], + start_new_session=True, pass_fds=(lock,)) + if interrupted is not None: + with contextlib.suppress(ProcessLookupError): + os.killpg(child.pid, interrupted) + code = child.wait() + if interrupted is not None: + raise KeyboardInterrupt + if code: + raise ReleaseError("Bundled installation failed; fix the reported cause and rerun the command") + finally: + for number, handler in handlers.items(): + signal.signal(number, handler) + + def install(version, arguments): if sys.version_info < (3, 9): raise ReleaseError("Python 3.9+ is required") if platform.system() != "Linux" or platform.machine() not in ("x86_64", "amd64"): raise ReleaseError("Core installation currently requires Linux amd64") - print("==> Finding the " + ("latest stable release" if version == "latest" else "requested release") + "...", flush=True) - tag, bundle, sums = select_release(version) - print("Installing OpenAgentCore " + tag, flush=True) home = pathlib.Path.home() / ".oac" home.mkdir(mode=0o700, exist_ok=True) cache = home / "releases" cache.mkdir(mode=0o700, exist_ok=True) - extracted = pathlib.Path(tempfile.mkdtemp(prefix="release-", dir=cache)) + info = cache.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ReleaseError("Release directory must be private and owned by your current account") + extracted = None try: - with tempfile.TemporaryDirectory(prefix=".download-", dir=cache) as temporary: - archive = pathlib.Path(temporary) / bundle["name"] - checksum = pathlib.Path(temporary) / sums["name"] + with staging(cache) as (temporary, lock): + print("==> Finding the " + ("latest stable release" if version == "latest" else "requested release") + "...", flush=True) + tag, bundle, sums = select_release(version) + print("Installing OpenAgentCore " + tag, flush=True) + archive = temporary / bundle["name"] + checksum = temporary / sums["name"] download(sums, checksum, limit=1024) expected = checksum.read_text().strip() match = re.fullmatch(r"([0-9a-f]{64}) " + re.escape(bundle["name"]), expected) @@ -129,26 +311,34 @@ def install(version, arguments): download(bundle, archive) print("==> Verifying the archive checksum...", flush=True) digest = hashlib.sha256() - with archive.open("rb") as source: + with archive.open("rb") as source, Progress("Verifying", bundle["size"]) as progress: + checked = 0 for chunk in iter(lambda: source.read(1024 * 1024), b""): digest.update(chunk) + checked += len(chunk) + progress.update(checked) if digest.hexdigest() != match[1]: raise ReleaseError("Release checksum mismatch; installation was not started") stem = bundle["name"].removesuffix(".tar.gz") print("==> Extracting the verified archive...", flush=True) - root = extract(archive, extracted, stem) + root = extract(archive, temporary, stem) manifest = json.loads((root / "manifest.json").read_text()) - if manifest.get("source_commit") != ARCHIVE.fullmatch(bundle["name"])[1]: + if not isinstance(manifest, dict) or manifest.get("source_commit") != ARCHIVE.fullmatch(bundle["name"])[1]: raise ReleaseError("Release source does not match its bundle") + archive.unlink() + checksum.unlink() + print("==> Starting the bundled installer...", flush=True) + run_installer(root, arguments, lock) + # Only a successful install retains a verified bundle for same-version repair. + extracted = pathlib.Path(tempfile.mkdtemp(prefix="release-", dir=cache)) + root = root.rename(extracted / stem) + print("Verified bundle: " + str(root), flush=True) except BaseException: - shutil.rmtree(extracted) + # A second signal or a closed terminal must not interrupt temporary-file cleanup. + with ignore_interrupts(): + if extracted is not None: + shutil.rmtree(extracted) raise - # Keep the verified extracted bundle for same-version repair. - print("Verified bundle: " + str(root), flush=True) - print("==> Starting the bundled installer...", flush=True) - # Become the installer: it gets Ctrl-C, SIGTERM and SIGHUP itself, and nothing here can stop - # it while it removes a failed new installation. The temporary download is already gone. - os.execvp("bash", ["bash", str(root / "install.sh"), *arguments]) def main(argv): @@ -161,9 +351,17 @@ def main(argv): if __name__ == "__main__": + def interrupted(signum, frame): + raise KeyboardInterrupt + for signum in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(signum, interrupted) try: main(sys.argv[1:]) - except (ReleaseError, OSError, ValueError, tarfile.TarError) as error: - print("Installation failed: " + str(error), file=sys.stderr) - sys.exit(1) + except (ReleaseError, OSError, ValueError, tarfile.TarError, KeyboardInterrupt) as error: + message = "interrupted; rerun the same command" if isinstance(error, KeyboardInterrupt) else str(error) + if isinstance(error, OSError) and error.errno in (errno.ENOSPC, errno.EDQUOT): + message = "Disk space or quota exhausted. Free space in ~/.oac and rerun the command." + with contextlib.suppress(OSError): + print("Installation failed: " + message, file=sys.stderr) + sys.exit(130 if isinstance(error, KeyboardInterrupt) else 1) PY diff --git a/deploy/install/README.md b/deploy/install/README.md index dfa66eb30..831259690 100644 --- a/deploy/install/README.md +++ b/deploy/install/README.md @@ -41,7 +41,7 @@ This directory holds the Core/Web installer, the `oac` command and the node inst ## Versions and the lock -- Install only into an empty directory or over an [incomplete installation](#new-installations), or repair a complete installation of the same source revision. Require the current state format and matching source revision before changing installation files or services. Preserve data when those checks fail. +- Install only into an empty directory or over an [incomplete installation](#new-installations), or repair a complete installation of the same source revision. Require the current state format before changing installation files or services, and a matching source revision when repairing a complete installation. An incomplete installation of that state format is removed before installing the selected release, even when its source revision differs. Preserve data when those checks fail. - The packaged `oac.pyz` embeds its build revision and refuses a `state.json` whose `source_commit` differs. - The installer and every mutating `oac` command share `.oac.lock`. The installer holds it across creation, payload, native service and launcher repair, and apply, calling the already-locked apply implementation without locking again. Never replace the lock file; its inode must stay stable. Only the cleanup of a new installation, with the directory the installer created, and `oac uninstall` unlink it, last and while holding it. `locked` refuses a lock whose path no longer names the file it locked. @@ -50,7 +50,7 @@ This directory holds the Core/Web installer, the `oac` command and the node inst - An installation is complete after its first start: apply converged, the services are healthy and, for Web-only, its Core accepts the Core key. `create()` writes `state.json` before anything else, atomically, with the project name and `"complete": false`, and the installer sets it to `true` at that moment. A later failure, such as a refused sandbox selection, keeps the installation. - Until then, `oac apply`, `start`, `domain` and `rotate-core-key` and Web's domain setup refuse the installation and point to the installer and `oac uninstall`; `oac status` reports it. Only the installer starts an incomplete installation; the installer or `oac uninstall` removes it. - Any failure or interrupt (Ctrl-C, SIGTERM, SIGHUP) of an installation that is not complete runs `oac_cli.remove` before anything is printed, so a closed terminal can't stop it. The original error then goes on, and the installer prints it followed by `Nothing was kept; fix the problem and rerun the same command.` `remove` disables native Core's unit, runs `docker compose -p down --volumes --remove-orphans` from `/` without `COMPOSE_*` variables and with its output discarded, and deletes every file in the installation directory. The directory goes too when the installer created it; otherwise it stays with its lock. Loaded images stay. -- The release downloader execs the installer, so the installer itself receives Ctrl-C, SIGTERM and SIGHUP and nothing stops it while it removes a failed installation. +- The release downloader waits for the bundled installer in a separate process group. It forwards the first Ctrl-C, SIGTERM or SIGHUP and waits for cleanup before removing the temporary bundle; repeated signals do not interrupt that wait. The child inherits the download lock, so killing the downloader alone cannot expose its active files to another download. - `remove` touches only the `oac-<10 hex digits>` project and its `-core.service` named in `state.json`, and never follows a link. `state.json`, then the `oac` command, then `.oac.lock` are removed last, and the files stay when a service can't be removed, so the next run still recognizes the installation and no other command locks it afresh mid-removal. Those final unlinks ignore SIGINT, SIGTERM and SIGHUP; if the command cannot be unlinked, cleanup restores `state.json` for a retry. SIGKILL or power loss during those final unlinks can leave files that need manual removal. The error lists what is left and the commands that remove it; the printed Compose command uses the same directory and environment isolation as automatic cleanup. - A rerun over an incomplete installation, a `state.json` without `"complete": true`, removes it the same way under the lock and then checks the settings and ports and installs with the flags given now. Only that explicit marker proves completion. A directory without `state.json` is refused whatever it holds, and nothing in it is removed. @@ -89,7 +89,7 @@ A default combined Docker installation adds two Compose services from one pinned ## Output -- Progress describes the operation about to run. Do not imply fresh health checks on a no-change repair. +- Progress describes the operation about to run. Release downloads and archive hashing show byte percentages; extraction shows bytes processed, and quiet bundle verification and image loading show elapsed time. Dynamic output is terminal-only; redirected logs stay plain. Do not imply fresh health checks on a no-change repair. - Terminal styling is optional: honor `NO_COLOR` and keep redirected logs plain. - Summaries show credential file locations, never their values. - `install_display.py` owns shared terminal formatting; `install_output.py` and `node_output.py` own the completion guidance. Ship and checksum the display modules in both the node bootstrap and its retained helper. @@ -120,6 +120,8 @@ The distribution manifest is the one download contract for the Core and node ins - Only artifact requests may follow HTTPS redirects, and only without credentials or cookies. Metadata and enrollment requests stay on the configured console. The console publishes only fixed non-secret files and declared artifact names. - Download into private temporary files, verify size and SHA-256 before an atomic rename, resume interrupted transfers, and reuse only verified cache entries or exact image identities. Never select a release other than the pinned one. - Python zipapps bundle the shared resolver with each remote bootstrap. The node asset is the `oac-node` binary. +- The public release bootstrap retries transient network failures up to three times and resumes an interrupted asset within that run, using its immutable ID, declared size and final checksum. It checks free space before downloading and before extracting each file. These checks cannot reserve storage or measure a remote Docker engine's disk. +- The bootstrap serializes downloads with `~/.oac/releases/.download.lock`. It discards the private `.download/` staging directory left by an interrupted run before accessing the network, never another active downloader's files. Failed downloads and installations remove staging; only a successful installation retains its verified bundle for repair. Loaded Docker images follow the installation cleanup rule above. - Release downloads are anonymous. Never add repository credentials to installed node or Runtime configuration. - Manual builds use the `build-` release tag and tag builds the `v*` tag. The manifest's download base must match the release tag; artifact file names and source provenance keep the full source SHA. diff --git a/deploy/install/install.py b/deploy/install/install.py index 80701853b..a24059847 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -8,6 +8,7 @@ import argparse import base64 import contextlib +import errno import hashlib import ipaddress import json @@ -75,6 +76,10 @@ def error_text(error): text = str(error) elif isinstance(error, KeyboardInterrupt): text = "interrupted" + elif isinstance(error, OSError) and error.errno in (errno.ENOSPC, errno.EDQUOT): + text = "Disk space or quota exhausted; free space on the installation filesystem and rerun" + elif isinstance(error, PermissionError): + text = "Permission denied; use a directory writable by your current account (--install-dir)" else: text = "inspect prerequisites and private deployment files" removal = getattr(error, "removal", None) @@ -352,7 +357,11 @@ def read_core_key_file(source): def check_compose(): - version = run(["docker", "compose", "version", "--short"], capture_output=True, text=True).stdout.strip() + try: + version = run(["docker", "compose", "version", "--short"], capture_output=True, text=True, + timeout=30).stdout.strip() + except (OSError, subprocess.SubprocessError): + raise InstallError("Docker with the Compose plugin is required; check docker compose version") from None match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?", version) if not match or tuple(map(int, match.groups())) < (2, 26, 0): raise InstallError("Docker Compose 2.26.0 or newer is required for literal Core environment values") @@ -362,7 +371,11 @@ def check_host(): if platform.system() != "Linux" or platform.machine() not in ("x86_64", "amd64"): raise InstallError("Core installation requires Linux amd64 with Docker access") check_compose() - run(["docker", "info", "--format", "{{.ServerVersion}}"], stdout=subprocess.DEVNULL) + try: + run(["docker", "info", "--format", "{{.ServerVersion}}"], capture_output=True, timeout=30) + except (OSError, subprocess.SubprocessError): + raise InstallError("Cannot reach Docker as the current account. Check docker info, the daemon and this " + "account's Docker access; the installer does not require root or invoke sudo") from None def image_names(mode, native, managed=False): @@ -376,8 +389,10 @@ def image_loader(manifest, bundle): def load(names): images = {} for name in names: - step("Preparing " + {"database": "PostgreSQL", "core": "Core", "web": "Web"}.get(name, name) + " image") - images[name] = ensure_docker_image(manifest, name, lambda name=name: bundle / f"images/{name}.tar") + message = "Preparing " + {"database": "PostgreSQL", "core": "Core", "web": "Web"}.get(name, name) + " image" + step(message) + with install_display.busy(message): + images[name] = ensure_docker_image(manifest, name, lambda name=name: bundle / f"images/{name}.tar") return images return load @@ -661,7 +676,8 @@ def main(argv=None): step("Checking installation settings") prepared = prepare_fresh(args) step("Verifying installation files") - manifest = verify_bundle(bundle) + with install_display.busy("Verifying installation files"): + manifest = verify_bundle(bundle) # Refuse foreign state before even creating a lock; repeat under the lock to # protect against another current installer finishing between these reads. check_release(root, manifest) @@ -677,7 +693,7 @@ def main(argv=None): def check_release(root, manifest): if (root / "state.json").exists(): state = oac_cli.load_state(root) - if state.get("source_commit") != manifest["source_commit"]: + if state.get("complete") is True and state.get("source_commit") != manifest["source_commit"]: raise InstallError(oac_cli.UNSUPPORTED_VERSION) @@ -768,7 +784,7 @@ def interrupted(signum, frame): if __name__ == "__main__": # SIGTERM and SIGHUP, such as from a dropped SSH session, stop the installer as Ctrl-C does, # so a new installation still removes what it created. - for signum in (signal.SIGTERM, signal.SIGHUP): + for signum in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): signal.signal(signum, interrupted) try: main() diff --git a/deploy/install/install_display.py b/deploy/install/install_display.py index 67743a28d..5b003e9a6 100644 --- a/deploy/install/install_display.py +++ b/deploy/install/install_display.py @@ -1,7 +1,10 @@ """Shared terminal presentation for the Core and node installers.""" import os +import contextlib import sys import textwrap +import threading +import time def color(message, code, stream=None): @@ -15,6 +18,36 @@ def step(message): print("\n" + color("==> " + message + "...", "36"), flush=True) +@contextlib.contextmanager +def busy(message): + """Show elapsed time for a quiet operation whose completion percentage is unknown.""" + stream = sys.stderr + if not stream.isatty() or os.environ.get("TERM") == "dumb": + yield + return + stopped = threading.Event() + started = time.monotonic() + + def update(): + frames = "|/-\\" + while not stopped.wait(0.5): + try: + elapsed = int(time.monotonic() - started) + print(f"\r{frames[elapsed % 4]} {message} ({elapsed}s)", end="\033[K", file=stream, flush=True) + except OSError: + return + + worker = threading.Thread(target=update, daemon=True) + worker.start() + try: + yield + finally: + stopped.set() + worker.join() + with contextlib.suppress(OSError): + print("\r\033[K", end="", file=stream, flush=True) + + def heading(message): print("\n" + color(message, "1")) diff --git a/deploy/install/test_install.py b/deploy/install/test_install.py index 7d346f1ac..6d4e02651 100644 --- a/deploy/install/test_install.py +++ b/deploy/install/test_install.py @@ -2,6 +2,7 @@ """install.sh acceptance behavior: fresh install, repair and bundle checks.""" import contextlib +import errno import io import json import os @@ -108,6 +109,44 @@ def test_host_check_accepts_current_account_including_root(self): ["docker", "info", "--format", "{{.ServerVersion}}"], ]) + def test_host_failures_explain_the_required_action_without_secrets(self): + for command, failure, message in ( + ("compose", FileNotFoundError(), "Compose plugin"), + ("info", PermissionError("private detail"), "current account"), + ("info", subprocess.TimeoutExpired(["docker", "info"], 30), "current account")): + with self.subTest(command=command, failure=failure): + original = self.host.run + def run(args, **kwargs): + if command in args: + raise failure + return original(args, **kwargs) + with mock.patch.object(subprocess, "run", side_effect=run): + with self.assertRaisesRegex(install.InstallError, message) as raised: + self.install() + self.assertNotIn("private detail", str(raised.exception)) + self.assertFalse(self.root.exists()) + + def test_space_and_permission_errors_are_actionable_and_leave_no_installation(self): + for error, message in ((OSError(errno.ENOSPC, "full"), "Disk space"), + (OSError(errno.EDQUOT, "quota"), "quota"), + (PermissionError("private detail"), "current account")): + with self.subTest(error=error), mock.patch.object(install, "create", side_effect=error): + with self.assertRaises(OSError) as raised: + self.install() + self.assertIn(message, install.error_text(raised.exception)) + self.assertIn(install.NOTHING_KEPT, install.error_text(raised.exception)) + self.assertFalse(self.root.exists()) + + def test_new_release_cleans_an_incomplete_installation_before_starting_again(self): + self.install() + state = self.document("state.json") + previous_id = state["installation_id"] + install.oac_cli.save_state(self.root, dict(state, complete=False, source_commit="b" * 40)) + self.install("--ingress", "external") + self.assertNotEqual(self.document("state.json")["installation_id"], previous_id) + self.assertTrue(self.document("state.json")["complete"]) + self.assertTrue(any("down" in command for command in self.host.commands)) + def test_root_identity_is_preserved_in_service_configuration(self): with mock.patch.object(install.os, "getuid", return_value=0), \ mock.patch.object(install.os, "getgid", return_value=0): diff --git a/deploy/install/test_install_output.py b/deploy/install/test_install_output.py index a97c159d9..57b0ed802 100644 --- a/deploy/install/test_install_output.py +++ b/deploy/install/test_install_output.py @@ -2,6 +2,7 @@ import contextlib import io import os +import threading from pathlib import Path import unittest from unittest import mock @@ -16,6 +17,28 @@ def isatty(self): class OutputTests(unittest.TestCase): + def test_busy_indicator_stops_after_failure_and_keeps_logs_plain(self): + for stream in (Terminal(), io.StringIO()): + wrote = threading.Event() + original = stream.write + def write(value): + result = original(value) + if "Loading" in value: + wrote.set() + return result + with mock.patch.object(stream, "write", side_effect=write), \ + mock.patch.dict(os.environ, {"TERM": "xterm"}), contextlib.redirect_stderr(stream): + with self.assertRaises(RuntimeError): + with display.busy("Loading"): + if stream.isatty(): + self.assertTrue(wrote.wait(2)) + raise RuntimeError("stop") + if stream.isatty(): + self.assertIn("Loading (", stream.getvalue()) + self.assertTrue(stream.getvalue().endswith("\r\033[K")) + else: + self.assertEqual(stream.getvalue(), "") + def test_color_only_in_capable_terminals_and_no_color_wins(self): for stream, environment, colored in ((Terminal(), {"TERM": "xterm"}, True), (Terminal(), {"TERM": "dumb"}, False), diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index b0de61a17..cd347a980 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -43,7 +43,11 @@ The script picks the latest stable release, verifies its checksum and runs the b It creates no Project or key and makes no model request. It ends by printing the console address, the API base URL and the next steps. -If installation fails or is interrupted before the installer reports that the services are running, including by Ctrl-C or a closed SSH session, the installer removes everything it created: its Compose project with its volumes, native Core's service and the files in the installation directory, and the directory itself when the installer created it. Only the loaded images stay. Fix the cause it prints and run the same command again. A rerun also removes an installation that stopped without this cleanup, for example after a power loss, and then installs with the options given now; until then, the `oac` command refuses to change that installation. Once the installer has reported that the services are running, the installation is kept, and a rerun only [repairs](operations.md#installation-version-policy) it. +Downloads retry temporary network failures automatically. The terminal shows download progress and activity during long steps. + +If installation fails or is interrupted before the services first become healthy, fix the reported cause and rerun the same command. The installer removes its temporary download, new service project, volumes and installation files; loaded Docker images remain reusable. A rerun first clears an incomplete installation or download left by a forced exit or power loss. It never clears another active installation process or an unrelated directory. Once the services have started successfully, failures preserve the installation and its data; use [same-release repair](operations.md#installation-version-policy). + +For insufficient space or quota, free space on the filesystem named by the error. Image-loading failures can also require space in Docker's storage, which may be on a different filesystem. ## Sign in to Web diff --git a/scripts/install-release.test.py b/scripts/install-release.test.py index 3ef58ebd0..f15ff811a 100644 --- a/scripts/install-release.test.py +++ b/scripts/install-release.test.py @@ -1,6 +1,11 @@ """Public release bootstrap tests; fixtures never start Docker or a real installer.""" import contextlib +import errno +import signal +import shlex +import sys import hashlib +import http.client import io import json import os @@ -21,7 +26,9 @@ class BootstrapTests(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory() + temporary_root = pathlib.Path.home() / ".oac/tests/release" + temporary_root.mkdir(parents=True, exist_ok=True) + self.temp = tempfile.TemporaryDirectory(dir=temporary_root) self.addCleanup(self.temp.cleanup) self.root = pathlib.Path(self.temp.name) self.sha = "a" * 40 @@ -33,15 +40,17 @@ def setUp(self): self.archive = self.make_archive({ "install.sh": b"#!/bin/sh\nexit 0\n", "manifest.json": json.dumps({"source_commit": self.sha}).encode()}) + self.metadata["assets"][0]["size"] = len(self.archive) + self.metadata["assets"][1]["size"] = 67 + len(self.name) stack = contextlib.ExitStack() self.addCleanup(stack.close) self.http = stack.enter_context(mock.patch.object(bootstrap, "open_url", side_effect=self.response)) stack.enter_context(mock.patch.object(bootstrap.pathlib.Path, "home", return_value=self.root)) stack.enter_context(mock.patch.object(bootstrap.platform, "system", return_value="Linux")) stack.enter_context(mock.patch.object(bootstrap.platform, "machine", return_value="x86_64")) - stack.enter_context(mock.patch.object(bootstrap.os, "geteuid", return_value=1000)) - # The downloader replaces itself with the installer. - self.invoke = stack.enter_context(mock.patch.object(bootstrap.os, "execvp")) + stack.enter_context(mock.patch.object(bootstrap.time, "sleep")) + # The downloader waits for the installer before retaining its verified bundle. + self.invoke = stack.enter_context(mock.patch.object(bootstrap, "run_installer")) def make_archive(self, files): data = io.BytesIO() @@ -52,7 +61,7 @@ def make_archive(self, files): archive.addfile(member, io.BytesIO(content)) return data.getvalue() - def response(self, url, binary=False): + def response(self, url, binary=False, offset=0): if url.endswith("/releases/latest") or "/releases/tags/" in url: return io.BytesIO(json.dumps(self.metadata).encode()) if url.endswith("/assets/7"): @@ -67,10 +76,9 @@ def test_default_latest_is_resolved_once_and_installer_arguments_forwarded(self) self.assertEqual(sum(url.endswith("/releases/latest") for url in urls), 1) self.assertTrue(urls[1].endswith("/assets/8")) self.assertTrue(urls[2].endswith("/assets/7")) - program, command = self.invoke.call_args.args - self.assertEqual((program, command[0]), ("bash", "bash")) - self.assertEqual(command[2:], ["--host", "127.0.0.1", "--web-port", "8088", "--public-url", "https://core.example"]) - self.assertTrue(pathlib.Path(command[1]).is_file()) + root, arguments, lock = self.invoke.call_args.args + self.assertEqual(arguments, ["--host", "127.0.0.1", "--web-port", "8088", "--public-url", "https://core.example"]) + self.assertEqual(len(list((self.root / ".oac/releases").glob("release-*/" + self.stem + "/install.sh"))), 1) self.assertFalse(list((self.root / ".oac/releases").glob(".download-*"))) def test_control_plane_bundle_selected_without_execution_assets(self): @@ -80,7 +88,7 @@ def test_control_plane_bundle_selected_without_execution_assets(self): urls = [call.args[0] for call in self.http.call_args_list] self.assertEqual(len(urls), 3) self.assertTrue(urls[-1].endswith("/assets/7")) - self.assertFalse((pathlib.Path(self.invoke.call_args.args[1][1]).parent / "artifacts").exists()) + self.assertFalse((self.invoke.call_args.args[0] / "artifacts").exists()) def test_offline_only_release_is_not_silently_downloaded(self): self.metadata["assets"][0]["name"] = self.stem + "-offline.tar.gz" @@ -93,7 +101,7 @@ def test_selected_prerelease_is_allowed(self): self.metadata.update(tag_name="v2.0.0-rc.1", prerelease=True) bootstrap.main(["--version", "v2.0.0-rc.1", "--core-only"]) self.assertTrue(self.http.call_args_list[0].args[0].endswith("/releases/tags/v2.0.0-rc.1")) - self.assertEqual(self.invoke.call_args.args[1][-1], "--core-only") + self.assertEqual(self.invoke.call_args.args[1], ["--core-only"]) def test_latest_never_selects_prerelease_or_draft(self): for key in ("prerelease", "draft"): @@ -116,8 +124,9 @@ def test_missing_or_duplicate_asset_is_refused(self): def test_checksum_failure_never_executes_and_removes_only_own_download(self): existing = self.root / ".oac/releases/existing" existing.mkdir(parents=True) + existing.parent.chmod(0o700) (existing / "data").write_text("keep") - def response(url, binary=False): + def response(url, binary=False, offset=0): if url.endswith("/assets/8"): return io.BytesIO(("b" * 64 + " " + self.name + "\n").encode()) return self.response(url, binary) @@ -125,11 +134,12 @@ def response(url, binary=False): with self.assertRaisesRegex(bootstrap.ReleaseError, "checksum mismatch"): bootstrap.main([]) self.invoke.assert_not_called() - self.assertEqual(list(existing.parent.iterdir()), [existing]) + self.assertEqual(set(path.name for path in existing.parent.iterdir()), {"existing", ".download.lock"}) self.assertEqual((existing / "data").read_text(), "keep") def test_archive_source_mismatch_is_refused(self): self.archive = self.make_archive({"install.sh": b"exit 0", "manifest.json": b'{"source_commit":"wrong"}'}) + self.metadata["assets"][0]["size"] = len(self.archive) with self.assertRaisesRegex(bootstrap.ReleaseError, "source"): bootstrap.main([]) self.invoke.assert_not_called() @@ -164,10 +174,10 @@ def test_unsupported_platform_does_not_download(self): def test_root_uses_the_same_verified_installation_path(self): with mock.patch.object(bootstrap.os, "geteuid", return_value=0): bootstrap.main(["--core-only"]) - command = self.invoke.call_args.args[1] - self.assertEqual(command, ["bash", command[1], "--core-only"]) - self.assertTrue(pathlib.Path(command[1]).is_relative_to(self.root / ".oac/releases")) - self.assertTrue(pathlib.Path(command[1]).is_file()) + root, arguments, lock = self.invoke.call_args.args + self.assertEqual(arguments, ["--core-only"]) + self.assertTrue(root.is_relative_to(self.root / ".oac/releases")) + self.assertEqual(len(list((self.root / ".oac/releases").glob("release-*/" + self.stem + "/install.sh"))), 1) def test_new_private_directories_do_not_inherit_public_umask(self): previous = os.umask(0o022) @@ -184,6 +194,239 @@ def test_shell_entrypoint_help_needs_no_network(self): self.assertIn("--version", result.stdout) self.assertIn("latest stable release", result.stdout) + def test_short_download_resumes_only_the_selected_asset(self): + calls = [] + def response(url, binary=False, offset=0): + if url.endswith("/assets/7"): + calls.append(offset) + result = io.BytesIO(self.archive[:20] if not offset else self.archive[offset:]) + if offset: + result.status = 206 + result.headers = {"Content-Range": f"bytes {offset}-{len(self.archive) - 1}/{len(self.archive)}"} + return result + return self.response(url, binary, offset) + self.http.side_effect = response + bootstrap.main([]) + self.assertEqual(calls, [0, 20]) + self.invoke.assert_called_once() + + def test_range_ignored_restarts_instead_of_appending(self): + attempts = [] + def response(url, binary=False, offset=0): + if url.endswith("/assets/7"): + attempts.append(offset) + return io.BytesIO(self.archive[:20] if len(attempts) == 1 else self.archive) + return self.response(url, binary, offset) + self.http.side_effect = response + bootstrap.main([]) + self.assertEqual(attempts, [0, 20]) + self.invoke.assert_called_once() + + def test_network_failure_is_bounded_and_cleans_staging(self): + for failure in (bootstrap.TransferError("network"), TimeoutError(), ConnectionResetError()): + with self.subTest(failure=failure): + self.http.reset_mock() + self.http.side_effect = failure + with self.assertRaisesRegex(bootstrap.ReleaseError, "3 attempts"): + bootstrap.main([]) + self.assertEqual(self.http.call_count, 3) + self.assertFalse((self.root / ".oac/releases/.download").exists()) + self.invoke.assert_not_called() + + def test_truncated_http_metadata_is_retried_before_parsing(self): + class Socket: + def makefile(self, mode): + return io.BytesIO(b'HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{"tag_name":') + partial = http.client.HTTPResponse(Socket()) + partial.begin() + self.http.side_effect = [partial, self.response(bootstrap.API + "/releases/latest"), + self.response(bootstrap.API + "/releases/assets/8"), + self.response(bootstrap.API + "/releases/assets/7")] + bootstrap.main([]) + self.assertEqual(sum(call.args[0].endswith("/releases/latest") for call in self.http.call_args_list), 2) + self.invoke.assert_called_once() + + def test_malformed_metadata_and_asset_sizes_fail_without_execution(self): + for metadata in ([], {}, {**self.metadata, "assets": [None]}, + {**self.metadata, "assets": [{**self.metadata["assets"][0], "size": -1}, + self.metadata["assets"][1]]}): + with self.subTest(metadata=metadata): + self.http.side_effect = lambda *a, **kw: io.BytesIO(json.dumps(metadata).encode()) + with self.assertRaises(bootstrap.ReleaseError): + bootstrap.main([]) + self.assertFalse((self.root / ".oac/releases/.download").exists()) + self.invoke.assert_not_called() + + def test_space_check_and_write_failure_remove_partial_downloads(self): + with mock.patch.object(bootstrap.shutil, "disk_usage", return_value=types.SimpleNamespace(free=0)): + with self.assertRaisesRegex(bootstrap.ReleaseError, "disk space"): + bootstrap.main([]) + for failure in (OSError(errno.ENOSPC, "disk full"), OSError(errno.EDQUOT, "quota")): + with mock.patch.object(bootstrap, "extract", side_effect=failure): + with self.assertRaises(OSError): + bootstrap.main([]) + self.assertEqual([p.name for p in (self.root / ".oac/releases").iterdir()], [".download.lock"]) + self.invoke.assert_not_called() + + def test_extraction_checks_uncompressed_space(self): + archive = self.root / "archive.tar.gz" + archive.write_bytes(self.archive) + with mock.patch.object(bootstrap.shutil, "disk_usage", return_value=types.SimpleNamespace(free=0)): + with self.assertRaisesRegex(bootstrap.ReleaseError, "disk space"): + bootstrap.extract(archive, self.root, self.stem) + self.assertFalse((self.root / self.stem).exists()) + + def test_handoff_failure_removes_extracted_bundle(self): + self.invoke.side_effect = FileNotFoundError("bash") + with self.assertRaises(FileNotFoundError): + bootstrap.main([]) + self.assertEqual([p.name for p in (self.root / ".oac/releases").iterdir()], [".download.lock"]) + + def test_stale_download_is_removed_even_when_network_is_unavailable(self): + stale = self.root / ".oac/releases/.download" + stale.mkdir(parents=True, mode=0o700) + stale.parent.chmod(0o700) + (stale / "partial").write_text("discard") + outside = self.root / "user-data" + outside.write_text("keep") + (stale / "link").symlink_to(outside) + self.http.side_effect = bootstrap.TransferError("network") + with self.assertRaises(bootstrap.ReleaseError): + bootstrap.main([]) + self.assertFalse(stale.exists()) + self.assertEqual(outside.read_text(), "keep") + + def test_concurrent_download_does_not_remove_active_staging(self): + cache = self.root / ".oac/releases" + cache.mkdir(parents=True, mode=0o700) + with bootstrap.staging(cache) as (active, lock): + (active / "partial").write_text("keep") + with self.assertRaisesRegex(bootstrap.ReleaseError, "Another release download"): + bootstrap.main([]) + self.assertEqual((active / "partial").read_text(), "keep") + self.http.assert_not_called() + + def test_staging_symlink_is_refused_without_touching_target(self): + cache = self.root / ".oac/releases" + cache.mkdir(parents=True, mode=0o700) + outside = self.root / "user-data" + outside.mkdir() + (outside / "keep").write_text("keep") + (cache / ".download").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(bootstrap.ReleaseError, "staging directory"): + bootstrap.main([]) + self.assertEqual((outside / "keep").read_text(), "keep") + + def test_signal_during_download_cleans_before_exit(self): + source = SCRIPT.read_text().split("3<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + fixture = ''' + platform.system = lambda: "Linux" + platform.machine = lambda: "x86_64" + def blocked(version): + print("READY", flush=True) + time.sleep(30) + select_release = blocked + main([])''' + source = source.replace(" main(sys.argv[1:])", fixture) + for number in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + with self.subTest(signal=number): + process = subprocess.Popen([__import__("sys").executable, "-c", source], + env={**os.environ, "HOME": str(self.root)}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + for line in process.stdout: + if line.strip() == "READY": + break + else: + self.fail("Downloader did not reach the signal fixture") + process.send_signal(number) + _, error = process.communicate(timeout=5) + self.assertEqual(process.returncode, 130, error) + self.assertNotIn("Traceback", error) + self.assertFalse((self.root / ".oac/releases/.download").exists()) + finally: + if process.poll() is None: + process.kill() + process.communicate() + + def test_installer_cleanup_finishes_before_bundle_removal_and_lock_survives_parent(self): + source = SCRIPT.read_text().split("3<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + child = '''import os, pathlib, signal, time +def stop(number, frame): + time.sleep(0.1) + assert pathlib.Path.home().joinpath(".oac/releases/.download").is_dir() + pathlib.Path.home().joinpath("cleaned").touch() + raise SystemExit(130) +for number in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(number, stop) +print("CHILD_READY " + str(os.getpid()), flush=True) +while True: + signal.pause() +''' + archive = self.make_archive({"install.sh": ("#!/bin/bash\nexec python3 -c " + shlex.quote(child)).encode(), + "manifest.json": json.dumps({"source_commit": self.sha}).encode()}) + metadata = [dict(asset) for asset in self.metadata["assets"]] + metadata[0]["size"] = len(archive) + checksum = (hashlib.sha256(archive).hexdigest() + " " + self.name + "\n").encode() + fixture = f''' + platform.system = lambda: "Linux" + platform.machine = lambda: "x86_64" + select_release = lambda version: ("v1", {metadata[0]!r}, {metadata[1]!r}) + def download(asset, destination, limit=None): + destination.write_bytes({archive!r} if asset["id"] == 7 else {checksum!r}) + main([])''' + source = source.replace(" main(sys.argv[1:])", fixture) + for number in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGKILL): + with self.subTest(signal=number): + cleaned = self.root / "cleaned" + cleaned.unlink(missing_ok=True) + child_pid = None + process = subprocess.Popen([sys.executable, "-c", source], + env={**os.environ, "HOME": str(self.root)}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + for line in process.stdout: + if line.startswith("CHILD_READY "): + child_pid = int(line.split()[1]) + break + self.assertIsNotNone(child_pid) + process.send_signal(number) + if number == signal.SIGKILL: + process.wait(timeout=5) + with self.assertRaisesRegex(bootstrap.ReleaseError, "Another release download"): + bootstrap.main([]) + os.kill(child_pid, signal.SIGTERM) + _, error = process.communicate(timeout=5) + child_pid = None + self.assertTrue(cleaned.exists(), error) + if number == signal.SIGKILL: + self.http.side_effect = bootstrap.TransferError("network") + with self.assertRaises(bootstrap.ReleaseError): + bootstrap.main([]) + else: + self.assertEqual(process.returncode, 130, error) + self.assertFalse((self.root / ".oac/releases/.download").exists()) + self.assertFalse(list((self.root / ".oac/releases").glob("release-*"))) + finally: + if child_pid is not None: + with contextlib.suppress(ProcessLookupError): + os.kill(child_pid, signal.SIGKILL) + if process.poll() is None: + process.kill() + process.communicate() + + def test_progress_is_dynamic_only_for_terminals_and_does_not_claim_success_on_exit(self): + for terminal in (False, True): + stream = io.StringIO() + stream.isatty = lambda: terminal + with mock.patch.dict(os.environ, {"TERM": "xterm"}), contextlib.redirect_stderr(stream): + with bootstrap.Progress("Download", 100) as progress: + progress.update(50) + self.assertEqual("50%" in stream.getvalue(), terminal) + self.assertNotIn("100%", stream.getvalue()) + if not terminal: + self.assertEqual(stream.getvalue(), "") + if __name__ == "__main__": unittest.main()