diff --git a/deploy/install-release.sh b/deploy/install-release.sh index b174121ab..daf02eacd 100755 --- a/deploy/install-release.sh +++ b/deploy/install-release.sh @@ -12,7 +12,6 @@ import pathlib import platform import re import shutil -import subprocess import sys import tarfile import tempfile @@ -147,7 +146,9 @@ def install(version, arguments): # Keep the verified extracted bundle for same-version repair. print("Verified bundle: " + str(root), flush=True) print("==> Starting the bundled installer...", flush=True) - return subprocess.call(["bash", str(root / "install.sh"), *arguments]) + # Become the installer: it gets Ctrl-C, SIGTERM and SIGHUP itself, and nothing here can stop + # it while it removes a failed new installation. The temporary download is already gone. + os.execvp("bash", ["bash", str(root / "install.sh"), *arguments]) def main(argv): @@ -156,12 +157,12 @@ def main(argv): allow_abbrev=False) parser.add_argument("--version", default="latest", help="Release tag (default: latest stable release)") options, arguments = parser.parse_known_args(argv) - return install(options.version, arguments) + install(options.version, arguments) if __name__ == "__main__": try: - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) except (ReleaseError, OSError, ValueError, tarfile.TarError) as error: print("Installation failed: " + str(error), file=sys.stderr) sys.exit(1) diff --git a/deploy/install/README.md b/deploy/install/README.md index 93fc86806..48db202b7 100644 --- a/deploy/install/README.md +++ b/deploy/install/README.md @@ -24,13 +24,13 @@ This directory holds the Core/Web installer, the `oac` command and the node inst - The Core/Web installer never adds its own host as a node, imports no Runtime image and gives Core neither the Docker socket nor host devices. Nodes are added afterwards from Web, this host included. - Ingress is an installation concern, independent of Runtime and Sandbox Provider selection. - A repeated installation or repair never changes provider identity, the backend namespace or native history. The database, Projects and their keys, provider identity and the credential encryption key survive repair. -- Recovery never deletes data and never prunes containers, volumes or images. +- Recovery never deletes data and never prunes containers, volumes or images. The one removal is a [new installation's own cleanup](#new-installations). - Do not add another launcher, scheduler, supervisor or recovery path. ## Configuration and apply - Every process setting has one home: the installation's private `config.json`, described by `config.schema.json`. Keep the schema, `config_model.py`, the generator and the reference tables that `scripts/config-reference.py` renders into [configuration](../../docs/configuration.md) and [installation options](../../docs/getting-started/install-options.md) in step. -- Installation flags only seed `config.json`. A rerun of the installer accepts only `--install-dir` and repairs. +- Installation flags only seed `config.json`. A rerun of the installer over a complete installation accepts only `--install-dir` and repairs. - `oac apply` validates `config.json`, derives `generated/` and converges on what actually runs. Each service carries the digest of its inputs (Compose label `io.oac.inputs`, native `OAC_INPUTS`), and exactly the services whose running inputs differ are recreated or restarted. Decide restarts from what runs, never from recorded bookkeeping, so the next apply finishes an interrupted one. Apply contacts running services at the last applied address before changing listeners. - Health checks, setup, apply and generated service files derive addresses from the same `config.json`. `host` selects the gateway listener for managed ingress and the Core and Web listeners for external ingress. A non-loopback external bind requires an HTTPS public origin. - `configuration.listeners` lists every host listener from the same helpers that render the port mappings and listen addresses, so port checks and real binds cannot diverge. A new installation checks them all before it hashes the bundle or loads images, and `oac apply` checks those a change adds before it touches a service; the installation's own listeners do not count. The probe binds with `SO_REUSEADDR`, as the services do. Where a port overlaps one of the installation's own listeners, or the account may not bind a port below 1024, it reads the kernel's listening sockets instead. @@ -40,9 +40,18 @@ This directory holds the Core/Web installer, the `oac` command and the node inst ## Versions and the lock -- Install only into an empty directory, or repair the same source revision. Refuse older formats and different revisions before changing anything; keep their data and direct the operator to install separately. Distributions carry only current installation code: no conversion, migration or binary replacement. Keep the refusal checks and their tests. +- Install only into an empty directory or over an [incomplete installation](#new-installations), or repair a complete installation of the same source revision. Refuse older formats and different revisions before changing anything; keep their data and direct the operator to install separately. Distributions carry only current installation code: no conversion, migration or binary replacement. Keep the refusal checks and their tests. - The packaged `oac.pyz` embeds its build revision and refuses a `state.json` whose `source_commit` differs. -- The installer and every mutating `oac` command share `.oac.lock`. The installer holds it across creation, payload, native service and launcher repair, and apply, calling the already-locked apply implementation without locking again. Never unlink or replace the lock file, even after an interrupted fresh installation; its inode must stay stable. +- The installer and every mutating `oac` command share `.oac.lock`. The installer holds it across creation, payload, native service and launcher repair, and apply, calling the already-locked apply implementation without locking again. Never replace the lock file; its inode must stay stable. Only the cleanup of a new installation unlinks it, with the directory the installer created, while holding it, and `locked` refuses a lock whose path no longer names the file it locked. + +## New installations + +- An installation is complete after its first start: apply converged, the services are healthy and, for Web-only, its Core accepts the Core key. `create()` writes `state.json` before anything else, atomically, with the project name and `"complete": false`, and the installer sets it to `true` at that moment. A later failure, such as a refused sandbox selection, keeps the installation. +- Until then, `oac apply`, `start`, `domain` and `rotate-core-key` and Web's domain setup refuse the installation and point to the installer; `oac status` reports it. Only the installer starts or removes an incomplete installation. +- Any failure or interrupt (Ctrl-C, SIGTERM, SIGHUP) of an installation that is not complete runs `oac_cli.remove` before anything is printed, so a closed terminal can't stop it. The original error then goes on, and the installer prints it followed by `Nothing was kept; fix the problem and rerun the same command.` `remove` disables native Core's unit, runs `docker compose -p down --volumes --remove-orphans` from `/` without `COMPOSE_*` variables and with its output discarded, and deletes every file in the installation directory. The directory goes too when the installer created it; otherwise it stays with its lock. Loaded images stay. +- The release downloader execs the installer, so the installer itself receives Ctrl-C, SIGTERM and SIGHUP and nothing stops it while it removes a failed installation. +- `remove` touches only the `oac-<10 hex digits>` project and its `-core.service` named in `state.json`, and never follows a link. `state.json`, then the `oac` command, then `.oac.lock` are removed last, and the files stay when a service can't be removed, so the next run still recognizes the installation and no other command locks it afresh mid-removal. Those final unlinks ignore SIGINT, SIGTERM and SIGHUP; if the command cannot be unlinked, cleanup restores `state.json` for a retry. SIGKILL or power loss during those final unlinks can leave files that need manual removal. The error lists what is left and the commands that remove it; the printed Compose command uses the same directory and environment isolation as automatic cleanup. +- A rerun over an incomplete installation, a `state.json` without `"complete": true`, removes it the same way under the lock and then checks the settings and ports and installs with the flags given now. Only that explicit marker proves completion. A directory without `state.json` is refused whatever it holds, and nothing in it is removed. ## Install-time sandbox selection diff --git a/deploy/install/ingress.py b/deploy/install/ingress.py index bff1b8c31..e0c423edc 100644 --- a/deploy/install/ingress.py +++ b/deploy/install/ingress.py @@ -67,6 +67,8 @@ def status(root): def prepare(root, name, confirmation): """Validate and reserve a job while the caller holds the installation lock.""" config, state = oac_cli.load_config(root), oac_cli.load_state(root) + if state.get("complete") is not True: + raise DomainError("installation_incomplete", oac_cli.INCOMPLETE, 409) if not gateway.enabled(config): raise DomainError("domain_setup_unavailable", "Managed HTTPS is available in combined Docker installations only") try: diff --git a/deploy/install/install.py b/deploy/install/install.py index acb02d0d2..c48ff3e37 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -2,10 +2,12 @@ """Install one matched Core distribution, repair it, without changing versions. A new installation's flags seed /config.json. Afterwards, edit that file -and run /oac apply; rerunning this installer only repairs. +and run /oac apply; rerunning this installer only repairs. A new installation +that fails before its services first start removes what it created; rerun the same command. """ import argparse import base64 +import contextlib import hashlib import ipaddress import json @@ -15,11 +17,13 @@ import re import secrets import shutil +import signal import socket import stat import subprocess import sys import tempfile +import traceback import uuid from urllib.parse import urlsplit @@ -59,6 +63,26 @@ class InstallError(Exception): pass +REPORTED = (InstallError, oac_cli.OacError, config_model.ConfigError, sandbox_setup.SandboxSetupError, + DistributionError, RuntimeError) +NOTHING_KEPT = "Nothing was kept; fix the problem and rerun the same command." + + +def error_text(error): + """What the installer prints for an error, then what became of a new installation. + + It never includes generated configuration or command output. + """ + if isinstance(error, REPORTED): + text = str(error) + elif isinstance(error, KeyboardInterrupt): + text = "interrupted" + else: + text = "inspect prerequisites and private deployment files" + removal = getattr(error, "removal", None) + return text + ("\n" + removal if removal else "") + + def run(args, **kwargs): # Never print a generated Compose file, process environment or secret value. return subprocess.run(args, **dict({"check": True}, **kwargs)) @@ -517,33 +541,56 @@ def install_oac(root, bundle): def layout(root): if not root.exists() or not any(path.name != ".oac.lock" for path in root.iterdir()): return "empty" - config, legacy = (root / "config.json").exists(), (root / "installation.json").exists() - if config and legacy: - return "interrupted" - if config or legacy: - return "config" if config else "legacy" - generated = root / "generated" - never_applied = not (generated.is_dir() and any(generated.iterdir())) if (root / "state.json").exists(): try: - recorded = json.loads((root / "state.json").read_text()).get("generated") + complete = json.loads((root / "state.json").read_text()).get("complete") except (OSError, ValueError, AttributeError): - recorded = True - # Never applied: nothing started, so nothing depends on these secrets yet. - return "incomplete" if never_applied and not recorded else "missing-config" - if never_applied and {path.name for path in root.iterdir()} <= {"secrets", "generated", "state", ".oac.lock"}: - return "incomplete" - return "other" + complete = None + # create() writes state.json before anything else, with complete: false; the first + # successful start sets it. + if complete is not True: + return "incomplete" + return "config" if (root / "config.json").exists() else "missing-config" + # Without state.json nothing here is known to be the installer's, so nothing is taken over or removed. + return "config" if (root / "config.json").exists() else "other" + + +def written_state(root): + """state.json, or None before create() wrote it; then the directory holds only the lock.""" + return oac_cli.load_state(root) if (root / "state.json").exists() else None + + +def remove_created(root, state, created): + """Remove what this run created; returns the line printed after its error.""" + try: + if state is not None: + oac_cli.remove(root, state, keep_root=not created) + elif created: + # Never remove anything else from a directory without this installation's state.json. + with contextlib.suppress(OSError): + (root / ".oac.lock").unlink() + root.rmdir() + except oac_cli.OacError as left: + return str(left) + return NOTHING_KEPT def create(root, args, config, manifest, images): - """Write the new installation's secrets, config.json and state.json.""" + """Write the new installation's state.json, secrets and config.json, in that order.""" mode = config["mode"] token = read_core_key_file(args.core_key_file) if mode == "web-only" else secrets.token_hex(32) if root.parent == Path.home() / ".oac": oac_cli.private_parent(root) root.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(root, 0o700) + state = {"format": 2, "installation_id": str(uuid.uuid4()), "project": "oac-" + secrets.token_hex(5), + "uid": os.getuid(), "gid": os.getgid(), "mode": mode, "native_core": config.get("native_core", False), + "source_commit": manifest["source_commit"], "images": images, "secrets_sha256": {}, + "core_installation_id": None, "generated": {}, "complete": False} + if ingress_config.enabled(config): + state["ingress"] = ingress_config.preflight() + # state.json first, written whole: it marks everything after it as this installation's. + oac_cli.save_state(root, state) for name in ["secrets", "generated"] + ([] if mode == "web-only" else ["state", "state/e2b"]): (root / name).mkdir(mode=0o700) write = oac_cli.create_private @@ -551,17 +598,12 @@ def create(root, args, config, manifest, images): if mode != "web-only": write(root / "secrets/credential.key", base64.b64encode(secrets.token_bytes(32)).decode()) write(root / "secrets/database.password", secrets.token_hex(32)) - state = {"format": 2, "installation_id": str(uuid.uuid4()), "project": "oac-" + secrets.token_hex(5), - "uid": os.getuid(), "gid": os.getgid(), "mode": mode, "native_core": config.get("native_core", False), - "source_commit": manifest["source_commit"], "images": images, - "secrets_sha256": configuration.secret_digests(root, mode), "core_installation_id": None, - "generated": {}} if ingress_config.enabled(config): - state["ingress"] = ingress_config.preflight() ingress_config.prepare(root) - state["secrets_sha256"].pop("core.key") - # state.json first: whenever config.json exists, the installation can be repaired. - write(root / "state.json", json.dumps(state, indent=2) + "\n") + digests = configuration.secret_digests(root, mode) + digests.pop("core.key") + oac_cli.save_state(root, dict(state, secrets_sha256=digests)) + # config.json last: whenever it exists, the installation can be repaired. write(root / "config.json", json.dumps(config, indent=2) + "\n") @@ -575,23 +617,26 @@ def finish(root, bundle, manifest, fresh=False, selection=None, moved=()): install_oac(root, bundle) if ingress_config.enabled(oac_cli.load_config(root)): ingress_config.prepare(root) - retry = f"rerun ./install.sh --install-dir {root}" + args = argparse.Namespace(dry_run=False, yes=False, confirm_public_url_change=None) + step("Applying settings and starting services as needed") try: - args = argparse.Namespace(dry_run=False, yes=False, confirm_public_url_change=None) - step("Applying settings and starting services as needed") oac_cli._apply(root, args, False, True, sys.stdin.isatty(), - lambda message: print(message, flush=True), retry=retry) - except oac_cli.OacError as error: - if not selection: + lambda message: print(message, flush=True), retry=f"rerun ./install.sh --install-dir {root}") + except oac_cli.ApplyFailed as error: + if not fresh: raise - raise oac_cli.OacError(f"{str(error).rstrip('.')}. The sandbox backend was not chosen; after the " - f"repair, choose it {choose_where(state['mode'])}") from None + # The installer removes what it created and says how to retry. + raise InstallError(f"The services did not start: {error.cause}") from None config = oac_cli.load_config(root) mode = config["mode"] if mode == "web-only": step("Checking Core connection and authentication") if mode == "web-only" and oac_cli.paired_core(root, config)[0] != 200: - raise InstallError("Core key authentication failed. Inspect secrets/core.key and web.core_url; no model was called") + where = "--core-key-file and the Core URL" if fresh else "secrets/core.key and web.core_url" + raise InstallError(f"Core key authentication failed. Inspect {where}; no model was called") + if fresh: + # The first start finished, Web-only reaching its Core with the key: from now on the installation is kept. + oac_cli.save_state(root, dict(oac_cli.load_state(root), complete=True)) deployment = failure = None if selection: step("Configuring sandbox backend") @@ -637,8 +682,10 @@ def main(argv=None): if (old / "state.json").exists() or (old / "installation.json").exists(): raise InstallError(oac_cli.UNSUPPORTED_VERSION) # Settings and listeners take seconds to check, so they come before hashing the bundle. - step("Checking installation settings") - prepared = prepare_fresh(args) if layout(root) == "empty" else None + prepared = None + if layout(root) == "empty": + step("Checking installation settings") + prepared = prepare_fresh(args) step("Verifying installation files") manifest = verify_bundle(bundle) # Refuse foreign state before even creating a lock; repeat under the lock to @@ -646,10 +693,11 @@ def main(argv=None): check_release(root, manifest) if root.parent == Path.home() / ".oac": root.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + created = not root.exists() root.mkdir(mode=0o700, parents=True, exist_ok=True) with oac_cli.locked(root): check_release(root, manifest) - install_locked(args, root, bundle, manifest, prepared) + install_locked(args, root, bundle, manifest, prepared, created) def check_release(root, manifest): @@ -682,7 +730,7 @@ def prepare_fresh(args): return config, choice, e2b, moved -def install_locked(args, root, bundle, manifest, prepared): +def install_locked(args, root, bundle, manifest, prepared, created): kind = layout(root) if kind == "config": if args.given: @@ -705,11 +753,30 @@ def install_locked(args, root, bundle, manifest, prepared): raise InstallError(f"{root / 'config.json'} is missing. Restore it from a backup; " f"{root / 'generated/settings.json'} lists the last applied values. The secrets and " "database belong to this installation, so keep the directory. Nothing was changed") - if kind == "incomplete": - raise InstallError(f"An earlier installation into {root} stopped before writing config.json and started no " - "service. Preserve the directory and reinstall into a new empty directory") if kind == "other": raise InstallError("Installation directory is not empty; refusing to overwrite existing state") + if kind == "incomplete": + # A first installation stopped without its cleanup, such as by kill -9 or power loss. + step("Removing an incomplete earlier installation") + oac_cli.remove(root, oac_cli.load_state(root), keep_root=True) + if prepared is None: + # Checked only now that the earlier installation is gone, so the ports it held count as free. + step("Checking installation settings") + prepared = prepare_fresh(args) + try: + install_fresh(args, root, bundle, manifest, prepared) + except (Exception, KeyboardInterrupt) as error: + state = written_state(root) + if state and state.get("complete") is True: + raise + # A first installation that did not finish removes what it created, before printing + # anything, so the same command can run again. The error goes on with the outcome. + error.removal = remove_created(root, state, created) + raise + + +def install_fresh(args, root, bundle, manifest, prepared): + """Check the host, load images, create the installation and start it for the first time.""" config, choice, e2b, moved = prepared step("Checking host requirements") check_host() @@ -724,14 +791,24 @@ def install_locked(args, root, bundle, manifest, prepared): finish(root, bundle, manifest, fresh=True, selection=selection, moved=moved) +def interrupted(signum, frame): + raise KeyboardInterrupt + + if __name__ == "__main__": + # SIGTERM and SIGHUP, such as from a dropped SSH session, stop the installer as Ctrl-C does, + # so a new installation still removes what it created. + for signum in (signal.SIGTERM, signal.SIGHUP): + signal.signal(signum, interrupted) try: main() - except (InstallError, oac_cli.OacError, config_model.ConfigError, - sandbox_setup.SandboxSetupError, DistributionError, RuntimeError) as error: - install_display.error(str(error)) - sys.exit(1) - except (OSError, ValueError, KeyError, subprocess.CalledProcessError): - # Errors never include generated configuration or external process output. - install_display.error("inspect prerequisites and private deployment files") + except (*REPORTED, OSError, ValueError, KeyError, subprocess.CalledProcessError, KeyboardInterrupt) as error: + with contextlib.suppress(OSError): # The terminal may be gone. + install_display.error(error_text(error)) + sys.exit(130 if isinstance(error, KeyboardInterrupt) else 1) + except Exception as error: + # An unexpected error keeps its traceback, followed by what became of a new installation. + traceback.print_exc() + if getattr(error, "removal", None): + print(error.removal, file=sys.stderr) sys.exit(1) diff --git a/deploy/install/installer_fakes.py b/deploy/install/installer_fakes.py index fdff8f6fb..2160a9d9c 100644 --- a/deploy/install/installer_fakes.py +++ b/deploy/install/installer_fakes.py @@ -32,10 +32,10 @@ class FakeHost: def __init__(self, test): self.commands, self.requests = [], [] self.containers = {} # service -> {hash, inputs, running} - self.project = None + self.project = None # the Compose project the containers belong to self.recreated = [] - self.native = {"active": False, "starts": 0, "restarts": 0, "reloads": 0, "addr": None, "digests": [], - "inputs": None, "loaded": None, "environment": ""} + self.native = {"active": False, "enabled": False, "starts": 0, "restarts": 0, "reloads": 0, "addr": None, + "digests": [], "inputs": None, "loaded": None, "environment": ""} # fails: Core never starts; rejects(core.env text): Core refuses that configuration. self.core = {"port": None, "digests": [], "fails": False, "log": "", "rejects": lambda environment: False} self.web_port = None @@ -91,6 +91,10 @@ def run(self, args, check=False, **kwargs): code, stdout = 0, "" if args[:2] == ["docker", "compose"] and args[2:3] == ["-f"]: code, stdout = self.compose(Path(args[3]), args[4:]) + elif args[:2] == ["docker", "compose"] and args[2:3] == ["-p"]: + # Without a project file Compose acts on the named project's labels alone. + if args[4:5] == ["down"] and args[3] == self.project: + self.containers.clear() elif args[:2] == ["docker", "compose"]: stdout = "2.30.0" elif args[:3] == ["docker", "image", "inspect"]: @@ -123,6 +127,7 @@ def service_hash(self, document, name): def compose(self, path, args): document = json.loads(path.read_text()) if path.exists() else {"services": {}} services = document["services"] + self.project = document.get("name", self.project) if args[:1] == ["down"]: self.containers.clear() return 0, "" @@ -189,6 +194,7 @@ def systemctl(self, args): native["reloads"] += 1 native["loaded"] = self.unit_file() elif args[0] in ("enable", "restart", "start"): + native["enabled"] = native["enabled"] or args[0] == "enable" if args[0] != "restart" and native["active"]: return 0, "" # systemd leaves an active unit alone on start and enable --now native["starts" if args[0] != "restart" else "restarts"] += 1 @@ -209,10 +215,18 @@ def systemctl(self, args): native["environment"] = environment.read_text() elif args[0] == "stop": native["active"], native["inputs"] = False, None + elif args[0] == "disable": + if not native["enabled"]: + return 1, "" # the unit file was never linked + native["enabled"] = False + if "--now" in args: + native["active"], native["inputs"] = False, None elif args[0] == "is-active": return (0 if native["active"] else 3), "" elif args[0] == "show" and "--property=MainPID" in args: return 0, "4242" if native["active"] else "0" + elif args[0] == "show" and "--property=LoadState" in args: + return 0, "loaded" if native["enabled"] or native["active"] else "not-found" elif args[0] == "show": return 0, "252" return 0, "" diff --git a/deploy/install/native_service.py b/deploy/install/native_service.py index ccaa31db8..2d10b246c 100644 --- a/deploy/install/native_service.py +++ b/deploy/install/native_service.py @@ -170,6 +170,20 @@ def stop(root, state): _checked(["systemctl", "--user", "stop", unit_name(state)], "Cannot stop this installation's native Core service") +def remove(state): + """Stop and disable the unit, which removes the links enable made; a unit never enabled is already gone.""" + if not is_native(state): + return + unit = unit_name(state) + # disable fails for a unit that was never enabled; its load state below decides. + _run(["systemctl", "--user", "disable", "--now", unit], "Cannot remove this installation's native Core service") + daemon_reload() + loaded = _checked(["systemctl", "--user", "show", "--property=LoadState", "--value", unit], + "Cannot query this installation's native Core service") + if loaded != "not-found" or active(state): + raise RuntimeError("Cannot remove this installation's native Core service") + + def _process_environment(pid): try: raw = Path(f"/proc/{pid}/environ").read_bytes() diff --git a/deploy/install/oac_cli.py b/deploy/install/oac_cli.py index 5ee47e10e..c6c3821e6 100644 --- a/deploy/install/oac_cli.py +++ b/deploy/install/oac_cli.py @@ -16,7 +16,11 @@ import json import os from pathlib import Path +import re import secrets +import shlex +import shutil +import signal import socket import stat import subprocess @@ -36,12 +40,21 @@ SOURCE_COMMIT = None # Set by the packaged entrypoint from its build revision. UNSUPPORTED_VERSION = ("This installation version or historical conversion is not supported; " "preserve its data and reinstall into a new empty directory. Nothing was changed.") +INCOMPLETE = ("This installation did not finish installing. Rerun the installer command; it removes what is left " + "and installs again.") class OacError(Exception): pass +class ApplyFailed(OacError): + """apply wrote the files, but the services did not converge on them; cause says why.""" + def __init__(self, message, cause): + super().__init__(message) + self.cause = cause + + def run(args, **kwargs): # Never print a generated Compose file, process environment or secret value. return subprocess.run(args, **dict({"check": True}, **kwargs)) @@ -140,6 +153,12 @@ def save_state(root, state): write_private(root / "state.json", json.dumps(state, indent=2) + "\n") +def check_complete(state): + """Only the installer uses an installation before its first start has finished.""" + if state.get("complete") is not True: + raise OacError(INCOMPLETE) + + @contextlib.contextmanager def locked(root): if (root / "state.json").exists(): @@ -150,6 +169,13 @@ def locked(root): fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise OacError("Another oac command is running for this installation") from None + try: + current = os.stat(root / ".oac.lock", follow_symlinks=False) + except FileNotFoundError: + current = None + if current is None or not os.path.samestat(current, os.fstat(descriptor)): + # The holder removed the installation, lock file included, after this command opened it. + raise OacError("Another oac command is running for this installation") if (root / "state.json").exists(): load_state(root) yield @@ -563,7 +589,7 @@ def check_paired_core(root, config, state, previous, args, interactive, out): out("web.core_url changes; apply checks which Core it reaches.") return state.get("core_installation_id") status, installation = paired_core(root, config) - if status == 401: + if status == 401 and state.get("complete") is True: out("Warning: Core rejects this Web host's Core key; the key is out of date. " "Copy secrets/core.key from the Core host, then run oac apply.") elif status == 404: @@ -634,6 +660,7 @@ def apply(root, dry_run=False, yes=False, discard_edits=False, confirm_public_ur args = argparse.Namespace(dry_run=dry_run, yes=yes, confirm_public_url_change=confirm_public_url_change) interactive = sys.stdin.isatty() if interactive is None else interactive with locked(root): + check_complete(load_state(root)) return _apply(root, args, discard_edits, start, interactive, out, retry=retry) @@ -721,9 +748,9 @@ def _apply(root, args, discard_edits, start, interactive, out, rollback=True, re if line: out(line) if not (rollback and in_sync): - raise OacError(f"config.json not applied: {describe(error)}. The services were not all running with " + raise ApplyFailed(f"config.json not applied: {describe(error)}. The services were not all running with " f"the previous files, so nothing was rolled back; run oac status, fix the cause " - f"and {retry}") from None + f"and {retry}", describe(error)) from None # Record the restored files as oac's own before writing them back; a restored # hand edit stays one. restored = {name: data for name, data in disk.items() @@ -740,14 +767,95 @@ def _apply(root, args, discard_edits, start, interactive, out, rollback=True, re if ingress_config.enabled(config) and "gateway" in will_run: ingress_config.reload(root, disk["Caddyfile"].decode()) except (OacError, RuntimeError, subprocess.CalledProcessError) as second: - raise OacError(f"config.json not applied: {describe(error)}. The previous generated files were restored, " + raise ApplyFailed(f"config.json not applied: {describe(error)}. The previous generated files were restored, " f"but the services could not be started with them either ({describe(second)}); run " - f"oac status, fix the cause and {retry}") from None - raise OacError(f"config.json not applied: {describe(error)}. The previous generated files were restored " - f"and the services converged on them; fix config.json and {retry}") from None + f"oac status, fix the cause and {retry}", describe(error)) from None + raise ApplyFailed(f"config.json not applied: {describe(error)}. The previous generated files were restored " + f"and the services converged on them; fix config.json and {retry}", describe(error)) from None out("Applied config.json.") +# Removal ----------------------------------------------------------------------- + +def _remove_last_files(root, state, keep_root): + """Finish the few final unlinks without a handled signal losing the recovery state.""" + handlers = {} + try: + for signum in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + handlers[signum] = signal.signal(signum, signal.SIG_IGN) + try: + for name in ("state.json", "oac", ".oac.lock"): + if not (keep_root and name == ".oac.lock"): + (root / name).unlink(missing_ok=True) + if not keep_root: + root.rmdir() + except (OSError, KeyboardInterrupt): + # A failed command unlink must leave both the command and its state for a retry. + if (root / "oac").exists() and not (root / "state.json").exists(): + with contextlib.suppress(OSError): + save_state(root, state) + raise + finally: + for signum, handler in handlers.items(): + signal.signal(signum, handler) + + +def remove(root, state, keep_root=False): + """Remove one installation: native Core's unit, its Compose project with its volumes, then its files. + + state is the loaded state.json. Only this installation's own project and unit are + touched; loaded images are kept. keep_root keeps the directory and its .oac.lock, which + the caller holds, and removes everything else in it. The files stay while a service is + left, so state.json still names it. Nothing is printed, so a closed terminal can't stop + the removal. Raises OacError naming what is left and the commands that remove it. + """ + root = Path(root) + if not root.is_absolute() or root.is_symlink() or root.resolve() != root: + raise OacError("The installation directory must be canonical and not a symlink; nothing was removed") + project = state.get("project") + if not isinstance(project, str) or not re.fullmatch(r"oac-[0-9a-f]{10}", project): + raise OacError("state.json names no Compose project of this installation; nothing was removed") + left = [] + if native_service.is_native(state): + unit = native_service.unit_name(state) + try: + native_service.remove(state) + except (RuntimeError, KeyboardInterrupt): + left.append((f"native Core unit {unit}", f"systemctl --user disable --now {unit}")) + # -p without -f, outside any project directory and without COMPOSE_* settings: Compose + # reads no project file and acts on this project's labels alone. + down = ["docker", "compose", "-p", project, "down", "--volumes", "--remove-orphans"] + compose_names = sorted(name for name in os.environ if name.startswith("COMPOSE_")) + environment = {name: value for name, value in os.environ.items() if name not in compose_names} + try: + run(down, cwd="/", env=environment, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except (subprocess.CalledProcessError, OSError, KeyboardInterrupt): + manual = shlex.join(["env", *(arg for name in compose_names for arg in ("-u", name)), *down]) + left.append((f"Compose project {project} and its volumes", f"(cd / && {manual})")) + target = shlex.quote(str(root)) + files = (f"the files in {root}", f"find {target} -mindepth 1 -delete" if keep_root else f"rm -rf {target}") + if left: + left.append(files) + else: + try: + for path in root.iterdir(): + if path.name in ("state.json", "oac", ".oac.lock"): + continue + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + _remove_last_files(root, state, keep_root) + except (OSError, KeyboardInterrupt): + # A lock file alone, such as one another command just created, is harmless. + if root.is_dir() and any(path.name != ".oac.lock" for path in root.iterdir()): + left.append(files) + if left: + raise OacError("Removal did not finish. Left: " + "; ".join(what for what, _ in left) + ". Remove them with:\n" + + "\n".join(" " + command for _, command in left)) + + # Commands -------------------------------------------------------------------- def written_view(root, state, config): @@ -852,6 +960,7 @@ def status(root, out=print): out(f'Reverse proxy: /v1 and /api/v1 go to {configuration.service_address(config, "core", connect=True)}; ' f'everything else goes to {configuration.service_address(config, "web", connect=True)}') out("Service health does not prove model execution. This check makes no model requests.") + check_complete(state) if not healthy: raise OacError("One or more installed services are unavailable") @@ -861,6 +970,7 @@ def start(root, out=print): with locked(root): check_directories(root) state = load_state(root) + check_complete(state) config = load_config_or_report(root, out) if config is not None: rendered, disk, _ = render_now(root, config, state) @@ -901,6 +1011,7 @@ def rotate_core_key(root, yes=False, interactive=None, out=print): with locked(root): check_directories(root) state = load_state(root) + check_complete(state) if state["mode"] == "web-only": raise OacError("Core owns the Core key. Copy secrets/core.key from the Core host into this " "installation, then run oac apply.") diff --git a/deploy/install/test_install.py b/deploy/install/test_install.py index 1757ebb9e..6994e8ddb 100644 --- a/deploy/install/test_install.py +++ b/deploy/install/test_install.py @@ -7,6 +7,7 @@ import os from pathlib import Path import runpy +import signal import stat import subprocess import sys @@ -16,6 +17,7 @@ import config_model import distribution +import ingress import ingress_config import install import node_spec @@ -160,17 +162,138 @@ def test_repair_contention_changes_no_installation_bytes(self): self.install() self.assertEqual(before, self.snapshot()) - def test_interrupted_finish_repairs_with_original_identity_and_lock(self): - with mock.patch.object(install, "prepare_node_payload", side_effect=KeyboardInterrupt): - with self.assertRaises(KeyboardInterrupt): - self.install() - before = self.document("state.json") - inode = (self.root / ".oac.lock").stat().st_ino - secret = (self.root / "secrets/core.key").read_bytes() - self.install() - self.assertEqual(before["installation_id"], self.document("state.json")["installation_id"]) - self.assertEqual(inode, (self.root / ".oac.lock").stat().st_ino) - self.assertEqual(secret, (self.root / "secrets/core.key").read_bytes()) + def test_an_interrupted_first_start_removes_what_it_created(self): + self.root.mkdir() + with mock.patch.object(install.oac_cli, "health", side_effect=KeyboardInterrupt), \ + self.assertRaises(KeyboardInterrupt) as raised: + self.install() + self.assertEqual(install.error_text(raised.exception), "interrupted\n" + install.NOTHING_KEPT) + # The directory existed, so it stays, with only the lock that a waiting command may hold. + self.assertEqual([path.name for path in self.root.iterdir()], [".oac.lock"]) + self.assertIn(["docker", "compose", "-p", self.host.project, "down", "--volumes", "--remove-orphans"], + self.host.commands) + self.assertEqual(self.host.containers, {}) + + def test_a_rerun_replaces_an_incomplete_installation(self): + self.install("--sandbox", "none") + for marker in (False, None): + with self.subTest(marker=marker): + old = self.host.project + # A first start killed before completion; its services still hold their ports. + state = dict(self.document("state.json"), complete=marker) + if marker is None: + del state["complete"] + install.oac_cli.save_state(self.root, state) + for command in (install.oac_cli.start, install.oac_cli.apply, install.oac_cli.status): + with self.assertRaisesRegex(install.oac_cli.OacError, "did not finish installing. Rerun the installer"): + command(self.root, out=lambda _: None) + with self.assertRaisesRegex(ingress.DomainError, "did not finish installing"): + ingress.prepare(self.root, "core.example", None) + self.host.busy = {8080, 8091} + remove = install.oac_cli.remove + with mock.patch.object(install.oac_cli, "remove", + side_effect=lambda *args, **kwargs: (remove(*args, **kwargs), self.host.busy.clear())): + self.install("--core-only", "--sandbox", "none") + self.assertIn(["docker", "compose", "-p", old, "down", "--volumes", "--remove-orphans"], self.host.commands) + self.assertEqual((self.document("config.json")["mode"], self.document("config.json")["ports"]["core"]), + ("core-only", 8091)) + state = self.document("state.json") + self.assertNotEqual(state["project"], old) + self.assertTrue(state["complete"]) + self.assertEqual(self.host.running(), {"database", "core"}) + + def test_removal_keeps_recovery_state_on_failure_and_finishes_despite_signals(self): + self.install("--sandbox", "none") + state = dict(self.document("state.json"), complete=False) + install.oac_cli.save_state(self.root, state) + command = (self.root / "oac").read_bytes() + unlink = Path.unlink + + def fail_command(path, *args, **kwargs): + if path == self.root / "oac": + raise PermissionError("command unlink refused") + return unlink(path, *args, **kwargs) + + with install.oac_cli.locked(self.root), mock.patch.object(Path, "unlink", fail_command), \ + self.assertRaisesRegex(install.oac_cli.OacError, "Removal did not finish"): + install.oac_cli.remove(self.root, state, keep_root=True) + self.assertEqual(self.document("state.json"), state) + self.assertEqual((self.root / "oac").read_bytes(), command) + self.assertEqual(install.layout(self.root), "incomplete") + + signals = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) + for signum in signals: + previous = signal.signal(signum, install.interrupted) + self.addCleanup(signal.signal, signum, previous) + delivered = [] + + def interrupt_command(path, *args, **kwargs): + if path == self.root / "oac": + self.assertFalse((self.root / "state.json").exists()) + for signum in signals: + signal.raise_signal(signum) + delivered.append(signum) + return unlink(path, *args, **kwargs) + + with install.oac_cli.locked(self.root), mock.patch.object(Path, "unlink", interrupt_command): + install.oac_cli.remove(self.root, state, keep_root=True) + self.assertEqual(delivered, list(signals)) + self.assertEqual([path.name for path in self.root.iterdir()], [".oac.lock"]) + for signum in signals: + self.assertIs(signal.getsignal(signum), install.interrupted) + + def test_manual_removal_command_keeps_automatic_compose_isolation(self): + self.install("--sandbox", "none") + state = self.document("state.json") + foreign = self.work / "other-project" + foreign.mkdir() + (foreign / "compose.yaml").write_text("volumes:\n database:\n name: unrelated-database\n") + binaries = self.work / "bin" + binaries.mkdir() + docker = binaries / "docker" + docker.write_text(f"#!{sys.executable}\nimport json, os, sys\n" + "print(json.dumps({'cwd': os.getcwd(), 'args': sys.argv[1:], " + "'compose': [key for key in os.environ if key.startswith('COMPOSE_')], " + "'docker_host': os.environ.get('DOCKER_HOST')}))\n") + docker.chmod(0o700) + environment = dict(os.environ, COMPOSE_FILE=str(foreign / "compose.yaml"), COMPOSE_PROJECT_NAME="unrelated", + DOCKER_HOST="unix:///synthetic-docker.sock", PATH=str(binaries) + os.pathsep + os.environ["PATH"]) + with mock.patch.dict(os.environ, environment, clear=True): + with install.oac_cli.locked(self.root), \ + mock.patch.object(install.oac_cli, "run", side_effect=OSError("Docker unavailable")) as run, \ + self.assertRaises(install.oac_cli.OacError) as raised: + install.oac_cli.remove(self.root, state) + self.assertEqual(run.call_args.kwargs["cwd"], "/") + self.assertFalse(any(name.startswith("COMPOSE_") for name in run.call_args.kwargs["env"])) + manual = str(raised.exception).splitlines()[1].strip() + result = REAL_RUN(["sh", "-c", manual], cwd=foreign, capture_output=True, text=True, check=True) + observed = json.loads(result.stdout) + self.assertEqual(observed, {"cwd": "/", "args": run.call_args.args[0][1:], "compose": [], + "docker_host": environment["DOCKER_HOST"]}) + self.assertEqual(self.document("state.json"), state) + + def test_a_directory_without_state_json_is_refused_and_untouched(self): + key = self.root / "secrets/e2b.key" + key.parent.mkdir(parents=True) + key.write_text("synthetic-e2b-key-0123456789") + key.chmod(0o600) + before = self.snapshot() + with self.assertRaisesRegex(install.InstallError, "not empty"): + self.install("--sandbox", "e2b", "--e2b-api-key-file", key, "--e2b-template", BUILD, + "--public-url", "https://core.example") + self.assertEqual(self.snapshot(), before) + + def test_a_complete_installation_is_never_removed(self): + self.install("--sandbox", "none") + before = self.snapshot() + with self.assertRaisesRegex(install.InstallError, "configured by"): + self.install("--core-only") + self.host.containers.clear() + self.host.core["fails"] = True + with self.assertRaisesRegex(install.oac_cli.OacError, "config.json not applied"): + self.install() + self.assertEqual(self.snapshot(), before) + self.assertFalse(any("down" in command for command in self.host.commands)) def test_fresh_web_refuses_old_core_before_creating_installation(self): self.host.remote_core["https://core.example"] = (404, None) @@ -356,11 +479,7 @@ def test_retired_flags_and_old_layouts_name_their_replacement(self): (self.root / "installation.json").write_text("{}") with self.assertRaisesRegex(install.InstallError, "not supported;.*reinstall"): self.install() - # A fresh install stopped before config.json started nothing: starting over is safe. (self.root / "installation.json").unlink() - (self.root / "secrets").mkdir() - with self.assertRaisesRegex(install.InstallError, "stopped before writing config.json"): - self.install() (self.root / "state.json").write_text('{"format": 1, "generated": {}}') (self.root / "state.json").chmod(0o600) with self.assertRaisesRegex(install.oac_cli.OacError, "not supported;.*reinstall"): @@ -507,18 +626,29 @@ def test_sandbox_flag_errors_create_nothing(self): self.install(*flags) self.assertFalse(self.root.exists()) - def test_a_failed_first_start_says_the_sandbox_backend_was_not_chosen(self): - self.host.core["fails"] = True - with self.assertRaisesRegex(install.oac_cli.OacError, - f"rerun ./install.sh --install-dir {self.root}. The sandbox backend was not chosen; " - "after the repair, choose it on the Nodes page in Web$"): - self.install("--sandbox", "microsandbox") - self.assertNotIn("Installation complete.", self.output.getvalue()) - self.assertIn("==> Applying settings and starting services as needed...", self.output.getvalue()) - self.host.core["fails"] = False - self.install() - self.assertIn("Installation settings checked.", self.output.getvalue()) - self.assertEqual(self.host.deployment_posts, []) + def test_a_failed_first_start_removes_what_it_created(self): + for flags, cause in (((), "`docker compose up` failed"), (("--native-core",), "Core did not become healthy")): + with self.subTest(flags=flags): + self.root = self.host.native_root = self.work / ("native" if flags else "compose") + self.host.containers, self.output = {}, io.StringIO() + self.host.core["fails"] = True + with self.assertRaises(install.InstallError) as raised: + self.install("--sandbox", "microsandbox", *flags) + self.assertEqual(install.error_text(raised.exception), + f"The services did not start: {cause}\n{install.NOTHING_KEPT}") + self.assertNotIn("Installation complete.", self.output.getvalue()) + self.assertFalse(self.root.exists()) + self.assertIn(["docker", "compose", "-p", self.host.project, "down", "--volumes", "--remove-orphans"], + self.host.commands) + self.assertEqual(self.host.containers, {}) + if flags: + self.assertIn(["systemctl", "--user", "disable", "--now", self.host.project + "-core.service"], + self.host.commands) + self.assertFalse(self.host.native["enabled"] or self.host.native["active"]) + # The same command installs once the cause is fixed, sandbox backend included. + self.host.core["fails"] = False + self.install("--sandbox", "microsandbox", *flags) + self.assertEqual(self.host.deployment_posts[-1]["provider"], "microsandbox") def test_a_refused_selection_leaves_the_services_running(self): secret = "synthetic-e2b-key-0123456789" @@ -592,7 +722,7 @@ def test_wrong_image_identity_creates_no_installation(self): if "inspect" in arguments else original(arguments, **kwargs))): with self.assertRaisesRegex(distribution.DistributionError, "identity or platform"): self.install() - self.assertEqual({p.name for p in self.root.iterdir()}, {".oac.lock"}) + self.assertFalse(self.root.exists()) def test_cli_failure_does_not_print_external_command_secrets(self): secret = "synthetic-sensitive-command-value" diff --git a/deploy/install/test_oac.py b/deploy/install/test_oac.py index 2845f0417..e9a182276 100644 --- a/deploy/install/test_oac.py +++ b/deploy/install/test_oac.py @@ -128,6 +128,8 @@ def install(self, mode="all", native=False, **values): key.chmod(0o600) images = {name: IMAGES[name] for name in install.image_names(mode, native)} install.create(self.root, SimpleNamespace(core_key_file=key), config, {"source_commit": "a" * 40}, images) + # These tests use finished installations; the installer records that after the first start. + oac_cli.save_state(self.root, dict(oac_cli.load_state(self.root), complete=True)) self.apply(start=True) self.host.recreated.clear() self.output.clear() diff --git a/docs/configuration.md b/docs/configuration.md index bf3bc003d..cc0c24031 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,7 +116,7 @@ The installer creates the installation directory, `~/.oac/core` by default, with | --- | --- | --- | | `config.json` | [Process settings](#process-settings-configjson). The only file you edit | You, then `oac apply`; managed domain setup for `public_url` | | `oac` | The [management command](getting-started/operations.md#the-oac-command) | The installer | -| `state.json` | Installation ID, Compose project name, image IDs, source commit and the digests of generated files | The tools only | +| `state.json` | Installation ID, Compose project name, image IDs, source commit, the digests of generated files and whether the services have started once | The tools only | | `secrets/core.key` | The [Core key](getting-started/operations.md#core-key) | `oac rotate-core-key` | | `secrets/credential.key` | Encryption key for what Core stores sealed in the database: model providers, the E2B key, Vault credentials, Skills, initial files and environment setup | Nothing. Keep it with the database; `oac apply` refuses a changed file | | `secrets/database.password` | PostgreSQL password | Nothing. PostgreSQL reads it only when the database is created; `oac apply` refuses a changed file | diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index 2db7db5b2..3b3001bfb 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -37,7 +37,7 @@ These options choose an installation location or perform initial setup; they are | Option | Purpose | | --- | --- | -| `--install-dir DIR` | Absolute installation directory; defaults to `~/.oac/core`. A new installation requires an empty or missing directory | +| `--install-dir DIR` | Absolute installation directory; defaults to `~/.oac/core`. A new installation requires an empty or missing directory, or one holding an [installation that never started](install.md#install) | | `--sandbox docker\|microsandbox\|e2b\|none` | Select the initial [sandbox backend](#sandbox-backend), saved in Core's database; change it later in Web | | `--accept-docker-risks` | Accept Docker's weaker isolation without an interactive prompt | | `--e2b-api-key-file FILE` | With E2B: absolute path to a private key file, no group/other access and at most 4 KiB | diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index aab86a7e2..b0de61a17 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -43,6 +43,8 @@ The script picks the latest stable release, verifies its checksum and runs the b It creates no Project or key and makes no model request. It ends by printing the console address, the API base URL and the next steps. +If installation fails or is interrupted before the installer reports that the services are running, including by Ctrl-C or a closed SSH session, the installer removes everything it created: its Compose project with its volumes, native Core's service and the files in the installation directory, and the directory itself when the installer created it. Only the loaded images stay. Fix the cause it prints and run the same command again. A rerun also removes an installation that stopped without this cleanup, for example after a power loss, and then installs with the options given now; until then, the `oac` command refuses to change that installation. Once the installer has reported that the services are running, the installation is kept, and a rerun only [repairs](operations.md#installation-version-policy) it. + ## Sign in to Web 1. Open the console address the installer printed, such as `http://SERVER_IP:8080`, or your public URL if you passed one. Behind NAT, use the IP address your browser reaches. Until a domain is set, Web accepts IP addresses only, not host names. diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 8eabf4e32..b490ad99e 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -142,7 +142,7 @@ An installation runs one release for its whole life. In-place version upgrades, To move to a new release, install it into a new, empty directory, with its own database, Core key and nodes, and add nodes from its Web. Keep the old installation, its data and its nodes until their work is finished. Nodes run the program of the console that added them and are never upgraded in place; Core accepts only nodes that speak its own node protocol. -Repair the current release by rerunning `./install.sh --install-dir DIR` from the exact same bundle; the downloader keeps it under `~/.oac/releases/`. Repair reloads missing images, restores the `oac` command, applies `config.json` and starts the services. It preserves identity, settings, secrets and history, accepts only `--install-dir`, and refuses a bundle from another release. +Repair the current release by rerunning `./install.sh --install-dir DIR` from the exact same bundle; the downloader keeps it under `~/.oac/releases/`. Repair reloads missing images, restores the `oac` command, applies `config.json` and starts the services. It preserves identity, settings, secrets and history, accepts only `--install-dir`, and refuses a bundle from another release. An installation the installer never reported as running is not repaired but [removed and installed again](install.md#install). The installer and mutating `oac` commands hold the same installation lock, `.oac.lock`, including during repair and interrupted apply recovery. If another command holds it, retry after that command finishes; never remove or replace `.oac.lock` to get past a busy installation. Reinstallation never deletes another installation's files, database, Runtime resources or Session history. @@ -163,6 +163,9 @@ The installer and mutating `oac` commands hold the same installation lock, `.oac | `config.json has changes that are not applied` | Run `oac apply` | | `Core rejects secrets/core.key …` | Run `oac apply`, which restarts Core with the key's digest | | `config.json not applied: …` | `oac apply` printed Core's startup error above; fix `config.json` and apply again | +| `The services did not start: …` | A new installation's first start failed, and the installer [removed what it created](install.md#install). Compose's or Core's error is printed above it; fix the cause and run the same command again | +| `Removal did not finish. Left: …` | The installer could not remove everything a failed new installation created. Run the printed commands to remove what is left, then run the same command again | +| `This installation did not finish installing …` | The installer stopped before reporting that the services were running. Rerun the installer command, which [removes what is left](install.md#install) and installs again | | `… already in use on this server. Automatic HTTPS cannot run beside another program …` during domain setup | Another program holds port 80 or 443. Stop it, using the printed `ss` command to find it, and retry; automatic HTTPS cannot share [these ports](install-options.md#ports) | | `HTTPS verification failed …` during domain setup | DNS points elsewhere, a firewall or NAT blocks inbound ports 80 and 443, or the certificate request failed; see [Configure the domain and HTTPS](install.md#configure-the-domain-and-https) | | Web answers 403 `Forbidden` | Open exactly the console address `oac status` prints; a reverse proxy must pass the original Host | diff --git a/scripts/install-release.test.py b/scripts/install-release.test.py index 419456aa0..3ef58ebd0 100644 --- a/scripts/install-release.test.py +++ b/scripts/install-release.test.py @@ -40,7 +40,8 @@ def setUp(self): stack.enter_context(mock.patch.object(bootstrap.platform, "system", return_value="Linux")) stack.enter_context(mock.patch.object(bootstrap.platform, "machine", return_value="x86_64")) stack.enter_context(mock.patch.object(bootstrap.os, "geteuid", return_value=1000)) - self.invoke = stack.enter_context(mock.patch.object(bootstrap.subprocess, "call", return_value=0)) + # The downloader replaces itself with the installer. + self.invoke = stack.enter_context(mock.patch.object(bootstrap.os, "execvp")) def make_archive(self, files): data = io.BytesIO() @@ -61,13 +62,13 @@ def response(self, url, binary=False): raise AssertionError("Unexpected URL: " + url) def test_default_latest_is_resolved_once_and_installer_arguments_forwarded(self): - self.assertEqual(bootstrap.main(["--host", "127.0.0.1", "--web-port", "8088", "--public-url", "https://core.example"]), 0) + bootstrap.main(["--host", "127.0.0.1", "--web-port", "8088", "--public-url", "https://core.example"]) urls = [c.args[0] for c in self.http.call_args_list] self.assertEqual(sum(url.endswith("/releases/latest") for url in urls), 1) self.assertTrue(urls[1].endswith("/assets/8")) self.assertTrue(urls[2].endswith("/assets/7")) - command = self.invoke.call_args.args[0] - self.assertEqual(command[0], "bash") + program, command = self.invoke.call_args.args + self.assertEqual((program, command[0]), ("bash", "bash")) self.assertEqual(command[2:], ["--host", "127.0.0.1", "--web-port", "8088", "--public-url", "https://core.example"]) self.assertTrue(pathlib.Path(command[1]).is_file()) self.assertFalse(list((self.root / ".oac/releases").glob(".download-*"))) @@ -79,7 +80,7 @@ def test_control_plane_bundle_selected_without_execution_assets(self): urls = [call.args[0] for call in self.http.call_args_list] self.assertEqual(len(urls), 3) self.assertTrue(urls[-1].endswith("/assets/7")) - self.assertFalse((pathlib.Path(self.invoke.call_args.args[0][1]).parent / "artifacts").exists()) + self.assertFalse((pathlib.Path(self.invoke.call_args.args[1][1]).parent / "artifacts").exists()) def test_offline_only_release_is_not_silently_downloaded(self): self.metadata["assets"][0]["name"] = self.stem + "-offline.tar.gz" @@ -92,7 +93,7 @@ def test_selected_prerelease_is_allowed(self): self.metadata.update(tag_name="v2.0.0-rc.1", prerelease=True) bootstrap.main(["--version", "v2.0.0-rc.1", "--core-only"]) self.assertTrue(self.http.call_args_list[0].args[0].endswith("/releases/tags/v2.0.0-rc.1")) - self.assertEqual(self.invoke.call_args.args[0][-1], "--core-only") + self.assertEqual(self.invoke.call_args.args[1][-1], "--core-only") def test_latest_never_selects_prerelease_or_draft(self): for key in ("prerelease", "draft"): @@ -162,17 +163,12 @@ def test_unsupported_platform_does_not_download(self): def test_root_uses_the_same_verified_installation_path(self): with mock.patch.object(bootstrap.os, "geteuid", return_value=0): - self.assertEqual(bootstrap.main(["--core-only"]), 0) - command = self.invoke.call_args.args[0] + bootstrap.main(["--core-only"]) + command = self.invoke.call_args.args[1] self.assertEqual(command, ["bash", command[1], "--core-only"]) self.assertTrue(pathlib.Path(command[1]).is_relative_to(self.root / ".oac/releases")) self.assertTrue(pathlib.Path(command[1]).is_file()) - def test_installer_failure_is_returned_and_bundle_retained(self): - self.invoke.return_value = 17 - self.assertEqual(bootstrap.main([]), 17) - self.assertTrue(pathlib.Path(self.invoke.call_args.args[0][1]).is_file()) - def test_new_private_directories_do_not_inherit_public_umask(self): previous = os.umask(0o022) try: