diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index efc303c2c..804abe4fd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1756,7 +1756,9 @@ remains independent of Docker's local store identity. The manifest is the shared download contract for Core, node and self-hosted installers: flat versioned filenames, compressed Runtime size/hash and unpacked -size/hash, with HTTPS release URLs or the explicit offline payload. Download into +size/hash. Nodes and self-hosted executors download artifacts only from their +console's payload (or a local offline bundle), never from a release URL the build +recorded; release URLs serve people downloading bundles. Download into private temporary files, verify before atomic promotion, and reuse only verified cache entries or exact image identities. Core's default image must not acquire execution-only payloads. Python zipapps bundle the shared resolver with each @@ -1806,9 +1808,10 @@ applications, nodes, sandbox guests and self-hosted executors, and also the cons origin. Core derives the daemon `wss` URL, the self-hosted `remote_url`, hosted Runtime bootstrap and the deployment's read-only `core_url` from it; the deployment API does not accept a Core address, and no deployment row stores one. Bootstrap -never uses request Host or caller-supplied placement fields. Each node records the -address it enrolled with; after the public URL changes, it receives no new -sandboxes until re-added. This does +never uses request Host or caller-supplied placement fields. Enrollment names the +Core address the node uses; Core refuses one that is not the public URL (409, +token unconsumed) and records it. After the public URL changes, a node receives no +new sandboxes until re-added. This does not widen sandbox network policies or change credential admission. The distribution build sets umask 022 for non-root-readable payloads; installation @@ -1828,15 +1831,23 @@ same opt-in rule. Missing KVM fails when microsandbox is selected without changi that choice. The thin distribution supplies native Core binaries. Provider helpers, the node agent, Runtime launcher and pinned msb runtime/firmware are separate, same-revision -assets resolved only when selected. Core packaging is independent of provider: +assets. Nodes obtain them only from the console's payload, so a console serves +nodes only with the offline bundle's assets or release assets placed in the bundle's +`artifacts/` directory before `install.sh` runs; `/console/config` reports which +providers it can serve. Core packaging is independent of provider: `--native-core` runs Core as a systemd user service, with PostgreSQL/Web in Compose and a private loopback database port. Native Core needs no KVM or node assets. Core receives no Docker socket or node identity mount in either mode. The ordinary standalone node service owns its provider processes outside the Core container. Its `KillMode=process` preserves resident microVM/helper processes across a node-service restart. User KVM -access and the Linux runtime libraries are prerequisites for microsandbox; both node -providers require a systemd user session with linger. Do not add another launcher, -scheduler or recovery path for local installation. +access and the Linux runtime libraries are prerequisites for microsandbox. The node +runs as a systemd user service and needs linger. The only other launcher allowed +is the root-run node installation (sudo mode) planned for phase 2b: one root-owned +system service per installation that runs the same node program as a dedicated +unprivileged service user. Do not add any other launcher, scheduler or recovery +path. Node services restart after failures without a start limit, so a node +outlasts a Core outage, and stop restarting when the node program exits 78 +because Core answered 401 to its credential (a removed or retired node). The basic API image and binary builds remain independent artifacts. The standalone API release and Core distribution both include the Hosted Sandbox Manager guide at the relative path used by their packaged README. Include the @@ -1893,8 +1904,11 @@ dialog, using the deployment's `core_url` (the installation public URL). Do not expose routine network wiring or manual runtime setup as the primary flow. Generate a one-time command only on user intent, never retry enrollment writes automatically, and discard credentials and late responses when the dialog closes -or the Core connection changes. Detect successful addition against the node IDs -present before enrollment; an existing node reconnecting is not a new enrollment. +or the Core connection changes. Web detects successful addition against the node +IDs present before enrollment; an existing node reconnecting is not a new +enrollment. Core also reports the command's `enrollment_id` on the node it +registered (null for nodes enrolled before Core recorded it); Web's exact match on +it lands with the Web follow-up. The command verifies the installer checksum before execution, retains normal TLS verification, and passes the enrollment credential only to the installer process. diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 83c2af347..35dd18c72 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -422,6 +422,9 @@ definitions: type: object api.SandboxEnrollmentToken: properties: + enrollment_id: + description: Public, non-secret handle of this command; never a credential. The node it registers reports the same value as enrollment_id. + type: string expires_at: type: string token: @@ -939,6 +942,10 @@ definitions: - microsandbox_artifacts_unavailable - capacity_insufficient type: string + enrollment_id: + description: The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. + type: string + x-nullable: true id: type: string last_seen_at: @@ -1021,6 +1028,10 @@ definitions: - microsandbox_artifacts_unavailable - capacity_insufficient type: string + enrollment_id: + description: The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. + type: string + x-nullable: true history: $ref: '#/definitions/store.NodeHostHistory' host: diff --git a/contracts/agents-api/runtime.openapi.yaml b/contracts/agents-api/runtime.openapi.yaml index 113b8ae8f..ea9c87ee1 100644 --- a/contracts/agents-api/runtime.openapi.yaml +++ b/contracts/agents-api/runtime.openapi.yaml @@ -56,6 +56,9 @@ definitions: properties: backend_fingerprint: type: string + core_url: + description: The Core origin this node stores and connects to, such as https://core.example. It must equal the installation public URL; otherwise enrollment gets 409 sandbox_node_address_mismatch and the token stays unused. + type: string credential: type: string deployment_generation: @@ -183,7 +186,7 @@ paths: post: consumes: - application/json - description: Node machine connection. Consumes a one-use enrollment token; grants no project or administrator access. Responses contain only explicit safe fields. + description: Node machine connection. Consumes a one-use enrollment token; grants no project or administrator access. Responses contain only explicit safe fields. core_url is required and must equal the installation public URL; a different address gets 409 sandbox_node_address_mismatch and leaves the token unused. parameters: - description: Request in: body diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index ec7b2005c..6b9d4b177 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -37,14 +37,17 @@ Node capacity is separate from the deployment specification. The administrator's `max_retained`, defaulting to 2 and 8. Microsandbox uses both limits. Docker never suspends, so its `max_retained` always equals `max_active`; Core replaces any submitted value, here and in `PATCH /core/v1/sandbox/nodes/{node_id}`. Core stores -that approval with the token and copies it to the registered node. Node enrollment +that approval with the token and copies it to the registered node. The response +is `{token, expires_at, enrollment_id}`: `enrollment_id` is a public, non-secret +handle of that command and never a credential. Node enrollment cannot submit capacity overrides. The existing node configuration/identity reads expose approved limits for that credential; node updates remain administrator operations. Local host capacity checks may reject a deployment that cannot run safely, but never raise its limits. -Each node in `GET /core/v1/sandbox/nodes` and its detail reports `core_url`: the -installation public URL when the node enrolled. A node whose `core_url` differs from +Each node in `GET /core/v1/sandbox/nodes` and its detail reports `enrollment_id`, +the handle of the command that registered it (null for nodes enrolled before Core +recorded it), and `core_url`: the installation public URL when the node enrolled. A node whose `core_url` differs from the current public URL receives no new placements. Work already placed on it finishes there: a hosted Environment that was placed but not yet allocated before the change is still allocated on that node, and its retained sandboxes can still @@ -252,8 +255,11 @@ or E2B credential. It is available only for node-backed providers. The installer reads this configuration before preparing local assets. Its provider file records `generation` and `specification`, alongside the host's socket, paths and network policy. Enrollment at `POST /api/v1/sandbox-node/enroll` includes -`deployment_generation` and `specification_digest`. A mismatch rejects before -token consumption. Retained node authentication checks the same generation and +`deployment_generation`, `specification_digest` and `core_url`, the Core origin the +node stores. A specification mismatch rejects with 409 +`sandbox_specification_mismatch` and a `core_url` other than the installation public +URL with 409 `sandbox_node_address_mismatch`, both before token consumption. A +missing `core_url` gets 400 `invalid_request_error` with `param: "core_url"`. Retained node authentication checks the same generation and digest. A changed local resource setting, Runtime or generation must fail rather than rewrite the retained identity or silently use a local default. diff --git a/deploy/install/distribution.py b/deploy/install/distribution.py index f64e8d5ca..8c8103a31 100644 --- a/deploy/install/distribution.py +++ b/deploy/install/distribution.py @@ -111,12 +111,12 @@ def safe_url(value): return value -class SecureRedirect(urllib.request.HTTPRedirectHandler): +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Downloads come only from the console, which never redirects; a redirect + could hand verified names to another origin, so every one is refused.""" def redirect_request(self, request, fp, code, message, headers, newurl): - safe_url(newurl) - if urlsplit(request.full_url).scheme == 'https' and urlsplit(newurl).scheme != 'https': - raise DistributionError('Artifact redirect cannot downgrade HTTPS') - return super().redirect_request(request, fp, code, message, headers, newurl) + raise DistributionError('Artifact and metadata downloads do not follow redirects; check the console URL ' + 'and the reverse proxy in front of it') def checked_path(path): @@ -156,7 +156,7 @@ def obtain_artifact(manifest, logical_path, destination, offline_root=None): fd, temporary = tempfile.mkstemp(prefix='.artifact-', dir=target.parent) try: with os.fdopen(fd, 'wb') as output: - stream = source.open('rb') if source else urllib.request.build_opener(SecureRedirect()).open(url, timeout=30) + stream = source.open('rb') if source else urllib.request.build_opener(NoRedirect()).open(url, timeout=30) with stream: count = 0 while True: @@ -232,7 +232,7 @@ def read(name): url = safe_url(source_url.rstrip('/') + '/node-install/' + name) for attempt in range(3): try: - with urllib.request.build_opener(SecureRedirect()).open(url, timeout=30) as stream: + with urllib.request.build_opener(NoRedirect()).open(url, timeout=30) as stream: data = stream.read(1024 * 1024 + 1) break except urllib.error.HTTPError as error: @@ -259,8 +259,8 @@ def read(name): if (manifest.get('platform') != 'linux/amd64' or not re.fullmatch(r'[0-9a-f]{40}', manifest.get('source_commit', ''))): raise DistributionError('Unsupported distribution platform or revision') - if not manifest.get('artifact_base_url') and source_url: - manifest['artifact_base_url'] = source_url.rstrip('/') + '/node-install/artifacts' + # Artifacts come only from the console, never from a release URL the build recorded. + manifest['artifact_base_url'] = source_url.rstrip('/') + '/node-install/artifacts' if source_url and offline_root is None else '' return manifest except (ValueError, TypeError, AttributeError): raise DistributionError('Invalid distribution metadata') from None diff --git a/deploy/install/node_install.py b/deploy/install/node_install.py index b49259f6d..d8b550eb1 100644 --- a/deploy/install/node_install.py +++ b/deploy/install/node_install.py @@ -11,6 +11,7 @@ from pathlib import Path import platform import re +import shutil import socket import stat import subprocess @@ -53,17 +54,53 @@ def origin(value): return value.rstrip("/") -def checked(arguments, failure, **kwargs): +def checked(arguments, failure, explain=None, **kwargs): + # explain(stderr) may replace the failure with an InstallError carrying fixed text only. try: result = subprocess.run(arguments, check=False, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=kwargs.pop("timeout", 30), **kwargs) if result.returncode: - raise InstallError(failure) + raise explain(result.stderr) if explain else InstallError(failure) return result.stdout.decode().strip() except (OSError, subprocess.SubprocessError): raise InstallError(failure) from None +REGISTRATION_UNCONFIRMED = ("Node enrollment was not confirmed. Check the Core URL, enrollment expiry and local provider " + "prerequisites; keep its state and rerun the command to recover.") +# The node program's rejection line: Core's status and fixed error code, nothing else. +REJECTION = re.compile(rb"node enrollment rejected \(HTTP (\d{3})(?: ([a-z_]{1,64}))?\)") + + +class AddressChanged(InstallError): + """Core refused the node's address before consuming the token, so it has no record of this node.""" + + +def registration_failure(stderr): + """A fixed message for Core's answer to registration; never the node program's own text.""" + matches = list(REJECTION.finditer(stderr or b"")) + if not matches: + return InstallError(REGISTRATION_UNCONFIRMED) + status, code = matches[-1].group(1).decode(), (matches[-1].group(2) or b"").decode() + if code == "sandbox_node_address_mismatch": + return AddressChanged(node_spec.PUBLIC_URL_CHANGED + " The token was not used; downloaded files are kept.") + if status == "401": + return InstallError("The enrollment command expired or was already used. Generate a new command on the Nodes " + "page and run it on this host; downloaded files are kept.") + return InstallError(REGISTRATION_UNCONFIRMED + " Core answered HTTP " + status + (" " + code if code else "") + ".") + + +def discard_unregistered(root): + """Remove the files that name the old address, so a new command can register this host. + + Only for a node Core never recorded: downloads, the image and the service file stay.""" + for name in ("installation.json", "provider.json"): + if existing_file(root / name): + (root / name).unlink() + if (root / "state/node").is_dir() and not (root / "state/node").is_symlink(): + shutil.rmtree(root / "state/node") + + def preflight(provider): if sys.version_info < (3, 9) or platform.system() != "Linux" or platform.machine() not in ("x86_64", "amd64") or os.getuid() == 0: raise InstallError("Run with Python 3.9+ as a non-root user on Linux amd64") @@ -136,8 +173,8 @@ def read(name): distribution.image_identities(manifest, "runtime") for name in (COMMON[0], "images/runtime.tar.gz") + MICRO: distribution.artifact(manifest, name) - if not manifest.get("artifact_base_url") and source: - manifest["artifact_base_url"] = source + "/node-install/artifacts" + # Nodes download only from their console, never from a release URL the build recorded. + manifest["artifact_base_url"] = source + "/node-install/artifacts" if source else "" return manifest, sums @@ -272,10 +309,13 @@ def matches(): def service_unit(root): def quote(value): return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"' - return ("[Unit]\nDescription=Parsar sandbox node\n\n[Service]\nType=exec\nExecStart=:" + # The node keeps retrying while Core is unreachable (no start limit). It exits 78 + # when Core rejects its credential, as after removal; that ends the restarts. + return ("[Unit]\nDescription=Parsar sandbox node\nStartLimitIntervalSec=0\n\n[Service]\nType=exec\nExecStart=:" + quote(root / COMMON[0]) + " run --config " + quote(root / "provider.json") + " --state-dir " + quote(root / "state/node") + "\nWorkingDirectory=" + str(root).replace("%", "%%") - + "\nRestart=on-failure\nKillMode=process\nUMask=0077\n\n[Install]\nWantedBy=default.target\n") + + "\nRestart=on-failure\nRestartSec=5s\nRestartPreventExitStatus=78\nKillMode=process\nUMask=0077" + + "\n\n[Install]\nWantedBy=default.target\n") def install(args, token): @@ -347,9 +387,13 @@ def install(args, token): with os.fdopen(descriptor, "w") as secret: secret.write(token) print("Registering this node with Core...", flush=True) - checked([str(root / COMMON[0]), "register", "--config", str(root / "provider.json"), "--state-dir", str(root / "state/node"), - "--core-url", args.core_url, "--name", socket.gethostname(), - "--enrollment-token-file", secret_path], "Node enrollment was not confirmed. Check the Core URL, enrollment expiry and local provider prerequisites; keep its state and rerun the command to recover.") + try: + checked([str(root / COMMON[0]), "register", "--config", str(root / "provider.json"), "--state-dir", str(root / "state/node"), + "--core-url", args.core_url, "--name", socket.gethostname(), + "--enrollment-token-file", secret_path], REGISTRATION_UNCONFIRMED, explain=registration_failure) + except AddressChanged: + discard_unregistered(root) + raise write_once(marker, json_text(state)) finally: if os.path.exists(secret_path): diff --git a/deploy/install/node_spec.py b/deploy/install/node_spec.py index fffb44877..3dff5f8ea 100644 --- a/deploy/install/node_spec.py +++ b/deploy/install/node_spec.py @@ -12,6 +12,10 @@ class SpecificationError(Exception): pass +PUBLIC_URL_CHANGED = ("Core's public URL changed after this command was generated. Generate a new command on the " + "Nodes page and run it on this host.") + + def release(manifest): return {"source_commit": manifest["source_commit"], "image_id": manifest["images"]["runtime"], @@ -36,6 +40,8 @@ def digest(provider, specification): def validate(data, args): + if isinstance(data, dict) and isinstance(data.get("core_url"), str) and data["core_url"] != args.core_url: + raise SpecificationError(PUBLIC_URL_CHANGED) try: provider, spec = data["provider"], data["specification"] if (provider not in ("docker", "microsandbox") or data["installation_id"] != args.installation_id diff --git a/deploy/install/test_distribution.py b/deploy/install/test_distribution.py index a55cd3cea..bdb9b7773 100644 --- a/deploy/install/test_distribution.py +++ b/deploy/install/test_distribution.py @@ -2,6 +2,7 @@ import gzip import hashlib import http.server +import io import json from pathlib import Path import tempfile @@ -28,6 +29,8 @@ class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): test.requests.append(self.path) self.send_response(test.status) + if test.status == 302: + self.send_header('Location', 'https://elsewhere.example' + self.path) self.end_headers() if test.status == 200: self.wfile.write(test.data) @@ -120,6 +123,17 @@ def test_url_and_path_boundaries(self): distribution.obtain_artifact(self.manifest, 'native/bin/node', self.root / 'link') + def test_redirects_are_refused(self): + # The console never redirects; following one could fetch verified names from another origin. + self.status = 302 + with self.assertRaisesRegex(distribution.DistributionError, 'do not follow redirects'): + distribution.obtain_artifact(self.manifest, 'native/bin/node', self.root / 'node') + with self.assertRaisesRegex(distribution.DistributionError, 'do not follow redirects'): + distribution.load_manifest(source_url=f'http://127.0.0.1:{self.server.server_port}') + self.assertEqual(len(self.requests), 2) + self.assertEqual(list(self.root.iterdir()), []) + + class DockerIdentityTests(unittest.TestCase): def setUp(self): self.config = 'sha256:' + 'a' * 64 @@ -183,5 +197,25 @@ def test_both_metadata_identities_are_required_before_docker_or_archive(self): self.manifest[field] = original +class ManifestSourceTests(unittest.TestCase): + """A release URL recorded by the build is never an artifact source.""" + + def test_console_is_the_only_artifact_source(self): + manifest = json.dumps({'source_commit': 'a' * 40, 'platform': 'linux/amd64', + 'artifact_base_url': 'https://github.com/example/releases/download/tag'}).encode() + sums = (hashlib.sha256(manifest).hexdigest() + ' manifest.json\n').encode() + root = Path.home() / '.parsar/tests/distribution' + root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as bundle: + (Path(bundle) / 'manifest.json').write_bytes(manifest) + (Path(bundle) / 'SHA256SUMS').write_bytes(sums) + self.assertEqual(distribution.load_manifest(offline_root=bundle)['artifact_base_url'], '') + files = {'SHA256SUMS': sums, 'manifest.json': manifest} + opener = Mock(open=lambda url, timeout: io.BytesIO(files[url.rsplit('/', 1)[1]])) + with patch.object(distribution.urllib.request, 'build_opener', return_value=opener): + loaded = distribution.load_manifest(source_url='https://console.example') + self.assertEqual(loaded['artifact_base_url'], 'https://console.example/node-install/artifacts') + + if __name__ == '__main__': unittest.main() diff --git a/deploy/install/test_node_install.py b/deploy/install/test_node_install.py index 49d42b214..f653f5534 100644 --- a/deploy/install/test_node_install.py +++ b/deploy/install/test_node_install.py @@ -40,6 +40,7 @@ def setUp(self): self.calls = [] self.fail_service = False self.fail_registration = False + self.register_stderr = None for patch in (mock.patch.object(installer.Path, "home", return_value=self.home), mock.patch.object(installer, "preflight"), mock.patch.object(installer, "wait_ready"), @@ -64,6 +65,8 @@ def configuration_response(self, request, **kwargs): return io.BytesIO(json.dumps(configuration).encode()) def artifact_response(self, url, **kwargs): + # The fixture manifest records a release URL; nodes still download from their console. + self.assertTrue(url.startswith(self.args.source_url + "/node-install/artifacts/"), url) for name, item in self.manifest["artifacts"].items(): if url.endswith("/" + item["filename"]): return io.BytesIO(self.payloads[name]) @@ -95,6 +98,8 @@ def checked(self, arguments, failure, **kwargs): path = self.root / "state/node/identity.json" path.write_text(json.dumps({"identity": identity, "credential": "a" * 64, "core_url": self.args.core_url})) path.chmod(0o600) + if self.register_stderr: + raise kwargs["explain"](self.register_stderr) if self.fail_registration: raise installer.InstallError(failure) if "enable" in arguments and self.fail_service: @@ -131,6 +136,9 @@ def test_docker_installs_matched_payload_registers_and_starts_persistent_service unit = (self.root / ("parsar-node-" + self.args.installation_id + ".service")).read_text() self.assertIn(" run --config ", unit) self.assertIn("KillMode=process", unit) + # Keeps retrying while Core is down; stops once Core rejects the removed node's credential. + self.assertIn("StartLimitIntervalSec=0", unit) + self.assertIn("RestartPreventExitStatus=78", unit) self.assertNotIn("synthetic-once-token", unit) self.assertEqual(list(self.root.glob(".enrollment-*")), []) self.assertTrue(any("register" in call for call, _ in self.calls)) @@ -202,6 +210,22 @@ def test_unconfirmed_registration_retains_state_removes_token_and_does_not_start self.assertEqual(list(self.root.glob(".enrollment-*")), []) self.assertFalse(any("enable" in call for call, _ in self.calls)) + def test_registration_failure_names_only_cores_fixed_answer(self): + self.assertEqual(str(installer.registration_failure(b"secret /home/path details")), installer.REGISTRATION_UNCONFIRMED) + + def test_changed_public_url_clears_unregistered_state_for_a_new_command(self): + # Core refused the address before consuming the token, so it has no record of this node. + self.register_stderr = b"node enrollment rejected (HTTP 409 sandbox_node_address_mismatch)\n" + with self.assertRaisesRegex(installer.InstallError, "public URL changed.*token was not used"): + self.install() + for name in ("installation.json", "provider.json", "state/node/identity.json", "registered.json"): + self.assertFalse((self.root / name).exists(), name) + self.assertTrue((self.root / installer.COMMON[0]).is_file()) + self.register_stderr = None + self.args.core_url = "https://core-new.example" + self.install() + self.assertEqual(json.loads((self.root / "registered.json").read_text())["core_url"], "https://core-new.example") + def test_microsandbox_registration_retry_retains_original_dns_policy(self): self.args.provider = "microsandbox" self.fail_registration = True diff --git a/deploy/install/test_node_spec.py b/deploy/install/test_node_spec.py index 2c0a30e63..b6b641e00 100644 --- a/deploy/install/test_node_spec.py +++ b/deploy/install/test_node_spec.py @@ -55,6 +55,11 @@ def test_partial_registration_can_use_enrollment_after_unauthenticated_identity( node_spec.fetch(self.args, token, self.retained, opener, allow_enrollment=allowed) self.assertEqual(opener.call_count, 1) + def test_changed_public_url_says_to_generate_a_new_command(self): + self.data["core_url"] = "https://core-new.example" + with self.assertRaisesRegex(node_spec.SpecificationError, "public URL changed"): + node_spec.fetch(self.args, "once", None, mock.Mock(return_value=self.response())) + def test_failures_name_their_cause(self): for failure, message in ((urllib.error.HTTPError("https://core.example", 404, "", {}, None), "route /api/v1"), (urllib.error.HTTPError("https://core.example", 409, "", {}, None), "maintenance"), diff --git a/docs/api/web-management.md b/docs/api/web-management.md index bf09da039..4a25ee63d 100644 --- a/docs/api/web-management.md +++ b/docs/api/web-management.md @@ -15,7 +15,7 @@ The browser uses the console's own origin and signs in with the | `GET /console/auth` | No body | `200 {"mode":"login"}` or `200 {"mode":"authenticated"}` | | `POST /console/auth/login` | `Content-Type: application/json`, body `{"core_key":"…"}`; other members are rejected | `200 {"mode":"authenticated"}` and an HttpOnly, SameSite=Strict session cookie (Secure over HTTPS) | | `POST /console/auth/logout` | No credential payload | `200 {"mode":"login"}`; clears the cookie and the server-side session | -| `GET /console/config` | Signed-in session | `node_installer`, `node_installer_sha256`, `self_hosted_installer` and `self_hosted_installer_sha256` | +| `GET /console/config` | Signed-in session | `node_installer`, `node_installer_sha256`, `self_hosted_installer`, `self_hosted_installer_sha256`, and `node_artifacts`: the providers (`docker`, `microsandbox`) whose node assets this console holds | Sign-in errors use the console's `{"error": "…"}` envelope: 400 for a malformed body, 401 for a wrong key, 415 for a non-JSON body, 429 with `Retry-After` when @@ -79,6 +79,7 @@ define deployment and node operations: - `GET /core/v1/sandbox/nodes`, `PATCH/DELETE /core/v1/sandbox/nodes/{node_id}`, and `GET /core/v1/sandbox/nodes/{node_id}/allocations`. - `POST /core/v1/sandbox/enrollment-tokens` for a one-time node installation command. + Its non-secret `enrollment_id` reappears on the node that command registers. PostgreSQL owns one provider, per-sandbox resource specification and immutable Runtime selection. POST initializes it; PUT replaces the complete selection using diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index 86ee21b78..15a77fec4 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -61,9 +61,14 @@ Adding a node downloads only its selected provider's assets. Runtime images use compressed archives; verified local files and already imported images are reused. Downloads use temporary files and bounded retries, so a truncated response is never promoted into the cache. An optional `-offline.tar.gz` bundle contains the -same assets locally. For console-based distribution without a release host, build -that offline bundle with no release URL. A bundle that records a release URL -retains that URL for remote node downloads; it does not silently change mirrors. +same assets locally. + +Nodes download these assets only from the console (Web) that generated their +command, never from a release URL the build recorded, so node hosts need no access +to GitHub. The installer copies them into the console's node payload from the +offline bundle, or from the release's `parsar-core--linux-amd64-*` assets +if you download them into the thin bundle's `artifacts/` directory before running +`install.sh`. A console without them does not offer Add node for that provider. ## Sign in to Web @@ -138,12 +143,13 @@ Choose English or Chinese through the System language selector. maintenance and completed cleanup. Microsandbox uses a five-minute idle timeout and one-day snapshot retention. 4. For own-machine hosting, click **Add node**, copy the command, and run it as a non-root user on the - target Linux amd64 host. It downloads the matched bootstrap from your console and execution assets from - the manifest's release location (or the offline console payload), verifies + target Linux amd64 host. It downloads the matched bootstrap and execution assets from your console + (never from a release URL), verifies checksums, reads the saved generation and specification without consuming enrollment, verifies the payload matches that Runtime, imports the image only when missing, and writes the matching provider configuration, - registers the node and starts a systemd user service. The installer waits for Core to confirm + registers the node and starts a systemd user service. The service keeps retrying while Core + is unreachable and stops for good once the node is removed. The installer waits for Core to confirm connection and provider readiness. Web refreshes node health automatically. Wait for the node to be online and its provider to be ready; registration alone does not mean it can accept work. A registered retry uses its @@ -257,7 +263,8 @@ container. ./install.sh --core-only --sandbox-provider true --provider docker --public-url https://core.example ``` -`--sandbox-provider true` installs a local node through the ordinary node installer. If `--provider` is +`--sandbox-provider true` installs a local node through the ordinary node installer, using the node assets in +the bundle: use the offline bundle, or place the release assets in its `artifacts/` directory. If `--provider` is omitted, it selects microsandbox. Supplying `--provider` without enabling the sandbox provider is an error. `--core-only` installs Core and PostgreSQL without Web and has no sandbox provider unless explicitly enabled. @@ -407,9 +414,8 @@ exact bundle before distribution. See the [contributor guide](https://github.com The release base must host the generated flat asset filenames over HTTPS. Use `CORE_DISTRIBUTION_OFFLINE=1` to also emit a full offline archive; a build without -any release URL must select offline mode. A fully disconnected console deployment -uses that empty-URL offline build; remote node installers then obtain assets from -the console. The release workflow prepares pinned +any release URL must select offline mode. Node installers always obtain assets +from the console, whichever build recorded a release URL. The release workflow prepares pinned harness dependencies, builds versioned assets, and uploads an Actions artifact. An explicit manual option can create an unpublished draft release. Neither a successful build nor a draft makes a private repository anonymously downloadable; @@ -444,8 +450,8 @@ same release identity. Publishing a Release does not change repository visibilit For an offline installation, extract the matching `-offline.tar.gz` archive and run its `install.sh`, which copies the bundled assets into the console's node payload. `install.sh` has no `--offline-root` option; only `self-hosted-install.pyz` accepts -one, for an executor host. Remote node commands use the manifest's release URL; use -the explicitly configured console-hosted offline build described above when node -hosts cannot access that URL. Download access errors should be fixed at the +one, for an executor host. Node commands always download from the console that +generated them, so install from the offline bundle (or place the release assets in +the bundle) to let the console serve nodes. Download access errors should be fixed at the distribution source, without passing repository credentials into Runtime or changing its executor authorization. diff --git a/docs/web/protocol-coverage.md b/docs/web/protocol-coverage.md index cd0b0a167..087fc8e3a 100644 --- a/docs/web/protocol-coverage.md +++ b/docs/web/protocol-coverage.md @@ -15,7 +15,7 @@ the administrator API are defined by the [administrator API contract](../../cont | Interface | Paths | Authentication | Console use | | --- | --- | --- | --- | -| Console server | `/console/auth`, `/console/auth/{login,logout}`, `/console/config` | Core key at sign-in, then the console session cookie | Sign-in with the Core key and sign-out; the node installer (`node_installer`, `node_installer_sha256`) | +| Console server | `/console/auth`, `/console/auth/{login,logout}`, `/console/config` | Core key at sign-in, then the console session cookie | Sign-in with the Core key and sign-out; the node installer (`node_installer`, `node_installer_sha256`), the self-hosted executor installer (`self_hosted_installer`, `self_hosted_installer_sha256`) and the providers whose node assets it holds (`node_artifacts`) | | Administrator API | `/core/v1/**` outside `/core/v1/sandbox` | Core key, added by the console server | Projects, keys, resource reads and deletion, executor credentials, provenance, summaries, Core metrics, the installation | | Sandbox administration | `/core/v1/sandbox/**` | Core key, added by the console server | Nodes page; fleet and capacity figures on Overview and Sandbox metrics; Runtime observations of every project | | Agents API | `/v1/**` | Project API key | Not used. Wherever a new key is shown, the console gives shell exports of `OPENAI_BASE_URL` (the installation's `api_base_url`) and `OPENAI_API_KEY` (the new key) with `curl` and Python examples for `/v1/agents`, and sends none of them; when the installation is `local_only` it says the API is reachable only on the Core machine, and without an `api_base_url` it says to set `public_url` | @@ -130,7 +130,9 @@ the sandbox deployment. | Runtime observations | `GET /core/v1/sandbox/runtime-observations` | Sandbox metrics: hosted Runtimes of every project, each labelled with its project; an E2B sandbox's dialog adds its `observation.disk` as used / limit (null elsewhere) | Signing in grants administration, so `/console/config` reports only the node -installer (`node_installer`, `node_installer_sha256`). These pages appear unless +installer (`node_installer`, `node_installer_sha256`), the self-hosted executor +installer (`self_hosted_installer`, `self_hosted_installer_sha256`) and the providers +whose node assets the console holds (`node_artifacts`). These pages appear unless the console has no `/console/config` (404) or reports `sandbox_admin: false`. An E2B deployment has no nodes; its API key is write-only. The Runtime release sent for Docker and microsandbox comes from the console's own `GET /node-install/manifest.json` diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 11f394330..7eb4dda35 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -75,6 +75,19 @@ export interface SandboxNode { created_at: string; /** Read-only: the Core address this node enrolled with. When it differs from the installation public URL, the node receives no new sandboxes and must be re-added. */ readonly core_url: string; + /** + * Read-only: the `enrollment_id` of the command that registered this node; null for nodes enrolled + * before Core recorded it. Core always sends the member; it is optional here only until Web's fixtures include it. + */ + readonly enrollment_id?: string | null; +} +/** A one-time node enrollment command issued by Core. */ +export interface SandboxEnrollment { + /** The one-use secret the node registers with. */ + token: string; + expires_at: string; + /** Public, non-secret handle of this command; never a credential. The node it registers reports the same `enrollment_id`. */ + enrollment_id: string; } /** The node machine's last heartbeat observation; unavailable measurements are null. */ export interface SandboxNodeHost { @@ -170,7 +183,7 @@ export class SandboxAdminClient { listAllocations(nodeId: string, options?: ReadOptions): Promise<{ data: SandboxAllocation[] }> { return this.#json(`/nodes/${encodeURIComponent(nodeId)}/allocations`, options); } - createEnrollment(options?: ReadOptions, capacity: { max_active?: number; max_retained?: number } = {}): Promise<{ token: string; expires_at: string }> { + createEnrollment(options?: ReadOptions, capacity: { max_active?: number; max_retained?: number } = {}): Promise { return this.#json("/enrollment-tokens", options, "POST", capacity); } updateNode(nodeId: string, input: SandboxNodeUpdate, options?: ReadOptions): Promise<{ id: string; updated: boolean }> { diff --git a/services/agents-api/HOSTED-SANDBOX-MANAGER.md b/services/agents-api/HOSTED-SANDBOX-MANAGER.md index 5c74136bd..d0c5c42ed 100644 --- a/services/agents-api/HOSTED-SANDBOX-MANAGER.md +++ b/services/agents-api/HOSTED-SANDBOX-MANAGER.md @@ -150,7 +150,10 @@ The current installation path uses a database-managed deployment. The paired console provides a complete installation command. It downloads only a matched bootstrap from `/node-install/`, then checksum-verified prebuilt assets -from the manifest release URL or offline console payload. It reuses verified cache +from the same console's payload, never from a release URL the build recorded. The +console must hold them: install from the offline bundle, or place the release assets +in the bundle's `artifacts/` directory first; `/console/config` lists the providers +whose assets it holds (`node_artifacts`). It reuses verified cache entries and exact imported images, then waits for Core to confirm connection and provider readiness. The enrollment token is transient and never a console/project credential. Python 3.9+, a systemd user @@ -175,6 +178,13 @@ credentials; Web does not forward them and no administrator credential applies: | `GET /api/v1/sandbox-node/identity?node_id=` | Node credential | | WebSocket `GET /api/v1/sandbox-node/connect?node_id=` | Node credential | +Enrollment sends the Core origin the node stores (`core_url`, from `register +--core-url`). Core refuses one that is not the installation public URL with 409 +`sandbox_node_address_mismatch`, before consuming the token, so a node never records +an address Core no longer uses. `POST /core/v1/sandbox/enrollment-tokens` also returns +a non-secret `enrollment_id`; the node it registers reports the same value in +`/core/v1/sandbox/nodes`, and nodes enrolled before Core recorded it report null. + A node and its Core must come from the same distribution. Nodes from releases that used the removed `/core/v1/sandbox` node paths cannot connect to this Core: drain with the previous release, then upgrade and enroll new nodes, as described @@ -262,6 +272,10 @@ unknown operations or cleanup records cannot be removed. Resolve those resources through their normal lifecycle and then retry; the API reports the conflict. Offline resources remain owned and visible. Explicit removal permanently retires the node identity; adding that host again requires a fresh private state directory. +The node program then exits with status 78 when Core answers 401 to its credential, +and the installed service does not restart it. Other failures, including an +unreachable Core or a 403 from a proxy in front of it, restart the service every +5 seconds without a start limit. Ordinary disconnects and host restarts reuse the original identity. A local node uses the same removal and resource checks as any other enrolled node. diff --git a/services/agents-api/cmd/sandbox-node/main.go b/services/agents-api/cmd/sandbox-node/main.go index 4c57b6e7c..a23267746 100644 --- a/services/agents-api/cmd/sandbox-node/main.go +++ b/services/agents-api/cmd/sandbox-node/main.go @@ -26,9 +26,22 @@ func main() { defer stop() if err := run(ctx, os.Args[1:]); err != nil { fmt.Fprintln(os.Stderr, err) - os.Exit(1) + os.Exit(exitCode(err)) } } + +// exitRejected tells the service manager not to restart the node: Core rejected +// its credential because the node was removed or retired, so no retry can succeed. +// The node units set RestartPreventExitStatus to this value (EX_CONFIG). +const exitRejected = 78 + +func exitCode(err error) int { + if errors.Is(err, node.ErrAuthentication) { + return exitRejected + } + return 1 +} + func run(ctx context.Context, args []string) error { if len(args) == 0 || (args[0] != "register" && args[0] != "run") { return errors.New("usage: parsar-sandbox-node register|run --config PATH --state-dir PATH") diff --git a/services/agents-api/cmd/sandbox-node/main_test.go b/services/agents-api/cmd/sandbox-node/main_test.go new file mode 100644 index 000000000..817b19ffa --- /dev/null +++ b/services/agents-api/cmd/sandbox-node/main_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox/node" + "github.com/google/uuid" +) + +// Only Core's rejection of the node credential stops the node service for good: +// exit 78, which the unit's RestartPreventExitStatus honors. Every other failure +// exits 1, so systemd restarts the node. +func TestExitCodeStopsTheNodeOnlyForARejectedCredential(t *testing.T) { + answering := func(status int) string { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) })) + t.Cleanup(server.Close) + return server.URL + } + closed := httptest.NewServer(http.NotFoundHandler()) + closed.Close() + for _, test := range []struct { + name string + err error + want int + }{ + {"credential rejected", refreshIdentity(t, answering(http.StatusUnauthorized)), exitRejected}, + {"proxy forbids", refreshIdentity(t, answering(http.StatusForbidden)), 1}, + {"Core unavailable", refreshIdentity(t, answering(http.StatusServiceUnavailable)), 1}, + {"Core unreachable", refreshIdentity(t, closed.URL), 1}, + {"specification changed", errors.New("provider configuration differs from retained node identity"), 1}, + } { + if test.err == nil { + t.Fatal(test.name, "refresh succeeded") + } + if got := exitCode(test.err); got != test.want { + t.Errorf("%s: exit %d, want %d (%v)", test.name, got, test.want, test.err) + } + } +} + +// refreshIdentity runs the node's startup identity refresh against coreURL. +func refreshIdentity(t *testing.T, coreURL string) error { + t.Helper() + dir := filepath.Join(t.TempDir(), "state") + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + identity := node.Identity{InstallationID: uuid.NewString(), Provider: "docker", BackendFingerprint: strings.Repeat("1", 64)} + if _, err := node.InitIdentity(dir, coreURL, identity); err != nil { + t.Fatal(err) + } + _, err := node.RefreshIdentity(t.Context(), dir) + return err +} diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index b3f73478e..648477b50 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -94,6 +94,8 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound writeError(w, http.StatusBadRequest, "invalid_sandbox_configuration", sandboxConfiguration.Message) case errors.Is(err, store.ErrSandboxPublicURLUnreachable): writeError(w, http.StatusConflict, "sandbox_configuration_error", err.Error()) + case errors.Is(err, store.ErrRuntimeNodeAddressMismatch): + writeError(w, http.StatusConflict, "sandbox_node_address_mismatch", "This node uses a different Core address than the installation public URL. Generate a new command on the Nodes page and run it on the host.") case errors.Is(err, store.ErrRuntimeSpecificationMismatch): writeError(w, http.StatusConflict, "sandbox_specification_mismatch", "The node resource limits or Runtime release do not match the active deployment. Restore its installed configuration or remove and enroll the node again after a drained deployment change.") case errors.Is(err, store.ErrProjectArchived): diff --git a/services/agents-api/internal/api/errors_test.go b/services/agents-api/internal/api/errors_test.go index b768c375c..21a243b49 100644 --- a/services/agents-api/internal/api/errors_test.go +++ b/services/agents-api/internal/api/errors_test.go @@ -127,6 +127,7 @@ func TestConflictErrorsUseConflictType(t *testing.T) { store.ErrSandboxDeploymentConflict: "sandbox_deployment_conflict", store.ErrRuntimeNodeInUse: "runtime_node_in_use", store.ErrRuntimeLocalNodeConfigured: "runtime_local_node_configured", + store.ErrRuntimeNodeAddressMismatch: "sandbox_node_address_mismatch", store.ErrEnvironmentUnavailable: "environment_unavailable", execution.ErrEnvironmentInputExpired: "environment_input_expired", execution.ErrEnvironmentInputCancelled: "environment_input_cancelled", diff --git a/services/agents-api/internal/api/sandbox_manager.go b/services/agents-api/internal/api/sandbox_manager.go index e26422e9d..6dd20649f 100644 --- a/services/agents-api/internal/api/sandbox_manager.go +++ b/services/agents-api/internal/api/sandbox_manager.go @@ -17,6 +17,8 @@ type SandboxAllocationList struct { type SandboxEnrollmentToken struct { Token string `json:"token"` ExpiresAt time.Time `json:"expires_at"` + // Public, non-secret handle of this command; never a credential. The node it registers reports the same value as enrollment_id. + EnrollmentID string `json:"enrollment_id"` } type SandboxMutationResponse struct { ID string `json:"id"` @@ -188,16 +190,16 @@ func (h *Handler) createSandboxEnrollment(w http.ResponseWriter, r *http.Request if input.MaxRetained != nil { capacity.MaxRetained = *input.MaxRetained } - token, expires, err := h.sandboxStore.CreateRuntimeEnrollment(r.Context(), capacity) + enrollment, err := h.sandboxStore.CreateRuntimeEnrollment(r.Context(), capacity) if err != nil { writeStoreError(w, r, err) return } - writeJSON(w, http.StatusCreated, SandboxEnrollmentToken{Token: token, ExpiresAt: expires}) + writeJSON(w, http.StatusCreated, SandboxEnrollmentToken{Token: enrollment.Token, ExpiresAt: enrollment.ExpiresAt, EnrollmentID: enrollment.ID}) } // @Summary Enroll a sandbox node -// @Description Node machine connection. Consumes a one-use enrollment token; grants no project or administrator access. Responses contain only explicit safe fields. +// @Description Node machine connection. Consumes a one-use enrollment token; grants no project or administrator access. Responses contain only explicit safe fields. core_url is required and must equal the installation public URL; a different address gets 409 sandbox_node_address_mismatch and leaves the token unused. // @Tags Sandbox Node // @Produce json // @Security NodeEnrollmentAuth @@ -217,10 +219,14 @@ func (h *Handler) enrollSandboxNode(w http.ResponseWriter, r *http.Request) { return } var input store.RuntimeNodeEnrollment - if decodeInputObject(raw, &input, "node_id", "credential", "name", "provider", "backend_fingerprint", "deployment_generation", "specification_digest") != nil { + if decodeInputObject(raw, &input, "node_id", "credential", "name", "provider", "backend_fingerprint", "deployment_generation", "specification_digest", "core_url") != nil { writeStoreError(w, r, store.ErrInvalidInput) return } + if input.CoreURL == "" { + writeError(w, http.StatusBadRequest, "invalid_request_error", "Enrollment requires core_url, the Core origin this node uses. Install the node with this Core's node installer.", "core_url") + return + } value, err := h.sandboxStore.EnrollRuntimeNode(r.Context(), token, input) if err != nil { writeStoreError(w, r, err) diff --git a/services/agents-api/internal/api/sandbox_manager_test.go b/services/agents-api/internal/api/sandbox_manager_test.go index eed1b2bd4..6cfc1b024 100644 --- a/services/agents-api/internal/api/sandbox_manager_test.go +++ b/services/agents-api/internal/api/sandbox_manager_test.go @@ -110,3 +110,21 @@ func TestSandboxEnrollmentCapacityIsAdministratorOnly(t *testing.T) { } } } + +// Enrollment names the Core address the node uses; without it the request fails +// before any token is read. +func TestSandboxNodeEnrollmentRequiresCoreURL(t *testing.T) { + project, _ := NewAuthenticator([]APIKey{callerBinding()}) + admin, _ := NewDeploymentAuthenticator([]string{device.HashCredential("administrator")}) + h, err := NewHandler(&recordingStore{}, project, "codex", WithSandboxManager(&store.Store{}, admin)) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/v1/sandbox-node/enroll", strings.NewReader(`{"node_id":"node","name":"node"}`)) + request.Header.Set("Authorization", "Bearer one-use") + response := httptest.NewRecorder() + h.ServeHTTP(response, request) + if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), `"param":"core_url"`) { + t.Fatal(response.Code, response.Body.String()) + } +} diff --git a/services/agents-api/internal/db/queries/runtime_nodes.sql b/services/agents-api/internal/db/queries/runtime_nodes.sql index 290e00e91..6e9d6ff34 100644 --- a/services/agents-api/internal/db/queries/runtime_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_nodes.sql @@ -5,8 +5,8 @@ UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', SELECT * FROM runtime_deployment WHERE singleton=true; -- name: InsertRuntimeNode :one -INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING *; +INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING *; -- name: GetRuntimeNode :one SELECT * FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL; @@ -46,7 +46,7 @@ SELECT id FROM runtime_nodes WHERE id=$1 FOR UPDATE; UPDATE runtime_nodes SET connection_id=NULL FROM runtime_deployment d WHERE runtime_nodes.id=$1 AND connection_id=$2 AND connected_epoch=d.owner_epoch AND d.owner_epoch=sqlc.arg(owner_epoch); -- name: CreateRuntimeEnrollment :exec -INSERT INTO runtime_node_enrollments(token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,clock_timestamp()+interval '10 minutes',$3,$4); +INSERT INTO runtime_node_enrollments(id,token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,$3,clock_timestamp()+interval '10 minutes',$4,$5); -- name: ConsumeRuntimeEnrollment :execrows UPDATE runtime_node_enrollments SET consumed_at=clock_timestamp(),node_id=$2 diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index a1ee51ee1..e07fa711b 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -310,6 +310,7 @@ type RuntimeNode struct { SpecificationDigest string `json:"specification_digest"` DeploymentGeneration int64 `json:"deployment_generation"` CoreUrl string `json:"core_url"` + EnrollmentID pgtype.UUID `json:"enrollment_id"` } type RuntimeNodeEnrollment struct { @@ -320,6 +321,7 @@ type RuntimeNodeEnrollment struct { NodeID pgtype.UUID `json:"node_id"` MaxActive int32 `json:"max_active"` MaxRetained int32 `json:"max_retained"` + ID pgtype.UUID `json:"id"` } type RuntimePlacement struct { diff --git a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go index f59b6ee2e..75d616d83 100644 --- a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go @@ -95,10 +95,11 @@ func (q *Queries) ConsumeRuntimeEnrollment(ctx context.Context, arg ConsumeRunti } const createRuntimeEnrollment = `-- name: CreateRuntimeEnrollment :exec -INSERT INTO runtime_node_enrollments(token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,clock_timestamp()+interval '10 minutes',$3,$4) +INSERT INTO runtime_node_enrollments(id,token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,$3,clock_timestamp()+interval '10 minutes',$4,$5) ` type CreateRuntimeEnrollmentParams struct { + ID pgtype.UUID `json:"id"` TokenSha256 string `json:"token_sha256"` InstallationID pgtype.UUID `json:"installation_id"` MaxActive int32 `json:"max_active"` @@ -107,6 +108,7 @@ type CreateRuntimeEnrollmentParams struct { func (q *Queries) CreateRuntimeEnrollment(ctx context.Context, arg CreateRuntimeEnrollmentParams) error { _, err := q.db.Exec(ctx, createRuntimeEnrollment, + arg.ID, arg.TokenSha256, arg.InstallationID, arg.MaxActive, @@ -192,7 +194,7 @@ func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, } const getRuntimeEnrollment = `-- name: GetRuntimeEnrollment :one -SELECT token_sha256, installation_id, expires_at, consumed_at, node_id, max_active, max_retained FROM runtime_node_enrollments WHERE token_sha256=$1 +SELECT token_sha256, installation_id, expires_at, consumed_at, node_id, max_active, max_retained, id FROM runtime_node_enrollments WHERE token_sha256=$1 ` func (q *Queries) GetRuntimeEnrollment(ctx context.Context, tokenSha256 string) (RuntimeNodeEnrollment, error) { @@ -206,12 +208,13 @@ func (q *Queries) GetRuntimeEnrollment(ctx context.Context, tokenSha256 string) &i.NodeID, &i.MaxActive, &i.MaxRetained, + &i.ID, ) return i, err } const getRuntimeNode = `-- name: GetRuntimeNode :one -SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL +SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL ` func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNode, error) { @@ -235,6 +238,7 @@ func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNo &i.SpecificationDigest, &i.DeploymentGeneration, &i.CoreUrl, + &i.EnrollmentID, ) return i, err } @@ -303,8 +307,8 @@ func (q *Queries) HeartbeatRuntimeNode(ctx context.Context, arg HeartbeatRuntime } const insertRuntimeNode = `-- name: InsertRuntimeNode :one -INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url +INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id) +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id ` type InsertRuntimeNodeParams struct { @@ -318,6 +322,7 @@ type InsertRuntimeNodeParams struct { SpecificationDigest string `json:"specification_digest"` DeploymentGeneration int64 `json:"deployment_generation"` CoreUrl string `json:"core_url"` + EnrollmentID pgtype.UUID `json:"enrollment_id"` } func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodeParams) (RuntimeNode, error) { @@ -332,6 +337,7 @@ func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodePa arg.SpecificationDigest, arg.DeploymentGeneration, arg.CoreUrl, + arg.EnrollmentID, ) var i RuntimeNode err := row.Scan( @@ -352,6 +358,7 @@ func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodePa &i.SpecificationDigest, &i.DeploymentGeneration, &i.CoreUrl, + &i.EnrollmentID, ) return i, err } @@ -465,7 +472,7 @@ func (q *Queries) ListNodeRuntimeAllocations(ctx context.Context, nodeID pgtype. } const listRuntimeNodes = `-- name: ListRuntimeNodes :many -SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, +SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, d.provider_kind, (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, @@ -496,6 +503,7 @@ type ListRuntimeNodesRow struct { SpecificationDigest string `json:"specification_digest"` DeploymentGeneration int64 `json:"deployment_generation"` CoreUrl string `json:"core_url"` + EnrollmentID pgtype.UUID `json:"enrollment_id"` Online bool `json:"online"` ProviderKind string `json:"provider_kind"` Active int64 `json:"active"` @@ -533,6 +541,7 @@ func (q *Queries) ListRuntimeNodes(ctx context.Context, nodeID pgtype.UUID) ([]L &i.SpecificationDigest, &i.DeploymentGeneration, &i.CoreUrl, + &i.EnrollmentID, &i.Online, &i.ProviderKind, &i.Active, @@ -627,7 +636,7 @@ func (q *Queries) SetRuntimeObservation(ctx context.Context, arg SetRuntimeObser } const updateRuntimeNode = `-- name: UpdateRuntimeNode :one -UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url +UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id ` type UpdateRuntimeNodeParams struct { @@ -663,6 +672,7 @@ func (q *Queries) UpdateRuntimeNode(ctx context.Context, arg UpdateRuntimeNodePa &i.SpecificationDigest, &i.DeploymentGeneration, &i.CoreUrl, + &i.EnrollmentID, ) return i, err } diff --git a/services/agents-api/internal/sandbox/node/agent.go b/services/agents-api/internal/sandbox/node/agent.go index 417a85dd5..a14f5ba87 100644 --- a/services/agents-api/internal/sandbox/node/agent.go +++ b/services/agents-api/internal/sandbox/node/agent.go @@ -180,7 +180,9 @@ func (a *agent) connect(ctx context.Context) error { if err != nil { if resp != nil { _ = resp.Body.Close() - if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + // Only Core's 401 rejects the credential for good; a 403 may come from a + // proxy or firewall in front of Core, so the node keeps retrying. + if resp.StatusCode == http.StatusUnauthorized { return ErrAuthentication } } diff --git a/services/agents-api/internal/sandbox/node/enrollment.go b/services/agents-api/internal/sandbox/node/enrollment.go index 43be11b9a..4ec8660b9 100644 --- a/services/agents-api/internal/sandbox/node/enrollment.go +++ b/services/agents-api/internal/sandbox/node/enrollment.go @@ -5,9 +5,12 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" + "regexp" + "strconv" "time" ) @@ -29,6 +32,7 @@ func Enroll(ctx context.Context, coreURL, dir, token string, input EnrollmentReq input.SpecificationDigest, input.DeploymentGeneration = stored.Identity.SpecificationDigest, stored.Identity.DeploymentGeneration input.NodeID, input.Credential = stored.Identity.NodeID, stored.Credential input.Provider, input.BackendFingerprint = stored.Identity.Provider, stored.Identity.BackendFingerprint + input.CoreURL = coreURL client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} // This also recovers a consumed registration whose success response was lost. target, err := endpoint(coreURL, "/api/v1/sandbox-node/identity") @@ -74,7 +78,7 @@ func readEnrollment(r *http.Response) (EnrollmentResponse, error) { defer r.Body.Close() var out EnrollmentResponse if r.StatusCode < 200 || r.StatusCode >= 300 { - return out, errors.New("node enrollment rejected") + return out, enrollmentRejection(r) } d := json.NewDecoder(io.LimitReader(r.Body, 16384)) d.DisallowUnknownFields() @@ -137,3 +141,28 @@ func RefreshIdentity(ctx context.Context, dir string) (StoredIdentity, error) { } return retainEnrollment(dir, stored, out) } + +// coreErrorCode is the shape of Core's fixed error codes, the only part of a +// rejection body that reaches the node's output. +var coreErrorCode = regexp.MustCompile(`^[a-z_]{1,64}$`) + +// enrollmentRejection names Core's status and error code. Only Core's 401 is +// ErrAuthentication: the node was removed or retired, or the enrollment token is +// invalid, so retrying cannot succeed. A 403 may come from a proxy or firewall in +// front of Core, so it stays retryable. +func enrollmentRejection(r *http.Response) error { + var body struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + _ = json.NewDecoder(io.LimitReader(r.Body, 16384)).Decode(&body) + detail := "HTTP " + strconv.Itoa(r.StatusCode) + if coreErrorCode.MatchString(body.Error.Code) { + detail += " " + body.Error.Code + } + if r.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("node enrollment rejected (%s): %w", detail, ErrAuthentication) + } + return fmt.Errorf("node enrollment rejected (%s)", detail) +} diff --git a/services/agents-api/internal/sandbox/node/node_test.go b/services/agents-api/internal/sandbox/node/node_test.go index c32e7b1c6..7a1e523b5 100644 --- a/services/agents-api/internal/sandbox/node/node_test.go +++ b/services/agents-api/internal/sandbox/node/node_test.go @@ -244,6 +244,9 @@ func TestEnrollmentLostResponseRecoversWithPersistedCredential(t *testing.T) { if json.NewDecoder(r.Body).Decode(&body) != nil || body.Credential != stored.Credential || body.NodeID != id.NodeID { t.Error("unpersisted credential used") } + if body.CoreURL != "http://"+r.Host { + t.Error("enrollment did not name the node's Core address", body.CoreURL) + } enrollments++ registered = true hijacker := w.(http.Hijacker) @@ -310,3 +313,25 @@ func TestHealthSendsOnlyFixedDiagnosticCode(t *testing.T) { } } } + +// A rejected credential stops the node for good; any other rejection names Core's +// status and error code so the installer can say what to fix. +func TestEnrollmentRejectionNamesCoreCode(t *testing.T) { + reject := func(status int, body string) error { + recorder := httptest.NewRecorder() + recorder.WriteHeader(status) + _, _ = recorder.WriteString(body) + _, err := readEnrollment(recorder.Result()) + return err + } + if err := reject(http.StatusUnauthorized, `{"error":{"code":"invalid_node_credential"}}`); !errors.Is(err, ErrAuthentication) { + t.Fatal("a rejected credential was not an authentication failure", err) + } + if err := reject(http.StatusForbidden, `forbidden by proxy`); errors.Is(err, ErrAuthentication) { + t.Fatal("a proxy's 403 stopped the node for good", err) + } + err := reject(http.StatusConflict, `{"error":{"code":"sandbox_node_address_mismatch","message":"x"}}`) + if errors.Is(err, ErrAuthentication) || err.Error() != "node enrollment rejected (HTTP 409 sandbox_node_address_mismatch)" { + t.Fatal(err) + } +} diff --git a/services/agents-api/internal/sandbox/node/wire.go b/services/agents-api/internal/sandbox/node/wire.go index 0526e53b5..bda6b2451 100644 --- a/services/agents-api/internal/sandbox/node/wire.go +++ b/services/agents-api/internal/sandbox/node/wire.go @@ -56,6 +56,8 @@ type EnrollmentRequest struct { Name string `json:"name"` Provider string `json:"provider"` BackendFingerprint string `json:"backend_fingerprint"` + // The retained Core origin; Core refuses an enrollment whose address is not its public URL. + CoreURL string `json:"core_url"` } type EnrollmentResponse struct { diff --git a/services/agents-api/internal/store/admin_session_archive_race_test.go b/services/agents-api/internal/store/admin_session_archive_race_test.go index d31dd37ad..c0f610fdc 100644 --- a/services/agents-api/internal/store/admin_session_archive_race_test.go +++ b/services/agents-api/internal/store/admin_session_archive_race_test.go @@ -20,7 +20,7 @@ func TestManagedSessionArchiveReleasesPendingNodePlacement(t *testing.T) { if _, err := w.InitializeSandboxDeployment(t.Context(), installation, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 1}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 1})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/credential_matrix_http_test.go b/services/agents-api/internal/store/credential_matrix_http_test.go index 5d92d2506..37583d57d 100644 --- a/services/agents-api/internal/store/credential_matrix_http_test.go +++ b/services/agents-api/internal/store/credential_matrix_http_test.go @@ -25,6 +25,7 @@ import ( // own /api/v1 machine connection routes. func TestCredentialNamespaceMatrix(t *testing.T) { s, _ := store.NewManagedTestStore(t) + s.SetPublicURL("https://core.example") ctx := t.Context() coreKey := uuid.NewString() admin, err := api.NewDeploymentAuthenticator([]string{device.HashCredential(coreKey)}) @@ -117,7 +118,7 @@ func TestCredentialNamespaceMatrix(t *testing.T) { created("POST", "/core/v1/sandbox/enrollment-tokens", coreKey, `{}`, &enrollment) nodeID, nodeCredential := uuid.NewString(), strings.Repeat("n", 64) enroll, _ := json.Marshal(store.RuntimeNodeEnrollment{NodeID: nodeID, Credential: nodeCredential, Name: "Matrix node", Provider: "docker", BackendFingerprint: strings.Repeat("b", 64), - DeploymentGeneration: 1, SpecificationDigest: specification.Digest("docker")}) + DeploymentGeneration: 1, SpecificationDigest: specification.Digest("docker"), CoreURL: "https://core.example"}) var node store.RuntimeNodeIdentity created("POST", "/api/v1/sandbox-node/enroll", enrollment.Token, string(enroll), &node) diff --git a/services/agents-api/internal/store/device_bootstrap_binding_test.go b/services/agents-api/internal/store/device_bootstrap_binding_test.go index 79dfe8841..1cda38083 100644 --- a/services/agents-api/internal/store/device_bootstrap_binding_test.go +++ b/services/agents-api/internal/store/device_bootstrap_binding_test.go @@ -12,7 +12,7 @@ import ( func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { s, writer, deployment := managerFixture(t, 4, 8) - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 8}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 8})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_lifecycle_nodes_test.go b/services/agents-api/internal/store/runtime_lifecycle_nodes_test.go index 0e12126a0..f9abc29ff 100644 --- a/services/agents-api/internal/store/runtime_lifecycle_nodes_test.go +++ b/services/agents-api/internal/store/runtime_lifecycle_nodes_test.go @@ -12,7 +12,7 @@ import ( func lifecycleTestNode(t *testing.T, s *Store) string { t.Helper() - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 100, MaxRetained: 100}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 100, MaxRetained: 100})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_node_capacity_test.go b/services/agents-api/internal/store/runtime_node_capacity_test.go index eee23bc66..38b109cc8 100644 --- a/services/agents-api/internal/store/runtime_node_capacity_test.go +++ b/services/agents-api/internal/store/runtime_node_capacity_test.go @@ -11,11 +11,11 @@ import ( func TestRuntimeEnrollmentApprovedCapacity(t *testing.T) { s, _, view, _ := webSpecificationFixture(t, "microsandbox") for _, capacity := range []RuntimeNodeCapacity{{0, 8}, {3, 2}, {1, 1000001}} { - if _, _, err := s.CreateRuntimeEnrollment(t.Context(), capacity); !errors.Is(err, ErrInvalidInput) { + if _, err := s.CreateRuntimeEnrollment(t.Context(), capacity); !errors.Is(err, ErrInvalidInput) { t.Fatal("invalid capacity accepted", err) } } - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{1, 3}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{1, 3})) if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestRuntimeEnrollmentApprovedCapacity(t *testing.T) { if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET maintenance=true"); err != nil { t.Fatal(err) } - if _, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{3, 2}); !errors.Is(err, ErrInvalidInput) { + if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{3, 2}); !errors.Is(err, ErrInvalidInput) { t.Fatal("invalid capacity reported as a conflict", err) } } @@ -72,7 +72,7 @@ func TestDockerRetainedLimitFollowsActive(t *testing.T) { t.Fatal(err) } } - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 3, MaxRetained: 1}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 3, MaxRetained: 1})) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestDockerRetainedLimitFollowsActive(t *testing.T) { if identity, err := s.EnrollRuntimeNode(t.Context(), token, current); err != nil || identity.MaxRetained != 3 { t.Fatal("enrollment kept a separate Docker retained limit", identity, err) } - legacyToken, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8}) + legacyToken, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go b/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go index d30427a2c..475ec5907 100644 --- a/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go +++ b/services/agents-api/internal/store/runtime_node_lifecycle_fixture_test.go @@ -144,7 +144,7 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { } func (f *nodeIsolationFixture) enroll(id string) { f.t.Helper() - token, _, err := f.store.CreateRuntimeEnrollment(f.t.Context(), store.RuntimeNodeCapacity{MaxActive: 100, MaxRetained: 100}) + token, err := store.EnrollmentTestToken(f.store.CreateRuntimeEnrollment(f.t.Context(), store.RuntimeNodeCapacity{MaxActive: 100, MaxRetained: 100})) if err != nil { f.t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_node_presence_test.go b/services/agents-api/internal/store/runtime_node_presence_test.go index 1b93d646e..de00dd178 100644 --- a/services/agents-api/internal/store/runtime_node_presence_test.go +++ b/services/agents-api/internal/store/runtime_node_presence_test.go @@ -22,7 +22,7 @@ func runtimePresenceContext(t *testing.T) context.Context { func runtimePresenceOtherNode(t *testing.T, s *Store) string { t.Helper() - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 1}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 1, MaxRetained: 1})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/runtime_node_types.go b/services/agents-api/internal/store/runtime_node_types.go index c6d6240a1..4026e4634 100644 --- a/services/agents-api/internal/store/runtime_node_types.go +++ b/services/agents-api/internal/store/runtime_node_types.go @@ -11,6 +11,9 @@ var ( ErrRuntimeNodeInUse = errors.New("sandbox node retains resources") ErrRuntimeNodeCredential = errors.New("invalid sandbox node credential") ErrRuntimeLocalNodeConfigured = errors.New("local sandbox node is enabled in deployment configuration") + // ErrRuntimeNodeAddressMismatch rejects an enrollment whose Core address is not + // the installation public URL. The token stays unconsumed. + ErrRuntimeNodeAddressMismatch = errors.New("sandbox node Core address differs from the public URL") ) type RuntimeNodeIdentity struct { @@ -23,6 +26,15 @@ type RuntimeNodeIdentity struct { MaxActive int `json:"max_active"` MaxRetained int `json:"max_retained"` } + +// RuntimeNodeEnrollmentToken is an issued one-use node enrollment token. ID is a +// public, non-secret handle that never authenticates; the node the token registers +// reports it as enrollment_id. +type RuntimeNodeEnrollmentToken struct { + Token string + ExpiresAt time.Time + ID string +} type RuntimeNodeCapacity struct { MaxActive int `json:"max_active"` MaxRetained int `json:"max_retained"` @@ -36,6 +48,8 @@ type RuntimeNodeEnrollment struct { Name string `json:"name"` Provider string `json:"provider"` BackendFingerprint string `json:"backend_fingerprint"` + // The Core origin this node stores and connects to, such as https://core.example. It must equal the installation public URL; otherwise enrollment gets 409 sandbox_node_address_mismatch and the token stays unused. + CoreURL string `json:"core_url"` } type RuntimeNodeHealth struct { Host *RuntimeNodeHost `json:"-"` @@ -65,6 +79,8 @@ type RuntimeNode struct { // The Core address this node enrolled with. A node whose address differs // from the installation public URL receives no new sandboxes; re-add it. CoreURL string `json:"core_url"` + // The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. + EnrollmentID *string `json:"enrollment_id" extensions:"x-nullable"` } type RuntimeNodeUpdate struct { Name string `json:"name"` diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index cb60b1bb0..99549bea2 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -93,6 +93,14 @@ func runtimeUUID(id pgtype.UUID) string { } return uuid.UUID(id.Bytes).String() } + +func optionalUUID(id pgtype.UUID) *string { + if !id.Valid { + return nil + } + value := runtimeUUID(id) + return &value +} func (s *Store) ListRuntimeNodes(ctx context.Context) ([]RuntimeNode, error) { rows, err := s.queries.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { @@ -114,21 +122,24 @@ func runtimeNodeViews(rows []sqlc.ListRuntimeNodesRow) ([]RuntimeNode, error) { return nil, err } health.ProviderReady = n.ProviderReady - out = append(out, RuntimeNode{RuntimeNodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: runtimeUUID(n.ID), Name: n.Name, CoreURL: n.CoreUrl, Provider: n.ProviderKind, Online: n.Online, LastSeenAt: seen, MaxActive: int(n.MaxActive), MaxRetained: retainedLimit(n.ProviderKind, int(n.MaxActive), int(n.MaxRetained)), Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt.Time}) + out = append(out, RuntimeNode{RuntimeNodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: runtimeUUID(n.ID), Name: n.Name, CoreURL: n.CoreUrl, EnrollmentID: optionalUUID(n.EnrollmentID), Provider: n.ProviderKind, Online: n.Online, LastSeenAt: seen, MaxActive: int(n.MaxActive), MaxRetained: retainedLimit(n.ProviderKind, int(n.MaxActive), int(n.MaxRetained)), Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt.Time}) } return out, nil } -func (s *Store) CreateRuntimeEnrollment(ctx context.Context, capacity RuntimeNodeCapacity) (string, time.Time, error) { + +// CreateRuntimeEnrollment issues a one-use enrollment token. Its ID is a public, +// non-secret handle: the node the token registers reports it as enrollment_id. +func (s *Store) CreateRuntimeEnrollment(ctx context.Context, capacity RuntimeNodeCapacity) (RuntimeNodeEnrollmentToken, error) { // The retained limit depends on the provider, so the transaction checks it. if err := validateRuntimeNode("enrollment", capacity.MaxActive, capacity.MaxActive); err != nil { - return "", time.Time{}, err + return RuntimeNodeEnrollmentToken{}, err } var bytes [32]byte if _, err := rand.Read(bytes[:]); err != nil { - return "", time.Time{}, err + return RuntimeNodeEnrollmentToken{}, err } - token := hex.EncodeToString(bytes[:]) - var expires time.Time + id := uuid.New() + result := RuntimeNodeEnrollmentToken{Token: hex.EncodeToString(bytes[:]), ID: id.String()} err := s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { capacity.MaxRetained = retainedLimit(d.ProviderKind, capacity.MaxActive, capacity.MaxRetained) if err := validateRuntimeNode("enrollment", capacity.MaxActive, capacity.MaxRetained); err != nil { @@ -137,14 +148,14 @@ func (s *Store) CreateRuntimeEnrollment(ctx context.Context, capacity RuntimeNod if d.Mode != "nodes" || d.Maintenance || unspecifiedNodeDeployment(d) { return ErrSandboxDeploymentConflict } - if err := q.CreateRuntimeEnrollment(ctx, sqlc.CreateRuntimeEnrollmentParams{TokenSha256: runtimeTokenDigest(token), InstallationID: d.InstallationID, MaxActive: int32(capacity.MaxActive), MaxRetained: int32(capacity.MaxRetained)}); err != nil { + if err := q.CreateRuntimeEnrollment(ctx, sqlc.CreateRuntimeEnrollmentParams{ID: pgtype.UUID{Bytes: id, Valid: true}, TokenSha256: runtimeTokenDigest(result.Token), InstallationID: d.InstallationID, MaxActive: int32(capacity.MaxActive), MaxRetained: int32(capacity.MaxRetained)}); err != nil { return err } - row, err := q.GetRuntimeEnrollment(ctx, runtimeTokenDigest(token)) - expires = row.ExpiresAt.Time + row, err := q.GetRuntimeEnrollment(ctx, runtimeTokenDigest(result.Token)) + result.ExpiresAt = row.ExpiresAt.Time return err }) - return token, expires, err + return result, err } func (s *Store) EnrollRuntimeNode(ctx context.Context, token string, input RuntimeNodeEnrollment) (RuntimeNodeIdentity, error) { id, err := parseConnectionGeneration(input.NodeID) @@ -179,12 +190,18 @@ func (s *Store) EnrollRuntimeNode(ctx context.Context, token string, input Runti if err != nil || input.DeploymentGeneration != uint64(d.Generation) || input.SpecificationDigest != spec.Digest(d.ProviderKind) { return ErrRuntimeSpecificationMismatch } + // The node must use the address Core advertises now. It read that address + // from its configuration, but the public URL may have changed since, or an + // operator may have registered by hand with another origin. + if input.CoreURL != s.publicURL { + return ErrRuntimeNodeAddressMismatch + } if _, err := q.GetRuntimeNode(ctx, id); err == nil { return ErrIdempotencyConflict } else if !errors.Is(err, pgx.ErrNoRows) { return err } - row, err := q.InsertRuntimeNode(ctx, sqlc.InsertRuntimeNodeParams{ID: id, InstallationID: d.InstallationID, Name: input.Name, BackendFingerprint: input.BackendFingerprint, CredentialSha256: runtimeTokenDigest(input.Credential), MaxActive: receipt.MaxActive, MaxRetained: int32(retainedLimit(d.ProviderKind, int(receipt.MaxActive), int(receipt.MaxRetained))), SpecificationDigest: input.SpecificationDigest, DeploymentGeneration: int64(input.DeploymentGeneration), CoreUrl: s.publicURL}) + row, err := q.InsertRuntimeNode(ctx, sqlc.InsertRuntimeNodeParams{ID: id, InstallationID: d.InstallationID, Name: input.Name, BackendFingerprint: input.BackendFingerprint, CredentialSha256: runtimeTokenDigest(input.Credential), MaxActive: receipt.MaxActive, MaxRetained: int32(retainedLimit(d.ProviderKind, int(receipt.MaxActive), int(receipt.MaxRetained))), SpecificationDigest: input.SpecificationDigest, DeploymentGeneration: int64(input.DeploymentGeneration), CoreUrl: input.CoreURL, EnrollmentID: receipt.ID}) if err != nil { return err } diff --git a/services/agents-api/internal/store/runtime_nodes_test.go b/services/agents-api/internal/store/runtime_nodes_test.go index 0212477aa..a844bdb9e 100644 --- a/services/agents-api/internal/store/runtime_nodes_test.go +++ b/services/agents-api/internal/store/runtime_nodes_test.go @@ -150,7 +150,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { } func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { s, w, d := managerFixture(t, 2, 4) - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_test.go index a65d8eafb..b1c8e6a0b 100644 --- a/services/agents-api/internal/store/sandbox_deployment_switch_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_switch_test.go @@ -45,7 +45,7 @@ func TestSandboxDirectDeploymentOwnershipAndCleanSwitch(t *testing.T) { if err != nil || setup.E2B.APIKey != input.E2B.APIKey { t.Fatal("internal credential unavailable", err) } - if _, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8}); !errors.Is(err, ErrSandboxDeploymentConflict) { t.Fatal("cloud enrolled a machine", err) } tenant := uuid.NewString() @@ -116,7 +116,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if _, err := w.InitializeSandboxDeployment(t.Context(), id, SandboxDeploymentSetupRequest{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } @@ -124,7 +124,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if _, err := s.EnrollRuntimeNode(t.Context(), token, node); err != nil { t.Fatal(err) } - unused, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8}) + unused, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 8})) if err != nil { t.Fatal(err) } @@ -318,7 +318,7 @@ func TestUnspecifiedNodeDeploymentDrainsBeforeReplacement(t *testing.T) { if _, err := w.InitializeSandboxDeployment(t.Context(), id, selection); err != nil { t.Fatal(err) } - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } @@ -346,7 +346,7 @@ func TestUnspecifiedNodeDeploymentDrainsBeforeReplacement(t *testing.T) { if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, ErrEnvironmentUnavailable) { t.Fatal("unspecified deployment admitted a fresh sandbox", err) } - if _, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}); !errors.Is(err, ErrSandboxDeploymentConflict) { + if _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}); !errors.Is(err, ErrSandboxDeploymentConflict) { t.Fatal("unspecified deployment issued an enrollment token", err) } diff --git a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go index 15a1b5900..23b0acc15 100644 --- a/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_switch_worker_test.go @@ -64,7 +64,7 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if _, err := w.InitializeSandboxDeployment(t.Context(), store.SandboxDeploymentSetupRequest{DeploymentSpec: store.SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - enrollment, _, err := s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}) + enrollment, err := store.EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/sandbox_deployment_worker_test.go b/services/agents-api/internal/store/sandbox_deployment_worker_test.go index 61c415ddf..32e4a932a 100644 --- a/services/agents-api/internal/store/sandbox_deployment_worker_test.go +++ b/services/agents-api/internal/store/sandbox_deployment_worker_test.go @@ -53,7 +53,7 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if _, err := s.CreateSession(t.Context(), uuid.NewString(), input); !errors.Is(err, store.ErrRuntimeNodeUnavailable) { t.Fatal("zero-node deployment admitted Session", err) } - token, _, err := s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 16}) + token, err := store.EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), store.RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 16})) if err != nil { t.Fatal(err) } diff --git a/services/agents-api/internal/store/sandbox_node_auth_order_http_test.go b/services/agents-api/internal/store/sandbox_node_auth_order_http_test.go index 5a44574e9..f87cfcdcf 100644 --- a/services/agents-api/internal/store/sandbox_node_auth_order_http_test.go +++ b/services/agents-api/internal/store/sandbox_node_auth_order_http_test.go @@ -41,7 +41,7 @@ func TestSandboxNodeRoutesAuthenticateBeforeDeploymentState(t *testing.T) { enrollment(token, uuid.NewString()) enrollment(claimedToken, claimed) enroll, _ := json.Marshal(store.RuntimeNodeEnrollment{NodeID: uuid.NewString(), Credential: strings.Repeat("n", 64), Name: "Early node", Provider: "docker", - BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: strings.Repeat("d", 64)}) + BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: 1, SpecificationDigest: strings.Repeat("d", 64), CoreURL: "https://core.example"}) nodeID := uuid.NewString() type check struct { name, method, path, authorization, nodeHeader, body string diff --git a/services/agents-api/internal/store/sandbox_specification_store_test.go b/services/agents-api/internal/store/sandbox_specification_store_test.go index 7170ffa11..03bb4c481 100644 --- a/services/agents-api/internal/store/sandbox_specification_store_test.go +++ b/services/agents-api/internal/store/sandbox_specification_store_test.go @@ -39,12 +39,12 @@ func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, Run func specificationNode(t *testing.T, s *Store, view RuntimeDeploymentView) RuntimeNodeEnrollment { t.Helper() - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 16}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 4, MaxRetained: 16})) if err != nil { t.Fatal(err) } input := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "specification fixture", Credential: strings.Repeat("n", 64), Provider: view.Provider, - BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest} + BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: s.publicURL} if _, err := s.EnrollRuntimeNode(t.Context(), token, input); err != nil { t.Fatal(err) } @@ -85,7 +85,7 @@ func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *test func TestSandboxSpecificationBootstrapReadDoesNotConsumeEnrollment(t *testing.T) { s, w, view, input := webSpecificationFixture(t, "docker") - token, _, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4}) + token, err := EnrollmentTestToken(s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 4})) if err != nil { t.Fatal(err) } @@ -366,6 +366,53 @@ func TestNodeBoundToAnotherPublicURLGetsNoNewSandboxes(t *testing.T) { } } +// Enrollment records the command's public ID and the node's Core address. A node +// using another address is refused without consuming the token, and a node +// enrolled with a token issued before Core recorded IDs reports none. +func TestEnrollmentRecordsItsIDAndRefusesAnotherAddress(t *testing.T) { + s, _, view, _ := webSpecificationFixture(t, "docker") + s.SetPublicURL("https://core.example") + issued, err := s.CreateRuntimeEnrollment(t.Context(), RuntimeNodeCapacity{MaxActive: 2, MaxRetained: 2}) + if err != nil || uuid.Validate(issued.ID) != nil || issued.Token == "" { + t.Fatal(issued.ID, err) + } + input := RuntimeNodeEnrollment{NodeID: uuid.NewString(), Name: "addressed", Credential: strings.Repeat("a", 64), Provider: "docker", + BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: "https://other.example"} + if _, err := s.EnrollRuntimeNode(t.Context(), issued.Token, input); !errors.Is(err, ErrRuntimeNodeAddressMismatch) { + t.Fatal("enrolled a node that uses another Core address", err) + } + input.CoreURL = "https://core.example" + if _, err := s.EnrollRuntimeNode(t.Context(), issued.Token, input); err != nil { + t.Fatal("the refused enrollment consumed its token", err) + } + earlier := strings.Repeat("e", 64) + if _, err := s.pool.Exec(t.Context(), "INSERT INTO runtime_node_enrollments(token_sha256,installation_id,expires_at) VALUES($1,$2,clock_timestamp()+interval '10 minutes')", + runtimeTokenDigest(earlier), view.InstallationID); err != nil { + t.Fatal(err) + } + older := input + older.NodeID, older.Credential = uuid.NewString(), strings.Repeat("o", 64) + if _, err := s.EnrollRuntimeNode(t.Context(), earlier, older); err != nil { + t.Fatal(err) + } + nodes, err := s.ListRuntimeNodes(t.Context()) + if err != nil || len(nodes) != 2 { + t.Fatal(nodes, err) + } + for _, node := range nodes { + switch node.ID { + case input.NodeID: + if node.EnrollmentID == nil || *node.EnrollmentID != issued.ID || node.CoreURL != "https://core.example" { + t.Fatal("the node did not record its enrollment", node.EnrollmentID, node.CoreURL) + } + case older.NodeID: + if node.EnrollmentID != nil { + t.Fatal("a token without an ID reported one", *node.EnrollmentID) + } + } + } +} + // An E2B selection saved before the public URL became loopback admits no new // Session, and its configuration stays readable for cleanup. func TestE2BAdmitsNothingWhileThePublicURLIsLoopback(t *testing.T) { diff --git a/services/agents-api/internal/store/sandbox_specification_test.go b/services/agents-api/internal/store/sandbox_specification_test.go index 98826e596..d01f21606 100644 --- a/services/agents-api/internal/store/sandbox_specification_test.go +++ b/services/agents-api/internal/store/sandbox_specification_test.go @@ -17,3 +17,8 @@ func SandboxDeploymentTestSpec(provider string) sandbox.DeploymentSpec { } return s } + +// EnrollmentTestToken keeps only the secret token of an issued node enrollment. +func EnrollmentTestToken(issued RuntimeNodeEnrollmentToken, err error) (string, error) { + return issued.Token, err +} diff --git a/services/agents-api/migrations/000077_node_enrollment_id.sql b/services/agents-api/migrations/000077_node_enrollment_id.sql new file mode 100644 index 000000000..2caaf1b15 --- /dev/null +++ b/services/agents-api/migrations/000077_node_enrollment_id.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- A public, non-secret handle for each node enrollment command, so the console can +-- tell which node a command registered. It never authenticates; the token does. +-- Tokens issued before this migration have none. +ALTER TABLE runtime_node_enrollments ADD COLUMN id uuid UNIQUE; +-- The enrollment that registered the node, copied at registration. There is no +-- foreign key: runtime_node_enrollments.node_id already references the node, and a +-- reverse key would only add a cycle. Nodes enrolled earlier keep null. +ALTER TABLE runtime_nodes ADD COLUMN enrollment_id uuid; + +-- +goose Down +ALTER TABLE runtime_nodes DROP COLUMN enrollment_id; +ALTER TABLE runtime_node_enrollments DROP COLUMN id; diff --git a/services/core-console/distribution_test.go b/services/core-console/distribution_test.go index 5a4d9753e..0abbc3709 100644 --- a/services/core-console/distribution_test.go +++ b/services/core-console/distribution_test.go @@ -50,3 +50,61 @@ func TestOfflineArtifactsAreManifestAllowlisted(t *testing.T) { } } } + +// /console/config names the providers whose node files this console holds, so +// Web offers Add node only when a node can download everything it needs. Node +// downloads can resume with HTTP Range. +func TestConsoleReportsServableNodeProviders(t *testing.T) { + dist, payload := t.TempDir(), t.TempDir() + write := func(path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(dist, "index.html"), "console") + write(filepath.Join(payload, "node-install.pyz"), "bootstrap") + write(filepath.Join(payload, "self-hosted-install.pyz"), "executor bootstrap") + artifacts := map[string]any{} + for logical := range map[string]bool{"native/bin/parsar-sandbox-node": true, "images/runtime.tar.gz": true, "native/microsandbox/msb": true} { + name := strings.ReplaceAll(logical, "/", "-") + artifacts[logical] = map[string]any{"filename": name, "size": len("runtime-bytes")} + write(filepath.Join(payload, "artifacts", name), "runtime-bytes") + } + raw, _ := json.Marshal(map[string]any{"artifacts": artifacts}) + write(filepath.Join(payload, "manifest.json"), string(raw)) + upstream, _ := url.Parse("http://127.0.0.1:1") + h, err := newConsole(config{origin: testOrigin, upstream: upstream, dist: dist, coreKey: testCoreKey, nodePayloadDir: payload}) + if err != nil { + t.Fatal(err) + } + defer h.Close() + server := serveSignedIn(t, h) + if _, body := responseBody(t, server, consoleRequest(t, server, "GET", "/console/config")); !strings.Contains(body, `"node_artifacts":["docker"]`) { + t.Fatal("microsandbox reported without its helper and firmware:", body) + } + request := consoleRequest(t, server, "GET", "/node-install/artifacts/images-runtime.tar.gz") + request.Header.Set("Range", "bytes=8-") + if response, body := responseBody(t, server, request); response.StatusCode != 206 || body != "bytes" { + t.Fatal("artifact download cannot resume", response.StatusCode, body) + } + if err := os.RemoveAll(filepath.Join(payload, "artifacts")); err != nil { + t.Fatal(err) + } + if _, body := responseBody(t, server, consoleRequest(t, server, "GET", "/console/config")); !strings.Contains(body, `"node_artifacts":[]`) { + t.Fatal("a console without node files offered them:", body) + } + // A console without any node payload also reports an empty list, never null. + bare, err := newConsole(config{origin: testOrigin, upstream: upstream, dist: dist, coreKey: testCoreKey}) + if err != nil { + t.Fatal(err) + } + defer bare.Close() + bareServer := serveSignedIn(t, bare) + if _, body := responseBody(t, bareServer, consoleRequest(t, bareServer, "GET", "/console/config")); !strings.Contains(body, `"node_artifacts":[]`) { + t.Fatal("a console without a node payload did not report an empty list:", body) + } +} diff --git a/services/core-console/node_installation.go b/services/core-console/node_installation.go index 25766bd66..b27325d58 100644 --- a/services/core-console/node_installation.go +++ b/services/core-console/node_installation.go @@ -99,12 +99,61 @@ func (h *console) serveNodePayload(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, info.Name(), info.ModTime(), f) } +// providerArtifacts lists the artifacts a node of each provider downloads from +// this console, besides the fixed payload files. +var providerArtifacts = map[string][]string{ + "docker": {"native/bin/parsar-sandbox-node", "images/runtime.tar.gz"}, + "microsandbox": {"native/bin/parsar-sandbox-node", "images/runtime.tar.gz", "native/bin/agents-api-microsandbox-provider", + "native/microsandbox/msb", "native/microsandbox/libkrunfw.so.5.6.1"}, +} + +// nodeArtifacts reports the providers whose node artifacts this console can serve: +// each is declared in manifest.json and present at its declared size. A thin +// distribution has none. Nodes verify every checksum themselves. It is read per +// request, so artifacts added by rerunning the installer show without a restart. +func (h *console) nodeArtifacts() []string { + available := []string{} + if h.nodePayload == nil { + return available + } + f, err := h.nodePayload.Open("manifest.json") + if err != nil { + return available + } + defer f.Close() + var manifest struct { + Artifacts map[string]struct { + Filename string `json:"filename"` + Size int64 `json:"size"` + } `json:"artifacts"` + } + if json.NewDecoder(io.LimitReader(f, 1024*1024)).Decode(&manifest) != nil { + return available + } + for _, provider := range []string{"docker", "microsandbox"} { + complete := true + for _, logical := range providerArtifacts[provider] { + entry, ok := manifest.Artifacts[logical] + info, err := h.nodePayload.Stat("artifacts/" + entry.Filename) + if !ok || entry.Filename == "" || strings.Contains(entry.Filename, "/") || err != nil || !info.Mode().IsRegular() || info.Size() != entry.Size { + complete = false + break + } + } + if complete { + available = append(available, provider) + } + } + return available +} + func (h *console) serveConsoleConfiguration(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(struct { - NodeInstaller bool `json:"node_installer"` - NodeInstallerSHA256 string `json:"node_installer_sha256"` - SelfHostedInstaller bool `json:"self_hosted_installer"` - SelfHostedInstallerSHA256 string `json:"self_hosted_installer_sha256"` - }{h.nodePayload != nil, h.nodeInstallerDigest, h.nodePayload != nil, h.selfHostedInstallerDigest}) + NodeInstaller bool `json:"node_installer"` + NodeInstallerSHA256 string `json:"node_installer_sha256"` + NodeArtifacts []string `json:"node_artifacts"` + SelfHostedInstaller bool `json:"self_hosted_installer"` + SelfHostedInstallerSHA256 string `json:"self_hosted_installer_sha256"` + }{h.nodePayload != nil, h.nodeInstallerDigest, h.nodeArtifacts(), h.nodePayload != nil, h.selfHostedInstallerDigest}) }