diff --git a/.github/workflows/cmo-ci.yml b/.github/workflows/cmo-ci.yml index d251a5f9ad7..d7de7888333 100644 --- a/.github/workflows/cmo-ci.yml +++ b/.github/workflows/cmo-ci.yml @@ -360,3 +360,37 @@ jobs: test "$masters" -eq 4 kubectl exec cmo-cmo-data-0 -- valkey-cli -h 127.0.0.1 -c --user admin -a "$pass" --no-auth-warning \ GET cmo:kind | grep -qx before + + site: + name: Site, integration, and browser + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Unit tests for the site and the management API + run: | + python3 tools/mgmt/test_cmo_mgmt.py + python3 tools/mgmt/test_cmo_browse.py + python3 tools/flowview/test_flow_feed.py + python3 site/test_site.py + python3 integration/test_integration.py + + - name: Install Playwright + working-directory: site/e2e + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Browser tests + working-directory: site/e2e + run: npx playwright test + + - name: Upload screenshots + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: site-screenshots + path: site/e2e/screenshots/*.png + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 058adfa5bde..20830587905 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ deploy/.env deploy/.generated/ deploy/results/ dist/ +site/e2e/node_modules/ +site/e2e/test-results/ +site/e2e/playwright-report/ +site/e2e/screenshots/ diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 00000000000..ba911527767 --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,46 @@ +# Adopt Cache Me Outside + +The integration directory is the whole package: a Python client, a browser/Node client, a loopback sidecar, and an updater. Version `1.0.0` is recorded in `integration/manifest.json`. Copy that directory into a GoDesk app, or install a `cmo-integration-vX.Y.Z` release. No token or password belongs in the repository. + +## Python + +```python +import os +from cache_me_outside import Client + +client = Client(os.environ["CMO_API"], os.environ["CMO_ADMIN_TOKEN"]) +print(client.list_databases()) +``` + +## Browser or Node + +```html + + +``` + +`sessionToken` comes from your own sign-in. The script does not contain one. + +## Sidecar + +```bash +CMO_API=http://127.0.0.1:8088 CMO_ADMIN_TOKEN="$CMO_ADMIN_TOKEN" \ + python3 integration/sidecar.py +``` + +The app then calls `GET http://127.0.0.1:8099/health`. The sidecar binds to loopback only. Cache get and set also need `CMO_URL` and `CMO_PREFIX`; a key outside that prefix is refused before a connection opens. + +## Updates + +```bash +python3 integration/updater.py --check +python3 integration/updater.py --apply +``` + +`--apply` reads GitHub releases named `cmo-integration-vX.Y.Z`, downloads `cache-me-outside-integration.tar.gz` and `SHA256SUMS`, and writes files only when the checksum matches. It refuses other hosts, symlinks, and paths that leave the integration directory. When no release is published yet, `--check` reports that and leaves the files alone. diff --git a/docs/cache-me-outside/README.md b/docs/cache-me-outside/README.md index 7855bb3e83b..953195f7f03 100644 --- a/docs/cache-me-outside/README.md +++ b/docs/cache-me-outside/README.md @@ -22,6 +22,8 @@ Valkey and the Redis code it inherited stay under the BSD 3-Clause License. See - [Warm restart](warm-restart.md) - [DRAM + flash hybrid storage](hybrid-storage.md) - [TTL-grouped storage](ttl-segments.md) +- [Management API and site](management-api.md) +- [Adopting the cache from another GoDesk app](../../INTEGRATION.md) ## What this fork changes diff --git a/docs/cache-me-outside/management-api.md b/docs/cache-me-outside/management-api.md index 84310b1234e..1ddde5d0fcf 100644 --- a/docs/cache-me-outside/management-api.md +++ b/docs/cache-me-outside/management-api.md @@ -19,8 +19,15 @@ Not done yet: - **Terraform provider:** the plan also lists one. It needs a separate Go module that calls this API; that is a follow-up. - **Live test:** the API has not been run against a live server. The tests use a fake ACL backend. -## Web console (plan item 61) +`GET /v1/databases/{name}/keys` is the read-only browser. It scans and reads only keys under `:` and refuses a key outside that prefix. A limit of 0 means unlimited. -Open `/` on the management API to use the console. Paste the admin token, give the new database a name, and click **Create database**. The page then shows the connection URL, a `valkey-cli` command, and the key prefix the database must use. The password is shown once. Each database also has a **New password** button. +## Site -The page keeps the token in memory only; nothing is saved in the browser. It sends a strict CSP header and refuses to be framed. It inserts text with textContent only, never innerHTML. +`python3 site/server.py --demo` serves the site on `127.0.0.1` with an in-memory database, a sample key, and the agent. It will not bind a public address. The real process is the same pages on `cmo_mgmt.py` or `site/server.py --config`, with `CMO_ADMIN_TOKEN` supplied in the environment. + +- `/` is the home page. +- `/console` creates, limits, rotates, browses, and deletes databases. The token stays in memory. The password is shown once in a field, not in a link. +- `/flow` is the live data-flow picture. Demo rates are labeled as demo traffic. +- `/integrate` shows the adoption snippets from [INTEGRATION.md](../../INTEGRATION.md). + +The agent runs list, create, delete, rotate, limit, browse, explain, and open-page requests without a model. Free-form text uses `OLLAMA_BASE_URL` only. Pages are scripts and stylesheets, so the CSP does not use `unsafe-inline`. The pages refuse to be framed and insert text with textContent only. diff --git a/integration/cache-me-outside.js b/integration/cache-me-outside.js new file mode 100644 index 00000000000..1abc45acb85 --- /dev/null +++ b/integration/cache-me-outside.js @@ -0,0 +1,43 @@ +/* Cache Me Outside integration, version 1.0.0. No secrets live in this file. */ +(function (root, factory) { + var api = factory(); + if (typeof module === "object" && module.exports) module.exports = api; + else root.CacheMeOutside = api; +})(typeof globalThis !== "undefined" ? globalThis : this, function () { + var VERSION = "1.0.0"; + + function attach(opts) { + if (!opts || !opts.api || !opts.token) throw new Error("api and token are required"); + var api = String(opts.api).replace(/\/$/, ""); + if (!/^https?:\/\//.test(api)) throw new Error("api must be http or https"); + var token = String(opts.token); + + function call(method, path, body) { + return fetch(api + path, { + method: method, + headers: {"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + body: body ? JSON.stringify(body) : undefined + }).then(function (response) { + return response.json().then(function (payload) { + if (!response.ok) throw new Error(payload.error || String(response.status)); + return payload; + }); + }); + } + + return { + version: VERSION, + listDatabases: function () { return call("GET", "/v1/databases").then(function (b) { return b.databases; }); }, + createDatabase: function (name, limits) { + return call("POST", "/v1/databases", Object.assign({name: name}, limits || {})); + }, + rotatePassword: function (name) { return call("POST", "/v1/databases/" + name + "/rotate-password", {}); }, + deleteDatabase: function (name) { return call("DELETE", "/v1/databases/" + name); }, + scanKeys: function (name, match) { + return call("GET", "/v1/databases/" + encodeURIComponent(name) + "/keys?match=" + encodeURIComponent(match || "*")); + } + }; + } + + return {attach: attach, version: VERSION}; +}); diff --git a/integration/cache_me_outside.py b/integration/cache_me_outside.py new file mode 100644 index 00000000000..6a15aa4457e --- /dev/null +++ b/integration/cache_me_outside.py @@ -0,0 +1,62 @@ +"""Small client for the Cache Me Outside management API. Standard library only.""" +import json +import os +import urllib.error +import urllib.request + +_MANIFEST = os.path.join(os.path.dirname(os.path.abspath(__file__)), "manifest.json") + + +def version(): + with open(_MANIFEST) as handle: + return json.load(handle)["version"] + + +class Client: + def __init__(self, api, token, timeout=15): + if not api or not str(api).startswith(("http://", "https://")): + raise ValueError("api must be an http or https URL") + if not token or not isinstance(token, str): + raise ValueError("token is required") + self.api = api.rstrip("/") + self.token = token + self.timeout = timeout + + def _call(self, method, path, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + self.api + path, data=data, method=method, + headers={"Authorization": "Bearer " + self.token, "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + payload = json.load(exc) + raise RuntimeError(payload.get("error") or str(exc.code)) from None + + def list_databases(self): + return self._call("GET", "/v1/databases")["databases"] + + def create_database(self, name, **limits): + body = {"name": name} + body.update(limits) + return self._call("POST", "/v1/databases", body) + + def get_database(self, name): + return self._call("GET", "/v1/databases/" + name) + + def update_limits(self, name, **limits): + return self._call("PATCH", "/v1/databases/" + name, limits) + + def rotate_password(self, name): + return self._call("POST", "/v1/databases/%s/rotate-password" % name, {}) + + def delete_database(self, name): + return self._call("DELETE", "/v1/databases/" + name) + + def scan_keys(self, name, match="*", cursor="0"): + return self._call("GET", "/v1/databases/%s/keys?cursor=%s&match=%s" % (name, cursor, match)) + + def show_key(self, name, key): + from urllib.parse import quote + return self._call("GET", "/v1/databases/%s/keys?key=%s" % (name, quote(key, safe=""))) diff --git a/integration/manifest.json b/integration/manifest.json new file mode 100644 index 00000000000..6dad9193a27 --- /dev/null +++ b/integration/manifest.json @@ -0,0 +1,8 @@ +{ + "name": "cache-me-outside", + "version": "1.0.0", + "api": "v1", + "repository": "https://github.com/GoDeskio/Cache-Me-Outside", + "release_prefix": "cmo-integration-v", + "asset": "cache-me-outside-integration.tar.gz" +} diff --git a/integration/sidecar.py b/integration/sidecar.py new file mode 100644 index 00000000000..15e0e9b0a96 --- /dev/null +++ b/integration/sidecar.py @@ -0,0 +1,170 @@ +"""Loopback sidecar so another GoDesk app can call Cache Me Outside without linking a library. + + CMO_API=http://127.0.0.1:8088 CMO_ADMIN_TOKEN=... python3 integration/sidecar.py + +Cache reads and writes need CMO_URL and CMO_PREFIX as well. Keys outside the +prefix are refused before a connection is opened. The process will not bind +a wildcard address. +""" +import json +import os +import socket +import sys + +import cache_me_outside + +MAX_VALUE = 65536 +COMMANDS = {"GET", "SET", "DEL"} + + +def check_key(key, prefix): + if not isinstance(key, str) or not prefix or not key.startswith(prefix): + raise ValueError("key is outside the prefix") + if len(key) > 512 or "\r" in key or "\n" in key or "\x00" in key: + raise ValueError("key refused") + return key + + +def _send(sock, args): + enc = [a if isinstance(a, bytes) else str(a).encode() for a in args] + sock.sendall(b"*%d\r\n" % len(enc) + b"".join(b"$%d\r\n%s\r\n" % (len(x), x) for x in enc)) + + +def _read(stream): + line = stream.readline() + if not line: + raise ConnectionError("closed") + if line[:1] == b"-": + raise RuntimeError(line[1:-2].decode(errors="replace")) + if line[:1] == b"$": + n = int(line[1:]) + if n < 0: + return None + return stream.read(n + 2)[:-2] + if line[:1] == b"+": + return line[1:-2] + if line[:1] == b":": + return int(line[1:]) + raise RuntimeError("unexpected reply") + + +def cache_call(url, prefix, args): + from urllib.parse import urlsplit + if args[0] not in COMMANDS: + raise ValueError("command refused") + check_key(args[1], prefix) + if args[0] == "SET" and len(args) > 2 and len(str(args[2]).encode()) > MAX_VALUE: + raise ValueError("value is too large") + parts = urlsplit(url) + if parts.scheme not in ("redis", "rediss", "valkey", "valkeys") or not parts.hostname: + raise ValueError("CMO_URL must be a redis or valkey URL") + if parts.scheme in ("rediss", "valkeys"): + raise ValueError("the sidecar speaks plain TCP; terminate TLS in front of the cache") + sock = socket.create_connection((parts.hostname, parts.port or 6379), 5) + stream = sock.makefile("rb") + try: + if parts.password is not None: + user = parts.username or "default" + _send(sock, ["AUTH", user, parts.password]) + _read(stream) + _send(sock, args) + return _read(stream) + finally: + stream.close() + sock.close() + + +def make_handler(client, cache_url, prefix): + from http.server import BaseHTTPRequestHandler + + class H(BaseHTTPRequestHandler): + server_version = "cmo-sidecar" + + def log_message(self, *a): + pass + + def _send(self, status, obj): + body = json.dumps(obj).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _body(self): + n = int(self.headers.get("Content-Length") or 0) + if n > MAX_VALUE + 1024: + raise ValueError("body too large") + data = json.loads(self.rfile.read(n) or b"{}") + if not isinstance(data, dict): + raise ValueError("body must be an object") + return data + + def do_GET(self): + if self.path == "/health": + return self._send(200, {"ok": True, "version": cache_me_outside.version()}) + if self.path == "/v1/version": + return self._send(200, {"name": "cache-me-outside", "version": cache_me_outside.version()}) + if self.path.split("?")[0].startswith("/v1/databases"): + if client is None: + return self._send(503, {"error": "set CMO_API and CMO_ADMIN_TOKEN"}) + try: + self._send(200, client._call("GET", self.path)) + except RuntimeError as exc: + self._send(502, {"error": str(exc)}) + return + self._send(404, {"error": "not found"}) + + def do_POST(self): + path = self.path.split("?")[0] + try: + body = self._body() + if path == "/v1/cache/get": + if not cache_url: + return self._send(503, {"error": "set CMO_URL and CMO_PREFIX"}) + value = cache_call(cache_url, prefix, ["GET", body.get("key", "")]) + shown = None if value is None else value.decode("utf-8", "replace") + return self._send(200, {"key": body.get("key"), "value": shown}) + if path == "/v1/cache/set": + if not cache_url: + return self._send(503, {"error": "set CMO_URL and CMO_PREFIX"}) + args = ["SET", body.get("key", ""), body.get("value", "")] + if "ttl" in body: + ttl = body["ttl"] + if isinstance(ttl, bool) or not isinstance(ttl, int) or ttl < 1 or ttl > 8640000: + raise ValueError("ttl must be 1 to 8640000 seconds") + args.extend(["EX", ttl]) + cache_call(cache_url, prefix, args) + return self._send(200, {"stored": True}) + if client and path == "/v1/databases": + return self._send(201, client.create_database(body.get("name", ""), **{ + k: body[k] for k in ("max_conns", "ops_per_sec", "bytes_per_sec") if k in body})) + except ValueError as exc: + return self._send(400, {"error": str(exc)}) + except (OSError, RuntimeError) as exc: + return self._send(502, {"error": str(exc)}) + self._send(404, {"error": "not found"}) + + return H + + +def main(argv=None): + import argparse + from http.server import ThreadingHTTPServer + ap = argparse.ArgumentParser(description="Cache Me Outside sidecar") + ap.add_argument("--listen", default="127.0.0.1:8099") + a = ap.parse_args(argv) + host, _, port = a.listen.rpartition(":") + if host not in ("127.0.0.1", "::1"): + ap.error("the sidecar listens on 127.0.0.1 only") + api = os.environ.get("CMO_API", "") + token = os.environ.get("CMO_ADMIN_TOKEN", "") + client = cache_me_outside.Client(api, token) if api and token else None + srv = ThreadingHTTPServer((host, int(port)), make_handler(client, os.environ.get("CMO_URL", ""), os.environ.get("CMO_PREFIX", ""))) + print("sidecar on http://%s" % a.listen, file=sys.stderr) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/integration/test_integration.py b/integration/test_integration.py new file mode 100644 index 00000000000..e299a8396b6 --- /dev/null +++ b/integration/test_integration.py @@ -0,0 +1,181 @@ +import hashlib +import io +import json +import os +import socket +import subprocess +import sys +import tarfile +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import cache_me_outside +import sidecar +import updater + +ASSET = updater.ASSET + + +def archive(files): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as handle: + for name, data in files.items(): + if isinstance(data, tuple): + info, payload = data + else: + info = tarfile.TarInfo(name) + info.size = len(data) + payload = data + handle.addfile(info, io.BytesIO(payload)) + return buf.getvalue() + + +class Release(BaseHTTPRequestHandler): + blob = b"" + sums = b"" + releases = [] + + def do_GET(self): + if self.path.startswith("/releases"): + body = json.dumps(self.releases).encode() + elif self.path == "/blob": + body = type(self).blob + elif self.path == "/sums": + body = type(self).sums + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + +def serve(handler): + httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd + + +class Updater(unittest.TestCase): + def setUp(self): + self.dest = tempfile.mkdtemp() + with open(os.path.join(self.dest, "manifest.json"), "w") as handle: + json.dump({"version": "1.0.0"}, handle) + manifest = json.dumps({"name": "cache-me-outside", "version": "1.1.0", "api": "v1"}).encode() + self.blob = archive({"pkg/manifest.json": manifest, "pkg/cache_me_outside.py": b"print(1)\n"}) + Release.blob = self.blob + Release.sums = (hashlib.sha256(self.blob).hexdigest() + " " + ASSET + "\n").encode() + Release.releases = [{ + "tag_name": "cmo-integration-v1.1.0", + "assets": [ + {"name": ASSET, "browser_download_url": ""}, + {"name": "SHA256SUMS", "browser_download_url": ""}, + ], + }] + self.httpd = serve(Release) + base = "http://127.0.0.1:%d" % self.httpd.server_address[1] + Release.releases[0]["assets"][0]["browser_download_url"] = base + "/blob" + Release.releases[0]["assets"][1]["browser_download_url"] = base + "/sums" + self.api = base + "/releases" + + def tearDown(self): + self.httpd.shutdown() + self.httpd.server_close() + + def test_apply_checks_the_hash_and_rejects_traversal(self): + result = updater.apply(self.api, self.dest) + self.assertEqual(result["status"], "applied") + self.assertEqual(updater.local_version(self.dest), "1.1.0") + self.assertTrue(os.path.isfile(os.path.join(self.dest, "cache_me_outside.py"))) + Release.sums = (b"0" * 64 + b" " + ASSET.encode() + b"\n") + with open(os.path.join(self.dest, "manifest.json"), "w") as handle: + json.dump({"version": "1.0.0"}, handle) + with self.assertRaises(ValueError): + updater.apply(self.api, self.dest) + self.assertEqual(updater.local_version(self.dest), "1.0.0") + bad = archive({"../manifest.json": b"{}"}) + with self.assertRaises(ValueError): + updater.unpack(bad) + link = tarfile.TarInfo("manifest.json") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + with self.assertRaises(ValueError): + updater.unpack(archive({"manifest.json": (link, b"")})) + + def test_no_release_is_not_a_failure(self): + Release.releases = [] + found = updater.check(self.api, self.dest) + self.assertEqual(found["status"], "none") + self.assertEqual(found["version"], "1.0.0") + + def test_file_url_is_refused(self): + found = updater.check("file:///tmp/releases", self.dest) + self.assertEqual(found["status"], "unreachable") + + +class ClientAndSidecar(unittest.TestCase): + def test_package_version_and_node_client(self): + self.assertEqual(cache_me_outside.version(), "1.0.0") + with self.assertRaises(ValueError): + cache_me_outside.Client("http://127.0.0.1:9", "") + script = ( + "const c=require('./cache-me-outside.js');" + "if(c.version!=='1.0.0') process.exit(1);" + "try{c.attach({});process.exit(1)}catch(e){}" + ) + subprocess.check_call(["node", "-e", script], cwd=HERE) + + def test_prefix_is_enforced_before_connect(self): + with self.assertRaises(ValueError): + sidecar.cache_call("redis://127.0.0.1:1", "app:", ["GET", "other:x"]) + with self.assertRaises(ValueError): + sidecar.cache_call("redis://127.0.0.1:1", "app:", ["FLUSHALL", "app:x"]) + with self.assertRaises(ValueError): + sidecar.cache_call("rediss://127.0.0.1:1", "app:", ["GET", "app:x"]) + + def test_get_roundtrip_and_health(self): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + + def once(): + conn, _addr = sock.accept() + conn.recv(1024) + conn.sendall(b"$5\r\nhello\r\n") + conn.close() + + threading.Thread(target=once, daemon=True).start() + port = sock.getsockname()[1] + self.assertEqual(sidecar.cache_call("redis://127.0.0.1:%d" % port, "app:", ["GET", "app:k"]), b"hello") + sock.close() + httpd = serve(sidecar.make_handler(None, "", "")) + try: + with urllib.request.urlopen("http://127.0.0.1:%d/health" % httpd.server_address[1]) as response: + self.assertEqual(json.load(response)["version"], "1.0.0") + req = urllib.request.Request( + "http://127.0.0.1:%d/v1/cache/get" % httpd.server_address[1], + data=b'{"key":"app:k"}', headers={"Content-Type": "application/json"}) + with self.assertRaises(urllib.error.HTTPError) as caught: + urllib.request.urlopen(req) + self.assertEqual(caught.exception.code, 503) + finally: + httpd.shutdown() + httpd.server_close() + + def test_sidecar_refuses_a_public_bind(self): + with self.assertRaises(SystemExit): + sidecar.main(["--listen", "0.0.0.0:8099"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/integration/updater.py b/integration/updater.py new file mode 100644 index 00000000000..095df950682 --- /dev/null +++ b/integration/updater.py @@ -0,0 +1,188 @@ +"""Update the integration directory from a published GitHub release. + + python3 integration/updater.py --check + python3 integration/updater.py --apply + +A release tag looks like cmo-integration-v1.2.3 and carries +cache-me-outside-integration.tar.gz plus SHA256SUMS. The archive is applied +only after the checksum matches, and only the known filenames are written. +Nothing in this file is a secret, and a missing release is not an error. +""" +import hashlib +import io +import json +import os +import tarfile +import urllib.request +from urllib.parse import urlsplit + +HERE = os.path.dirname(os.path.abspath(__file__)) +ASSET = "cache-me-outside-integration.tar.gz" +ALLOW = {"manifest.json", "cache_me_outside.py", "cache-me-outside.js", "sidecar.py", "updater.py"} +HOSTS = ("api.github.com", "github.com", "objects.githubusercontent.com", "release-assets.githubusercontent.com") +DEFAULT_API = "https://api.github.com/repos/GoDeskio/Cache-Me-Outside/releases?per_page=20" + + +def local_version(path=None): + with open(os.path.join(path or HERE, "manifest.json")) as handle: + return json.load(handle)["version"] + + +def semver(text): + parts = text.split(".") + if len(parts) != 3 or any(not part.isdigit() for part in parts): + raise ValueError("bad version %r" % text) + return tuple(int(part) for part in parts) + + +def allow_for(api): + base = urlsplit(api) + + def allow(url): + got = urlsplit(url) + if base.hostname in ("127.0.0.1", "localhost"): + return got.scheme == base.scheme and got.netloc == base.netloc + return got.scheme == "https" and got.hostname in HOSTS + return allow + + +class _Guard(urllib.request.HTTPRedirectHandler): + def __init__(self, allow): + self.allow = allow + + def redirect_request(self, req, fp, code, msg, headers, newurl): + if not self.allow(newurl): + raise ValueError("redirect refused") + return urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + + +def _get(url, allow, limit=2000000): + if not allow(url): + raise ValueError("refusing " + url) + opener = urllib.request.build_opener(_Guard(allow)) + with opener.open(url, timeout=20) as response: + blob = response.read(limit + 1) + if len(blob) > limit: + raise ValueError("download is too large") + return blob + + +def choose(releases, prefix="cmo-integration-v"): + best = None + best_key = None + for release in releases: + tag = release.get("tag_name") or "" + if not tag.startswith(prefix): + continue + key = semver(tag[len(prefix):]) + if best_key is None or key > best_key: + best, best_key = release, key + return best + + +def checksum(text, filename): + for line in text.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[-1].endswith(filename) and len(parts[0]) == 64: + return parts[0] + raise ValueError("checksum missing") + + +def unpack(blob): + archive = tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") + chosen = {} + for member in archive.getmembers(): + if member.isdir(): + continue + parts = member.name.split("/") + if member.issym() or member.islnk() or not member.isfile() or any(part in ("", "..") for part in parts): + raise ValueError("refusing %s" % member.name) + base = parts[-1] + if base not in ALLOW or base in chosen: + raise ValueError("unexpected file %s" % member.name) + data = archive.extractfile(member).read() + if len(data) > 200000: + raise ValueError("file too large") + chosen[base] = data + if "manifest.json" not in chosen: + raise ValueError("manifest missing") + json.loads(chosen["manifest.json"].decode()) + return chosen + + +def apply_files(files, dest): + os.makedirs(dest, exist_ok=True) + for name, data in files.items(): + target = os.path.join(dest, name) + if os.path.dirname(os.path.abspath(target)) != os.path.abspath(dest): + raise ValueError("refusing to write outside the integration directory") + tmp = target + ".tmp" + with open(tmp, "wb") as handle: + handle.write(data) + os.replace(tmp, target) + + +def check(api=DEFAULT_API, dest=None): + dest = dest or HERE + current = semver(local_version(dest)) + allow = allow_for(api) + try: + releases = json.loads(_get(api, allow).decode()) + except Exception as exc: + return {"status": "unreachable", "version": "%d.%d.%d" % current, "detail": str(exc)} + if not isinstance(releases, list): + return {"status": "unreachable", "version": "%d.%d.%d" % current, "detail": "unexpected release list"} + release = choose(releases) + if not release: + return {"status": "none", "version": "%d.%d.%d" % current} + remote = semver(release["tag_name"][len("cmo-integration-v"):]) + return { + "status": "update" if remote > current else "current", + "version": "%d.%d.%d" % current, + "remote": "%d.%d.%d" % remote, + "release": release, + } + + +def apply(api=DEFAULT_API, dest=None): + dest = dest or HERE + found = check(api, dest) + if found["status"] != "update": + return found + allow = allow_for(api) + assets = {item.get("name"): item.get("browser_download_url") for item in found["release"].get("assets") or []} + if ASSET not in assets or "SHA256SUMS" not in assets: + raise ValueError("release is missing the archive or SHA256SUMS") + blob = _get(assets[ASSET], allow) + digest = hashlib.sha256(blob).hexdigest() + if digest != checksum(_get(assets["SHA256SUMS"], allow).decode(), ASSET): + raise ValueError("checksum did not match; nothing was written") + apply_files(unpack(blob), dest) + found["status"] = "applied" + found["version"] = local_version(dest) + return found + + +def main(argv=None): + import argparse + import sys + ap = argparse.ArgumentParser(description="Cache Me Outside integration updater") + ap.add_argument("--check", action="store_true") + ap.add_argument("--apply", action="store_true") + ap.add_argument("--api", default=os.environ.get("CMO_RELEASE_API", DEFAULT_API)) + ap.add_argument("--dest", default=HERE) + a = ap.parse_args(argv) + if a.apply == a.check: + ap.error("choose --check or --apply") + try: + result = apply(a.api, a.dest) if a.apply else check(a.api, a.dest) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + print(result["status"] + " " + result.get("remote", result["version"])) + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/site/agent.py b/site/agent.py new file mode 100644 index 00000000000..ced8a9a1a13 --- /dev/null +++ b/site/agent.py @@ -0,0 +1,198 @@ +"""Deterministic site agent. OLLAMA_BASE_URL is optional and never required. + +The planner only recognizes a fixed sentence list. Anything else is sent to a +local Ollama server when that URL is set. The model may only ask for the same +actions the planner can already run. Unknown actions are dropped. +""" +import json +import os +import re +import urllib.request +from urllib.parse import urlsplit + +NAME = re.compile(r"^[a-z][a-z0-9-]{1,30}$") +NAV = {"home": "/", "console": "/console", "flow": "/flow", "integrate": "/integrate"} +TOPICS = { + "tenants": "Each database gets its own port, its own password, and keys that must start with its name.", + "browser": "The key browser is read-only. It only returns keys under that database's prefix.", + "flow": "The flow page shows requests, near-cache hits, cache hits, misses, and the database queries those misses become.", + "warm": "Warm restart can reload a checkpoint after a clean shutdown when RDB and AOF did not. It stays off until it is configured.", + "hybrid": "Hybrid storage can spill large strings to a flash log. It stays off until it is configured.", + "ttl": "TTL segments delete keys that expire in the same time bucket. They stay off until configured.", + "integration": "Other GoDesk apps adopt the integration directory. The token comes from the environment, never from the repository.", +} +HELP = ( + "I can list databases, create database , delete database , " + "rotate password , set ops for to , browse , " + "explain tenants, or open home, console, flow, or integrate. " + "Those run here with no model. Free-form chat uses OLLAMA_BASE_URL and does not call a paid provider." +) +ALLOWED = {"say", "navigate", "list", "create", "delete", "rotate", "limits", "browse", "explain"} +LIMITS = {"ops": "ops_per_sec", "connections": "max_conns", "bytes": "bytes_per_sec"} +SYSTEM = ( + "You control Cache Me Outside. Reply with one JSON object and no other text. " + 'Allowed actions: {"action":"say","text":"..."}, ' + '{"action":"navigate","href":"/"}, {"action":"list"}, ' + '{"action":"create","name":"example"}, {"action":"delete","name":"example"}, ' + '{"action":"rotate","name":"example"}, ' + '{"action":"limits","name":"example","field":"ops_per_sec","value":10}, ' + '{"action":"browse","name":"example"}, {"action":"explain","topic":"tenants"}. ' + "href is one of /, /console, /flow, /integrate. " + "field is ops_per_sec, max_conns, or bytes_per_sec. " + "topic is tenants, browser, flow, warm, hybrid, ttl, or integration. " + "Do not request any other action." +) + + +def plan(message): + low = " ".join(message.lower().split()) + if low in ("help", "what can you do", "?"): + return {"action": "say", "text": HELP} + m = re.match(r"(?:please )?open (?:the )?(home|console|flow|integrate)(?: page)?$", low) + if m: + return {"action": "navigate", "href": NAV[m.group(1)], "text": "Opening %s." % m.group(1)} + if low in ("list databases", "show databases"): + return {"action": "list"} + m = re.match(r"create database ([a-z0-9-]+)$", low) + if m: + return {"action": "create", "name": m.group(1)} + m = re.match(r"delete database ([a-z0-9-]+)$", low) + if m: + return {"action": "delete", "name": m.group(1)} + m = re.match(r"(?:rotate password|new password for) ([a-z0-9-]+)$", low) + if m: + return {"action": "rotate", "name": m.group(1)} + m = re.match(r"set (ops|connections|bytes) for ([a-z0-9-]+) to (\d+)$", low) + if m: + return {"action": "limits", "name": m.group(2), "field": LIMITS[m.group(1)], "value": int(m.group(3))} + m = re.match(r"(?:browse|show keys in) ([a-z0-9-]+)$", low) + if m: + return {"action": "browse", "name": m.group(1)} + m = re.match(r"explain ([a-z]+)$", low) + if m: + return {"action": "explain", "topic": m.group(1)} + return None + + +def _name(obj): + name = obj.get("name", "") + if not isinstance(name, str) or not NAME.match(name): + raise ValueError("name: 2-31 chars, lowercase letters, digits, dashes, starting with a letter") + return name + + +def perform(mgr, obj): + action = obj.get("action") + if action == "say": + text = obj.get("text", "") + if not isinstance(text, str): + raise ValueError("text must be a string") + return {"reply": text[:2000], "actions": []} + if action == "navigate": + href = obj.get("href") + if href not in NAV.values(): + raise ValueError("that page is not on this site") + return {"reply": obj.get("text") or "Opening that page.", "actions": [{"op": "navigate", "href": href}]} + if action == "explain": + topic = obj.get("topic") + if topic not in TOPICS: + raise ValueError("I can explain tenants, browser, flow, warm, hybrid, ttl, or integration.") + return {"reply": TOPICS[topic], "actions": []} + if action == "list": + rows = mgr.list() + names = ", ".join(d["name"] for d in rows) or "none yet" + return {"reply": "Databases: %s." % names, "actions": [{"op": "refresh"}]} + if action == "create": + out = mgr.create({"name": _name(obj)}) + reply = "Database %s is on port %s. Password, shown once: %s." % (out["name"], out["port"], out["password"]) + if out.get("sample_key"): + reply += " Sample key %s was added for the local demo." % out["sample_key"] + return {"reply": reply, "actions": [{"op": "refresh"}]} + if action == "delete": + out = mgr.delete(_name(obj)) + return {"reply": "Deleted %s. Keys under that prefix were kept." % out["deleted"], "actions": [{"op": "refresh"}]} + if action == "rotate": + out = mgr.rotate(_name(obj)) + return {"reply": "New password for %s, shown once: %s." % (out["name"], out["password"]), + "actions": [{"op": "refresh"}]} + if action == "limits": + field = obj.get("field") + value = obj.get("value") + if field not in ("ops_per_sec", "max_conns", "bytes_per_sec"): + raise ValueError("limit must be ops, connections, or bytes") + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("limit must be a non-negative integer") + out = mgr.update(_name(obj), {field: value}) + shown = "unlimited" if out[field] == 0 else str(out[field]) + return {"reply": "%s for %s is now %s." % (field, out["name"], shown), "actions": [{"op": "refresh"}]} + if action == "browse": + name = _name(obj) + found = mgr.keys(name, "/v1/databases/%s/keys" % name) + keys = found.get("keys") or [] + shown = ", ".join(keys[:20]) or "no keys" + return {"reply": "Keys in %s: %s." % (name, shown), + "actions": [{"op": "browse", "name": name}, {"op": "refresh"}]} + raise ValueError("I can't do that") + + +def parse_model(raw): + if not isinstance(raw, str): + return None + start, end = raw.find("{"), raw.rfind("}") + if start < 0 or end <= start: + return None + try: + obj = json.loads(raw[start:end + 1]) + except ValueError: + return None + if not isinstance(obj, dict) or obj.get("action") not in ALLOWED: + return None + return obj + + +class _RefuseRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("redirect refused") + + +def ask_ollama(message): + base = os.environ.get("OLLAMA_BASE_URL", "").strip().rstrip("/") + parts = urlsplit(base) + if parts.scheme not in ("http", "https") or not parts.hostname or parts.username or parts.password: + raise ValueError("OLLAMA_BASE_URL must be an http or https URL without a user") + model = os.environ.get("OLLAMA_MODEL", "llama3.2") + if not re.match(r"^[A-Za-z0-9._:-]{1,80}$", model): + raise ValueError("OLLAMA_MODEL is not a model name") + payload = json.dumps({ + "model": model, + "stream": False, + "messages": [ + {"role": "system", "content": SYSTEM}, + {"role": "user", "content": message}, + ], + }).encode() + opener = urllib.request.build_opener(_RefuseRedirect) + req = urllib.request.Request(base + "/api/chat", data=payload, headers={"Content-Type": "application/json"}) + with opener.open(req, timeout=8) as r: + data = json.loads(r.read(65536).decode()) + return data.get("message", {}).get("content", "") + + +def run(mgr, message, ask=None): + if not isinstance(message, str): + raise ValueError("message must be a string") + message = message.strip() + if not message or len(message) > 500: + raise ValueError("message must be 1-500 characters") + planned = plan(message) + if planned is None: + if ask is None and not os.environ.get("OLLAMA_BASE_URL"): + return {"reply": HELP, "actions": []} + try: + raw = (ask or ask_ollama)(message) + except Exception: + return {"reply": "The local model did not answer. " + HELP, "actions": []} + planned = parse_model(raw) + if planned is None: + return {"reply": "The local model did not ask for something I can do. " + HELP, "actions": []} + return perform(mgr, planned) diff --git a/site/e2e/package-lock.json b/site/e2e/package-lock.json new file mode 100644 index 00000000000..9ab5364448c --- /dev/null +++ b/site/e2e/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "cmo-site-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cmo-site-e2e", + "devDependencies": { + "@playwright/test": "1.55.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.1.tgz", + "integrity": "sha512-IVAh/nOJaw6W9g+RJVlIQJ6gSiER+ae6mKQ5CX1bERzQgbC1VSeBlwdvczT7pxb0GWiyrxH4TGKbMfDb4Sq/ig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.55.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.1.tgz", + "integrity": "sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.55.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.1.tgz", + "integrity": "sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/site/e2e/package.json b/site/e2e/package.json new file mode 100644 index 00000000000..011d8e14bba --- /dev/null +++ b/site/e2e/package.json @@ -0,0 +1,10 @@ +{ + "name": "cmo-site-e2e", + "private": true, + "scripts": { + "test": "playwright test" + }, + "devDependencies": { + "@playwright/test": "1.55.1" + } +} diff --git a/site/e2e/playwright.config.js b/site/e2e/playwright.config.js new file mode 100644 index 00000000000..65de02970e0 --- /dev/null +++ b/site/e2e/playwright.config.js @@ -0,0 +1,17 @@ +const { defineConfig } = require("@playwright/test"); + +module.exports = defineConfig({ + testDir: ".", + timeout: 30000, + workers: 1, + use: { + baseURL: "http://127.0.0.1:8091", + ...(process.env.PW_CHANNEL ? { channel: process.env.PW_CHANNEL } : {}), + }, + webServer: { + command: "python3 ../server.py --demo --listen 127.0.0.1:8091", + url: "http://127.0.0.1:8091/", + reuseExistingServer: false, + timeout: 20000, + }, +}); diff --git a/site/e2e/ui.spec.js b/site/e2e/ui.spec.js new file mode 100644 index 00000000000..aec9c1dab37 --- /dev/null +++ b/site/e2e/ui.spec.js @@ -0,0 +1,113 @@ +const { test, expect } = require("@playwright/test"); +const fs = require("fs"); + +fs.mkdirSync("screenshots", { recursive: true }); + +test("home, console, flow, and integrate", async ({ page }) => { + page.on("dialog", (dialog) => dialog.accept()); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + + await page.setViewportSize({ width: 1440, height: 900 }); + await page.goto("/"); + await expect(page.locator("#hero h1")).toHaveText("The cache that stays on your desk."); + await page.screenshot({ path: "screenshots/home-1440.png" }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.evaluate(() => window.scrollTo(0, 0)); + await page.screenshot({ path: "screenshots/home-390.png" }); + await page.setViewportSize({ width: 1440, height: 900 }); + await expect(page.getByRole("link", { name: "GitGoat" })).toHaveAttribute("href", "https://gitgoat.io"); + await expect(page.getByRole("link", { name: "RailYard" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Portkyl" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Komboe" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Cerebro" })).toBeVisible(); + await page.mouse.move(180, 120); + await expect.poll(() => page.evaluate(() => getComputedStyle(document.documentElement).getPropertyValue("--mx").trim())).toBe("180px"); + await page.getByRole("button", { name: "Explain tenants" }).click(); + await expect(page.locator("#agent-out")).toContainText("password"); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.getByRole("button", { name: "Menu" }).click(); + await expect(page.locator("#nav")).toBeVisible(); + await page.locator("#nav").getByRole("link", { name: "Console", exact: true }).click(); + + await page.setViewportSize({ width: 1440, height: 900 }); + await expect(page.locator("#demo-banner")).toBeVisible(); + await page.locator("#name").fill("A"); + await page.getByRole("button", { name: "Create database" }).click(); + await expect(page.locator("#err")).toContainText("Name"); + await page.locator("#name").fill("acme"); + await page.getByRole("button", { name: "Create database" }).click(); + await expect(page.locator("#result")).toContainText("shown once"); + await expect(page.locator("#result .url")).toHaveValue(/redis:\/\/acme:/); + await page.getByRole("button", { name: "Copy URL" }).click(); + await expect(page.locator("#dbs")).toContainText("acme"); + await expect(page.locator("#dbs")).toContainText("unlimited"); + + await page.getByRole("button", { name: "Browse" }).click(); + await page.getByRole("button", { name: "acme:welcome" }).click(); + await page.getByRole("button", { name: "Show value" }).click(); + await expect(page.locator("#kv")).toContainText("hello from Cache Me Outside"); + await page.getByRole("button", { name: "Hide value" }).click(); + await expect(page.locator("#kv pre")).toHaveText("hidden"); + await page.locator("#bmatch").fill("nope*"); + await page.getByRole("button", { name: "Search keys" }).click(); + await expect(page.locator("#keys")).toContainText("No keys match."); + await page.locator("#bmatch").fill("*"); + await page.getByRole("button", { name: "Search keys" }).click(); + await expect(page.locator("#keys")).toContainText("acme:welcome"); + + await page.getByRole("button", { name: "Limits" }).click(); + await page.locator("#lim-ops").fill("25"); + await page.getByRole("button", { name: "Save limits" }).click(); + await expect(page.locator("#dbs")).toContainText("ops/s 25"); + await page.getByRole("button", { name: "New password" }).click(); + await expect(page.locator("#result")).toContainText("shown once"); + await page.getByRole("button", { name: "List databases" }).click(); + await expect(page.locator("#agent-out")).toContainText("acme"); + await page.locator("#agent-q").fill("create database beta"); + await page.getByRole("button", { name: "Ask" }).click(); + await expect(page.locator("#dbs")).toContainText("beta"); + await page.getByRole("button", { name: "Create demo" }).click(); + await expect(page.locator("#dbs")).toContainText("demo"); + await page.locator("article", { hasText: "acme" }).getByRole("button", { name: "Delete" }).click(); + await expect(page.locator("#dbs")).not.toContainText("Port 7001"); + await page.evaluate(() => { + document.getElementById("result").hidden = true; + document.getElementById("agent-out").textContent = ""; + }); + await page.locator("#dbs").scrollIntoViewIfNeeded(); + await page.screenshot({ path: "screenshots/console-1440.png" }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: "screenshots/console-390.png" }); + + await page.setViewportSize({ width: 1440, height: 900 }); + await page.locator("#agent-q").fill("open flow"); + await page.getByRole("button", { name: "Ask" }).click(); + await expect(page).toHaveURL(/\/flow$/); + await expect(page.locator("#flow-note")).toBeVisible(); + await expect(page.locator("#stats")).toContainText("req/s"); + await expect.poll(() => page.locator("#dots circle").count()).toBeGreaterThan(0); + await page.screenshot({ path: "screenshots/flow-1440.png" }); + + await page.getByRole("link", { name: "Integrate" }).click(); + await page.getByRole("button", { name: "Show version" }).click(); + await expect(page.locator("#version-out")).toHaveText("cache-me-outside 1.0.0 (api v1)"); + await page.getByRole("button", { name: "Explain integration" }).click(); + await expect(page.locator("#agent-out")).toContainText("environment"); + expect(errors).toEqual([]); +}); + +test("reduced motion keeps the flow still", async ({ browser }) => { + const context = await browser.newContext({ reducedMotion: "reduce", viewport: { width: 390, height: 844 } }); + const page = await context.newPage(); + await page.goto("/flow"); + await page.mouse.move(40, 40); + await page.waitForTimeout(900); + await expect(page.locator("#dots circle")).toHaveCount(0); + await expect(page.locator("#stats")).toContainText("req/s"); + const mx = await page.evaluate(() => getComputedStyle(document.documentElement).getPropertyValue("--mx").trim()); + expect(mx).not.toBe("40px"); + await page.screenshot({ path: "screenshots/flow-390.png" }); + await context.close(); +}); diff --git a/site/gate.py b/site/gate.py new file mode 100644 index 00000000000..d8439896ac0 --- /dev/null +++ b/site/gate.py @@ -0,0 +1,124 @@ +"""Public pages and unauthenticated product routes for the management server. + +Scripts and styles are files, so the page does not need script-src 'unsafe-inline'. +The demo token is returned only to a loopback client of a demo process. +""" +import json +import os + +CSP = ( + "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; " + "img-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'" +) +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") +HERE = os.path.dirname(os.path.abspath(__file__)) +PAGES = { + "/": ("index.html", "text/html; charset=utf-8"), + "/console": ("console.html", "text/html; charset=utf-8"), + "/flow": ("flow.html", "text/html; charset=utf-8"), + "/integrate": ("integrate.html", "text/html; charset=utf-8"), + "/assets/style.css": ("style.css", "text/css; charset=utf-8"), + "/assets/site.js": ("site.js", "text/javascript; charset=utf-8"), + "/assets/console.js": ("console.js", "text/javascript; charset=utf-8"), + "/assets/flow.js": ("flow.js", "text/javascript; charset=utf-8"), + "/assets/favicon.svg": ("favicon.svg", "image/svg+xml"), +} +_AGENT = None +_FLOW = None + + +def _load(name, filename): + import importlib.util + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, filename)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _agent(): + global _AGENT + if _AGENT is None: + _AGENT = _load("cmo_site_agent", "agent.py") + return _AGENT + + +def _flow(): + global _FLOW + if _FLOW is None: + import importlib.util + path = os.path.normpath(os.path.join(HERE, "..", "tools", "flowview", "flow_feed.py")) + spec = importlib.util.spec_from_file_location("cmo_flow_feed", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + _FLOW = mod + return _FLOW + + +def manifest(): + path = os.path.normpath(os.path.join(HERE, "..", "integration", "manifest.json")) + with open(path) as f: + data = json.load(f) + return { + "name": data["name"], + "version": data["version"], + "api": data["api"], + "adopt": "See INTEGRATION.md. Pass the API address and token from the environment.", + } + + +def _send_file(handler, filename, ctype): + with open(os.path.join(ROOT, filename), "rb") as f: + body = f.read() + handler.send_response(200) + handler.send_header("Content-Type", ctype) + handler.send_header("Content-Length", str(len(body))) + handler.send_header("Cache-Control", "no-store") + handler.send_header("X-Content-Type-Options", "nosniff") + handler.send_header("Referrer-Policy", "no-referrer") + handler.send_header("Content-Security-Policy", CSP) + handler.send_header("X-Frame-Options", "DENY") + handler.end_headers() + handler.wfile.write(body) + + +def _sample(mgr): + src = getattr(mgr, "flow_source", None) + if src is None: + addr = None if getattr(mgr, "demo", False) else mgr.store.upstream + src = _flow().Source(addr) + mgr.flow_source = src + body = src.sample() + if getattr(mgr, "demo", False): + body = dict(body, source="demo") + return body + + +def try_public(handler, mgr, token, method): + path = handler.path.split("?")[0] + if method == "GET" and path in PAGES: + filename, ctype = PAGES[path] + _send_file(handler, filename, ctype) + return True + if method != "GET": + return False + if path == "/v1/integration": + handler._send(200, manifest()) + return True + if path == "/v1/flow": + try: + handler._send(200, _sample(mgr)) + except Exception: + handler._send(503, {"error": "feed unavailable"}) + return True + if path == "/v1/demo": + loopback = handler.client_address[0] in ("127.0.0.1", "::1") + if not getattr(mgr, "demo", False) or not loopback: + handler._send(404, {"error": "not found"}) + else: + handler._send(200, {"demo": True, "token": token}) + return True + return False + + +def agent_run(mgr, message): + return _agent().run(mgr, message) diff --git a/site/memstore.py b/site/memstore.py new file mode 100644 index 00000000000..3b854b3a919 --- /dev/null +++ b/site/memstore.py @@ -0,0 +1,70 @@ +"""In-memory RESP stand-in so the local demo can browse keys without a server.""" +import fnmatch +import socket +import threading + + +def enc(v): + if v is None: + return b"$-1\r\n" + if isinstance(v, bool) or not isinstance(v, (int, str, bytes, list)): + raise TypeError("unsupported reply") + if isinstance(v, int): + return b":%d\r\n" % v + if isinstance(v, str): + raw = v.encode() + return b"+%s\r\n" % raw if b"\r" not in raw and b"\n" not in raw else b"$%d\r\n%s\r\n" % (len(raw), raw) + if isinstance(v, bytes): + return b"$%d\r\n%s\r\n" % (len(v), v) + return b"*%d\r\n" % len(v) + b"".join(enc(x) for x in v) + + +class MemStore: + def __init__(self): + self.data = {} + self.lock = threading.Lock() + + def conn(self): + import cmo_browse as browse + left, right = socket.socketpair() + threading.Thread(target=self._loop, args=(right, browse), daemon=True).start() + return browse.Conn(None, None, None, None, sock=left) + + def _loop(self, sock, browse): + stream = sock.makefile("rb") + while True: + try: + cmd = browse._read(stream) + except Exception: + return + if not isinstance(cmd, list) or not cmd: + sock.sendall(b"-ERR empty\r\n") + continue + words = [x.decode() if isinstance(x, bytes) else str(x) for x in cmd] + try: + sock.sendall(enc(self._op(words))) + except Exception: + sock.sendall(b"-ERR failed\r\n") + + def _op(self, c): + op = c[0].upper() + with self.lock: + if op == "SCAN": + pat = c[3] if len(c) > 3 else "*" + keys = [k.encode() for k in sorted(self.data) if fnmatch.fnmatch(k, pat)] + return [b"0", keys] + key = c[1] if len(c) > 1 else "" + value = self.data.get(key) + if op == "TYPE": + if isinstance(value, bytes): + return "string" + return "none" + if op == "TTL": + return -1 + if value is None: + return None + if op == "STRLEN": + return len(value) + if op == "GET": + return value + raise ValueError("unsupported") diff --git a/site/server.py b/site/server.py new file mode 100644 index 00000000000..7a041cae1de --- /dev/null +++ b/site/server.py @@ -0,0 +1,96 @@ +"""Cache Me Outside site. + + python3 site/server.py --demo + CMO_ADMIN_TOKEN=... python3 site/server.py --config proxy.json --server 127.0.0.1:6379 + +--demo listens on 127.0.0.1 only, keeps a sample key per database, and never +reads a token from the repository. The real mode uses CMO_ADMIN_TOKEN. +""" +import os +import secrets +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MGMT = os.path.normpath(os.path.join(HERE, "..", "tools", "mgmt")) +if MGMT not in sys.path: + sys.path.insert(0, MGMT) + +import cmo_mgmt as mgmt # noqa: E402 +from memstore import MemStore # noqa: E402 + + +class _Acl: + def __init__(self): + self.users = {} + + def set_user(self, name, password): + self.users[name] = password + + def delete_user(self, name): + self.users.pop(name, None) + + +class DemoManager(mgmt.Manager): + def __init__(self, store): + super().__init__(store, _Acl(), "127.0.0.1") + self.demo = True + self.mem = MemStore() + self.browser_conn = self.mem.conn + + def create(self, body): + out = super().create(body) + with self.mem.lock: + self.mem.data[out["name"] + ":welcome"] = b"hello from Cache Me Outside" + out["sample_key"] = out["name"] + ":welcome" + return out + + +def build(argv=None): + import argparse + ap = argparse.ArgumentParser(description="Cache Me Outside site") + ap.add_argument("--demo", action="store_true", help="loopback site with an in-memory database") + ap.add_argument("--listen", default=None) + ap.add_argument("--config") + ap.add_argument("--server", default="127.0.0.1:6379") + ap.add_argument("--proxy-bind", default="127.0.0.1") + a = ap.parse_args(argv) + if a.demo: + listen = a.listen or "127.0.0.1:8088" + host, _, port = listen.rpartition(":") + if host not in ("127.0.0.1", "::1"): + ap.error("demo mode listens on 127.0.0.1 only") + token = secrets.token_urlsafe(24) + directory = tempfile.mkdtemp(prefix="cmo-demo-") + store = mgmt.Store(os.path.join(directory, "proxy.json"), "127.0.0.1:6379") + mgr = DemoManager(store) + return (host, int(port)), mgr, token, a + token = os.environ.get("CMO_ADMIN_TOKEN", "") + if len(token) < 24: + ap.error("set CMO_ADMIN_TOKEN (24+ chars)") + if not a.config: + ap.error("--config is required unless --demo") + listen = a.listen or "127.0.0.1:8088" + host, _, port = listen.rpartition(":") + if host in ("0.0.0.0", "::", "*"): + ap.error("refusing a wildcard listen address; put TLS on a specific address") + sh, _, sp = a.server.rpartition(":") + acl = mgmt.Acl(sh, int(sp), os.environ.get("CMO_ADMIN_USER", "default"), os.environ.get("CMO_ADMIN_PASSWORD")) + mgr = mgmt.Manager(mgmt.Store(a.config, a.server), acl, a.proxy_bind) + return (host, int(port)), mgr, token, a + + +def main(argv=None): + import http.server + addr, mgr, token, _a = build(argv) + srv = http.server.ThreadingHTTPServer(addr, mgmt.make_handler(mgr, token)) + where = "%s:%d" % addr + if getattr(mgr, "demo", False): + print("Cache Me Outside demo on http://%s (loopback only)" % where, file=sys.stderr) + else: + print("Cache Me Outside site on http://%s" % where, file=sys.stderr) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/site/static/console.html b/site/static/console.html new file mode 100644 index 00000000000..da7b198a816 --- /dev/null +++ b/site/static/console.html @@ -0,0 +1,69 @@ + + + + + +Console · Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+

Console

+

Databases

+

The token stays in this page's memory. Passwords are shown once and are not stored by this tool.

+
+
+
+
+ +

+

+
+

Create database

+ + + + + +
+ + + +
+
+
+
+ +
Cache Me OutsideHome
+ + + + diff --git a/site/static/console.js b/site/static/console.js new file mode 100644 index 00000000000..e07403a38f7 --- /dev/null +++ b/site/static/console.js @@ -0,0 +1,253 @@ +"use strict"; +const $ = function (id) { return document.getElementById(id); }; +let current = ""; + +function limitLabel(value) { + return value ? String(value) : "unlimited"; +} + +async function api(method, path, body) { + const token = await window.CMO.ensureToken(); + const response = await fetch(path, { + method: method, + headers: {"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + body: body ? JSON.stringify(body) : undefined, + }); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error || String(response.status)); + return payload; +} + +function showSecret(db) { + const box = $("result"); + box.hidden = false; + box.textContent = ""; + const title = document.createElement("strong"); + title.textContent = "Database " + db.name + " is ready. Copy the password now. It is shown once."; + const host = location.hostname; + const url = "redis://" + encodeURIComponent(db.name) + ":" + encodeURIComponent(db.password) + "@" + host + ":" + db.port; + const input = document.createElement("input"); + input.className = "url"; + input.readOnly = true; + input.value = url; + input.setAttribute("aria-label", "Connection URL, shown once"); + const copy = document.createElement("button"); + copy.className = "btn"; + copy.type = "button"; + copy.textContent = "Copy URL"; + copy.addEventListener("click", async function () { + try { + await navigator.clipboard.writeText(url); + copy.textContent = "Copied"; + } catch (e) { + input.focus(); + input.select(); + copy.textContent = "Select the URL and copy"; + } + }); + const cli = document.createElement("p"); + cli.textContent = "CLI: valkey-cli -h " + host + " -p " + db.port + " --user " + db.name + " --askpass"; + const prefix = document.createElement("p"); + prefix.textContent = "Keys must start with " + db.name + ":"; + box.append(title, input, copy, cli, prefix); + if (db.sample_key) { + const sample = document.createElement("p"); + sample.textContent = "Local demo sample key: " + db.sample_key; + box.append(sample); + } +} + +function button(label, onClick) { + const el = document.createElement("button"); + el.className = "btn"; + el.type = "button"; + el.textContent = label; + el.addEventListener("click", onClick); + return el; +} + +async function load() { + $("err").textContent = ""; + const list = $("dbs"); + list.textContent = ""; + try { + const body = await api("GET", "/v1/databases"); + if (!body.databases.length) { + const empty = document.createElement("p"); + empty.className = "fine"; + empty.textContent = "No databases yet."; + list.append(empty); + return; + } + body.databases.forEach(function (db) { + const card = document.createElement("article"); + card.className = "card"; + const title = document.createElement("h3"); + title.textContent = db.name; + const meta = document.createElement("p"); + meta.className = "meta"; + meta.textContent = "Port " + db.port + + " · ops/s " + limitLabel(db.ops_per_sec) + + " · connections " + limitLabel(db.max_conns) + + " · bytes/s " + limitLabel(db.bytes_per_sec); + const actions = document.createElement("div"); + actions.className = "row"; + actions.append( + button("Browse", function () { browse(db.name, "0"); }), + button("New password", async function () { + if (!window.confirm("Replace the password for " + db.name + "? The old one stops working.")) return; + try { showSecret(Object.assign({}, db, await api("POST", "/v1/databases/" + db.name + "/rotate-password", {}))); } + catch (e) { $("err").textContent = e.message; } + }), + button("Limits", function () { limits(db); }), + button("Delete", async function () { + if (!window.confirm("Delete " + db.name + "? Keys under " + db.name + ": stay on the server.")) return; + try { await api("DELETE", "/v1/databases/" + db.name); await load(); } + catch (e) { $("err").textContent = e.message; } + }) + ); + card.append(title, meta, actions); + list.append(card); + }); + } catch (e) { + $("err").textContent = e.message; + } +} + +function limits(db) { + const box = $("result"); + box.hidden = false; + box.textContent = ""; + const form = document.createElement("form"); + form.id = "limits"; + const ops = field("Ops per second (0 means unlimited)", "lim-ops", db.ops_per_sec); + const conns = field("Max connections (0 means unlimited)", "lim-conns", db.max_conns); + const bytes = field("Bytes per second (0 means unlimited)", "lim-bytes", db.bytes_per_sec); + const save = document.createElement("button"); + save.className = "btn p"; + save.type = "submit"; + save.textContent = "Save limits"; + form.append(ops, conns, bytes, save); + form.addEventListener("submit", async function (e) { + e.preventDefault(); + try { + await api("PATCH", "/v1/databases/" + db.name, { + ops_per_sec: Number(ops.querySelector("input").value), + max_conns: Number(conns.querySelector("input").value), + bytes_per_sec: Number(bytes.querySelector("input").value), + }); + box.hidden = true; + await load(); + } catch (err) { + $("err").textContent = err.message; + } + }); + const heading = document.createElement("h3"); + heading.textContent = "Limits for " + db.name; + box.append(heading, form); +} + +function field(label, id, value) { + const wrap = document.createElement("label"); + wrap.className = "field"; + wrap.textContent = label; + const input = document.createElement("input"); + input.id = id; + input.type = "number"; + input.min = "0"; + input.required = true; + input.value = String(value || 0); + wrap.append(input); + return wrap; +} + +async function browse(name, cursor) { + $("err").textContent = ""; + $("browser").hidden = false; + $("bname").textContent = "Keys in " + name; + if (cursor === "0") $("keys").textContent = ""; + current = name; + try { + const body = await api("GET", "/v1/databases/" + encodeURIComponent(name) + + "/keys?cursor=" + encodeURIComponent(cursor) + + "&match=" + encodeURIComponent($("bmatch").value)); + if (!body.keys.length && cursor === "0") { + const empty = document.createElement("li"); + empty.textContent = "No keys match."; + $("keys").append(empty); + } + body.keys.forEach(function (key) { + const item = document.createElement("li"); + item.append(button(key, function () { openKey(name, key); })); + $("keys").append(item); + }); + $("bmore").hidden = body.cursor === "0"; + $("bmore").onclick = function () { browse(name, body.cursor); }; + } catch (e) { + $("err").textContent = e.message; + } +} + +async function openKey(name, key) { + try { + const body = await api("GET", "/v1/databases/" + encodeURIComponent(name) + "/keys?key=" + encodeURIComponent(key)); + const box = $("kv"); + box.hidden = false; + box.textContent = ""; + const meta = document.createElement("p"); + meta.textContent = body.key + " · " + body.type + " · TTL " + (body.ttl < 0 ? "none" : body.ttl + "s") + + (body.size !== undefined ? " · size " + body.size : "") + + (body.truncated ? " · first 100 shown" : ""); + const pre = document.createElement("pre"); + const real = JSON.stringify(body.value, null, 2); + pre.textContent = "hidden"; + const eye = document.createElement("button"); + eye.className = "btn"; + eye.type = "button"; + eye.textContent = "Show value"; + eye.setAttribute("aria-pressed", "false"); + eye.addEventListener("click", function () { + const on = eye.getAttribute("aria-pressed") !== "true"; + eye.setAttribute("aria-pressed", on ? "true" : "false"); + pre.textContent = on ? real : "hidden"; + eye.textContent = on ? "Hide value" : "Show value"; + }); + box.append(meta, eye, pre); + } catch (e) { + $("err").textContent = e.message; + } +} + +$("load").addEventListener("click", load); +$("create").addEventListener("submit", async function (e) { + e.preventDefault(); + $("err").textContent = ""; + const name = $("name").value; + if (!/^[a-z][a-z0-9-]{1,30}$/.test(name)) { + $("err").textContent = "Name: 2-31 characters, a lowercase letter, then letters, digits, or dashes."; + return; + } + const body = {name: name}; + ["ops", "conns", "bytes"].forEach(function (id) { + const raw = $(id).value; + if (raw === "") return; + const key = {ops: "ops_per_sec", conns: "max_conns", bytes: "bytes_per_sec"}[id]; + body[key] = Number(raw); + }); + try { + showSecret(await api("POST", "/v1/databases", body)); + $("name").value = ""; + await load(); + } catch (err) { + $("err").textContent = err.message; + } +}); +$("bform").addEventListener("submit", function (e) { + e.preventDefault(); + if (current) browse(current, "0"); +}); +document.addEventListener("cmo-refresh", load); +document.addEventListener("cmo-browse", function (e) { + if (e.detail) browse(e.detail, "0"); +}); +window.CMO.ensureToken().then(function (token) { if (token) load(); }); diff --git a/site/static/favicon.svg b/site/static/favicon.svg new file mode 100644 index 00000000000..d3efaf3a916 --- /dev/null +++ b/site/static/favicon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/site/static/flow.html b/site/static/flow.html new file mode 100644 index 00000000000..38ab713a7fb --- /dev/null +++ b/site/static/flow.html @@ -0,0 +1,60 @@ + + + + + +Flow · Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+

Live data flow

+

From the request to the database

+ +

waiting for the flow feed

+
+
+
+
+ + + + + + +request +near cache +near hit +cache hit +miss +DB query + + +
+
+
+ + + + + + diff --git a/site/static/flow.js b/site/static/flow.js new file mode 100644 index 00000000000..a2afca2289b --- /dev/null +++ b/site/static/flow.js @@ -0,0 +1,58 @@ +"use strict"; +const NS = "http://www.w3.org/2000/svg"; +const lanes = {req: "req", near_hit: "near", cache_hit: "hit", miss: "miss", db: "db"}; +const pathOf = {req: "p-req", near: "p-near", hit: "p-hit", miss: "p-miss", db: "p-db"}; +const reduce = window.matchMedia("(prefers-reduced-motion: reduce)").matches; +let rates = {}; +let dots = []; +const acc = {}; +const MAX_DOTS = 400; +const SCALE = 0.05; +const SPEED = 0.6; + +function spawn(cls) { + if (reduce || dots.length >= MAX_DOTS) return; + const dot = document.createElementNS(NS, "circle"); + dot.setAttribute("class", "dot " + (cls === "db" ? "dbdot" : cls)); + dot.setAttribute("r", "4"); + document.getElementById("dots").appendChild(dot); + dots.push({el: dot, p: document.getElementById(pathOf[cls]), t: 0}); +} + +let last = performance.now(); +function tick(now) { + const dt = Math.min(0.1, (now - last) / 1000); + last = now; + Object.keys(lanes).forEach(function (key) { + acc[key] = (acc[key] || 0) + (rates[key] || 0) * SCALE * dt; + while (acc[key] >= 1) { spawn(lanes[key]); acc[key] -= 1; } + }); + dots = dots.filter(function (dot) { + dot.t += SPEED * dt; + if (dot.t >= 1 || !dot.p) { dot.el.remove(); return false; } + const point = dot.p.getPointAtLength(dot.t * dot.p.getTotalLength()); + dot.el.setAttribute("cx", point.x); + dot.el.setAttribute("cy", point.y); + return true; + }); + if (!reduce) requestAnimationFrame(tick); +} + +async function poll() { + const stats = document.getElementById("stats"); + const note = document.getElementById("flow-note"); + try { + const response = await fetch("/v1/flow", {cache: "no-store"}); + if (!response.ok) throw new Error("feed unavailable"); + rates = await response.json(); + const ratio = rates.hit_ratio == null ? "none" : (rates.hit_ratio * 100).toFixed(1) + "%"; + stats.textContent = "req/s " + rates.req + " near " + rates.near_hit + " hit " + rates.cache_hit + + " miss " + rates.miss + " db " + rates.db + " hit ratio " + ratio; + if (note) note.hidden = rates.source !== "demo"; + } catch (e) { + stats.textContent = "feed unavailable"; + } +} +poll(); +setInterval(poll, 1000); +if (!reduce) requestAnimationFrame(tick); diff --git a/site/static/index.html b/site/static/index.html new file mode 100644 index 00000000000..a85b829df90 --- /dev/null +++ b/site/static/index.html @@ -0,0 +1,72 @@ + + + + + +Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+
+

Cache Me Outside

+

The cache that stays on your desk.

+

Create a database, read its keys, and watch hits and misses move. The agent on this page can do those things with you.

+
Open the consoleWatch the flow
+
+ +
+
+
+
+

What it does

+

Everything on this page is Cache Me Outside. Names of other products appear only where a card points at a similar feature.

+
+

Databases

Each database gets a port, a password that is returned once, and a key prefix. A limit of zero means unlimited.

+

Key browser

Browse is read-only. It will not return a key that sits outside the database prefix, and it will not run a write.

+

Live flow

Requests split into a near-cache hit, a cache hit, or a miss that continues to the database.

+

Warm restart

A clean shutdown can leave a checkpoint for the next process when the usual snapshot did not. It stays off until you turn it on.

+

Hybrid storage

Large strings can spill to a flash log. That path stays off until you configure it.

+

TTL segments

Keys that expire in the same time bucket can be removed together. That path stays off until you configure it.

+

Same wire protocol

Clients that already speak this protocol keep working. Comparable caches:

ValkeyRedisMemcached

+ +
+
+
+
+ + + + + diff --git a/site/static/integrate.html b/site/static/integrate.html new file mode 100644 index 00000000000..71633e57bd5 --- /dev/null +++ b/site/static/integrate.html @@ -0,0 +1,73 @@ + + + + + +Integrate · Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+

Integrate

+

Drop the cache into another GoDesk app.

+

Copy the integration directory or point a script tag at a release. The token stays in the environment.

+
+
+
+
+
+

Python

+
import os
+from cache_me_outside import Client
+
+client = Client(os.environ["CMO_API"], os.environ["CMO_ADMIN_TOKEN"])
+print(client.list_databases())
+
+
+

Browser or Node

+
const cache = CacheMeOutside.attach({
+  api: process.env.CMO_API,
+  token: process.env.CMO_ADMIN_TOKEN
+});
+const rows = await cache.listDatabases();
+
+
+

Sidecar

+
CMO_API=http://127.0.0.1:8088 \
+CMO_ADMIN_TOKEN="$CMO_ADMIN_TOKEN" \
+python3 integration/sidecar.py
+

Then GET http://127.0.0.1:8099/health from the app.

+
+
+

This build

+

The running site reports its integration version. Updating is a separate command, integration/updater.py, and it checks a published checksum before replacing files.

+ +

+
+
+
+
+ +
Cache Me OutsideHome
+ + + diff --git a/site/static/site.js b/site/static/site.js new file mode 100644 index 00000000000..7be0a4fc7f3 --- /dev/null +++ b/site/static/site.js @@ -0,0 +1,86 @@ +"use strict"; +const reduce = window.matchMedia("(prefers-reduced-motion: reduce)").matches; +const menu = document.getElementById("menu"); +const nav = document.getElementById("nav"); +if (menu && nav) { + menu.addEventListener("click", function () { + const open = nav.classList.toggle("open"); + menu.setAttribute("aria-expanded", open ? "true" : "false"); + }); +} +if (!reduce) { + window.addEventListener("pointermove", function (e) { + document.documentElement.style.setProperty("--mx", e.clientX + "px"); + document.documentElement.style.setProperty("--my", e.clientY + "px"); + }); +} + +let sessionToken = ""; +async function ensureToken() { + const field = document.getElementById("tok"); + if (field && field.value) return field.value; + if (sessionToken) return sessionToken; + const response = await fetch("/v1/demo", {cache: "no-store"}); + if (!response.ok) return ""; + const body = await response.json(); + sessionToken = body.token || ""; + if (field) field.value = sessionToken; + const banner = document.getElementById("demo-banner"); + if (banner) banner.hidden = false; + return sessionToken; +} + +async function agentAsk(message) { + const out = document.getElementById("agent-out"); + const token = await ensureToken(); + if (!token) { + out.textContent = "Paste the admin token on the console. This page does not keep a token of its own."; + return; + } + const response = await fetch("/v1/agent", { + method: "POST", + headers: {"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + body: JSON.stringify({message: message}), + }); + const body = await response.json(); + if (!response.ok) { + out.textContent = body.error || "The agent could not do that."; + return; + } + out.textContent = body.reply || ""; + (body.actions || []).forEach(function (action) { + if (action.op === "navigate" && action.href) window.location.assign(action.href); + if (action.op === "refresh") document.dispatchEvent(new Event("cmo-refresh")); + if (action.op === "browse") document.dispatchEvent(new CustomEvent("cmo-browse", {detail: action.name})); + }); +} + +const agentForm = document.getElementById("agent-form"); +if (agentForm) { + agentForm.addEventListener("submit", function (e) { + e.preventDefault(); + agentAsk(document.getElementById("agent-q").value); + }); + document.querySelectorAll("[data-ask]").forEach(function (button) { + button.addEventListener("click", function () { + document.getElementById("agent-q").value = button.getAttribute("data-ask"); + agentAsk(button.getAttribute("data-ask")); + }); + }); +} + +const versionButton = document.getElementById("show-version"); +if (versionButton) { + versionButton.addEventListener("click", async function () { + const slot = document.getElementById("version-out"); + try { + const response = await fetch("/v1/integration", {cache: "no-store"}); + const body = await response.json(); + slot.textContent = body.name + " " + body.version + " (api " + body.api + ")"; + } catch (e) { + slot.textContent = "The version endpoint did not answer."; + } + }); +} + +window.CMO = {ensureToken: ensureToken}; diff --git a/site/static/style.css b/site/static/style.css new file mode 100644 index 00000000000..c3cfaa6fa83 --- /dev/null +++ b/site/static/style.css @@ -0,0 +1,217 @@ +:root { + --bg: #101614; + --tx: #f6f1e7; + --mu: #b7c2ba; + --ac: #f0c48a; + --gold: #e7b56a; + --mint: #9be7c4; + --line: rgba(255, 255, 255, .14); + --mx: 50vw; + --my: 18vh; + --shadow: 8px 8px 18px #070b09, -6px -6px 16px #24302b; + --inset: inset 5px 5px 10px #070b09, inset -4px -4px 10px #2a3832; +} +* { box-sizing: border-box; } +html { scroll-behavior: smooth; } +body { + margin: 0; + min-height: 100vh; + font-family: "Avenir Next", "Segoe UI", sans-serif; + background: var(--bg); + color: var(--tx); + line-height: 1.55; +} +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + z-index: 0; + background: + radial-gradient(480px circle at var(--mx) var(--my), rgba(240, 196, 138, .16), transparent 42%), + radial-gradient(820px circle at 100% -10%, rgba(155, 231, 196, .12), transparent 40%); +} +body > * { position: relative; z-index: 1; } +a { color: var(--ac); } +.wrap { max-width: 1120px; margin: 0 auto; padding: 0 24px; } +.skip { position: absolute; left: -999px; } +.skip:focus { left: 12px; top: 12px; z-index: 20; background: #fff; color: #111; padding: 8px 12px; } +header { + position: sticky; + top: 0; + z-index: 8; + background: rgba(16, 22, 20, .62); + backdrop-filter: blur(18px); + border-bottom: 1px solid var(--line); +} +.nav { display: flex; align-items: center; gap: 22px; min-height: 68px; } +.brand { display: flex; align-items: center; gap: 10px; color: var(--tx); text-decoration: none; font-weight: 700; font-size: 18px; } +.mark { display: block; filter: drop-shadow(0 6px 12px rgba(0, 0, 0, .35)); } +.links { display: flex; gap: 18px; margin-left: auto; align-items: center; flex-wrap: wrap; } +.links a { color: var(--mu); text-decoration: none; } +.links a:hover, .links a[aria-current="page"], .links a:focus-visible { color: var(--tx); } +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + min-height: 44px; + padding: 12px 18px; + border: 0; + border-radius: 14px; + background: #17211d; + color: var(--tx); + font: inherit; + font-weight: 650; + text-decoration: none; + cursor: pointer; + box-shadow: var(--shadow); +} +.btn.p { + background: linear-gradient(165deg, #e7b56a, #c47a3a); + color: #1a1208; + box-shadow: 7px 7px 16px #070b09, -3px -3px 10px rgba(155, 231, 196, .28); +} +.btn:hover { transform: translateY(-1px); } +.btn:focus-visible, a:focus-visible, input:focus-visible, button:focus-visible, select:focus-visible { + outline: 2px solid var(--mint); + outline-offset: 3px; +} +.btn:disabled { opacity: .6; cursor: progress; transform: none; } +#menu { + display: none; + margin-left: auto; + min-width: 44px; + min-height: 44px; + border: 0; + border-radius: 12px; + background: #17211d; + color: var(--tx); + font: inherit; + box-shadow: var(--shadow); + cursor: pointer; +} +.hero { padding: 72px 0 36px; } +.hero h1 { font-size: clamp(36px, 6vw, 64px); line-height: 1.05; letter-spacing: -.03em; margin: 0 0 14px; } +.hero p { color: var(--mu); font-size: clamp(17px, 2vw, 20px); max-width: 640px; } +.eyebrow { color: var(--gold); font-size: 13px; letter-spacing: .14em; text-transform: uppercase; margin-bottom: 8px; } +.cta { display: flex; gap: 12px; flex-wrap: wrap; } +.hero-grid { display: grid; grid-template-columns: minmax(0, 1.2fr) minmax(260px, .8fr); gap: 28px; align-items: center; } +.page-head { padding: 36px 0 8px; } +.page-head h1 { font-size: clamp(32px, 4vw, 48px); margin: 0 0 8px; letter-spacing: -.02em; } +section { padding: 28px 0 48px; } +h2 { font-size: clamp(26px, 3vw, 36px); margin: 0 0 8px; letter-spacing: -.02em; } +.lead { color: var(--mu); max-width: 680px; } +.grid { display: grid; gap: 16px; grid-template-columns: repeat(auto-fit, minmax(min(100%, 240px), 1fr)); } +.card { + background: rgba(255, 255, 255, .06); + border: 1px solid var(--line); + border-radius: 22px; + padding: 22px; + backdrop-filter: blur(18px); + box-shadow: 0 16px 40px rgba(0, 0, 0, .28); +} +.card h3 { margin: 0 0 8px; font-size: 18px; } +.card p, .fine { color: var(--mu); } +.srcs { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 12px; } +.src { + display: inline-block; + padding: 4px 10px; + border-radius: 999px; + background: rgba(240, 196, 138, .12); + color: var(--ac); + text-decoration: none; + font-size: 13px; + font-weight: 650; +} +.field { display: block; margin: 12px 0; font-size: 14px; } +.field input, .field select { + display: block; + width: 100%; + margin-top: 6px; + min-height: 44px; + padding: 10px 12px; + border: 0; + border-radius: 14px; + background: #0c1210; + color: var(--tx); + font: inherit; + box-shadow: var(--inset); +} +.row { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; } +.err { + color: #ffd0d0; + background: rgba(120, 24, 36, .45); + border: 1px solid rgba(255, 180, 180, .35); + border-radius: 12px; + padding: 10px 12px; + margin: 8px 0; +} +.err:empty { display: none; } +.banner { color: var(--gold); margin: 0 0 12px; } +.db { display: grid; gap: 12px; } +.db article { display: grid; gap: 8px; } +.meta { color: var(--mu); font-variant-numeric: tabular-nums; } +.keys { list-style: none; padding: 0; margin: 12px 0; display: grid; gap: 8px; } +.keys button { text-align: left; } +pre, code { font-family: ui-monospace, "SFMono-Regular", monospace; } +pre { + white-space: pre-wrap; + background: rgba(0, 0, 0, .35); + border: 1px solid var(--line); + border-radius: 12px; + padding: 12px; + overflow: auto; +} +.url { + width: 100%; + min-height: 44px; + border: 0; + border-radius: 12px; + padding: 10px 12px; + background: #0c1210; + color: var(--tx); + box-shadow: var(--inset); +} +.assistant { margin: 0 auto 40px; } +.out { min-height: 3em; } +footer { border-top: 1px solid var(--line); padding: 28px 0 40px; color: var(--mu); } +footer .wrap { display: flex; gap: 16px; flex-wrap: wrap; justify-content: space-between; } +footer a { color: var(--mu); text-decoration: none; } +svg.flow { width: 100%; height: auto; display: block; } +.node rect { fill: #17211d; stroke: #e7b56a; stroke-width: 2; } +.node text { fill: var(--tx); text-anchor: middle; font-size: 14px; } +.path { fill: none; stroke: #2c4038; stroke-width: 3; } +.dot { r: 4; } +.req { fill: #f0c48a; } +.near { fill: #9be7c4; } +.hit { fill: #7ddec0; } +.miss { fill: #e07a7a; } +.dbdot { fill: #e7b56a; } +@media (max-width: 860px) { + .hero-grid { grid-template-columns: 1fr; } + .hero-copy { text-align: center; } + .hero-copy p, .hero-copy .cta { margin-left: auto; margin-right: auto; } + .hero-copy .cta { justify-content: center; } + .links { + display: none; + position: absolute; + top: 68px; + left: 0; + right: 0; + flex-direction: column; + align-items: stretch; + gap: 8px; + padding: 12px 20px 18px; + background: rgba(16, 22, 20, .94); + backdrop-filter: blur(16px); + border-bottom: 1px solid var(--line); + } + .links.open { display: flex; } + #menu { display: inline-flex; align-items: center; justify-content: center; } +} +@media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + body::before { background: none; } + .btn:hover { transform: none; } +} diff --git a/site/test_site.py b/site/test_site.py new file mode 100644 index 00000000000..fc78a13aa32 --- /dev/null +++ b/site/test_site.py @@ -0,0 +1,190 @@ +import json +import os +import sys +import threading +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "..", "tools", "mgmt")) + +import agent +import cmo_mgmt as mgmt +import server + + +def start(argv): + addr, mgr, token, _args = server.build(argv) + httpd = ThreadingHTTPServer(addr, mgmt.make_handler(mgr, token)) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd, token, mgr + + +def get(url): + with urllib.request.urlopen(url) as response: + return response.status, dict(response.headers), response.read() + + +class Site(unittest.TestCase): + def setUp(self): + self.httpd, self.token, self.mgr = start(["--demo", "--listen", "127.0.0.1:0"]) + self.base = "http://127.0.0.1:%d" % self.httpd.server_address[1] + + def tearDown(self): + self.httpd.shutdown() + self.httpd.server_close() + + def call(self, method, path, body=None, token=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + self.base + path, data=data, method=method, + headers={"Authorization": "Bearer " + (token if token is not None else self.token), + "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as exc: + return exc.code, json.load(exc) + + def test_pages_and_csp(self): + status, headers, body = get(self.base + "/") + html = body.decode() + self.assertEqual(status, 200) + self.assertIn('id="hero"', html) + self.assertNotIn("unsafe-inline", headers["Content-Security-Policy"]) + self.assertNotIn("innerHTML", html) + self.assertIn("GitGoat", html) + self.assertNotIn("GitGoat", html.split('id="hero"')[1].split("")[0]) + console = get(self.base + "/console")[2].decode() + self.assertIn("Create database", console) + self.assertNotIn('pattern="[a-z][a-z0-9-]', console) + + def test_demo_token_and_database_roundtrip(self): + demo = self.call("GET", "/v1/demo", token="ignored") + # /v1/demo is public. The helper still sends a token; that is fine. + status, payload = demo + self.assertEqual(status, 200) + self.assertEqual(payload["token"], self.token) + created = self.call("POST", "/v1/databases", {"name": "acme"})[1] + self.assertEqual(created["sample_key"], "acme:welcome") + scanned = self.call("GET", "/v1/databases/acme/keys?match=w*")[1] + self.assertEqual(scanned["keys"], ["acme:welcome"]) + shown = self.call("GET", "/v1/databases/acme/keys?key=acme%3Awelcome")[1] + self.assertEqual(shown["value"], "hello from Cache Me Outside") + self.assertEqual(self.call("GET", "/v1/databases/acme/keys?key=other%3Ax")[0], 403) + self.assertEqual(self.call("PATCH", "/v1/databases/acme", {"ops_per_sec": 25})[1]["ops_per_sec"], 25) + self.assertEqual(self.call("DELETE", "/v1/databases/acme")[0], 200) + + def test_agent_creates_without_a_model(self): + os.environ.pop("OLLAMA_BASE_URL", None) + status, payload = self.call("POST", "/v1/agent", {"message": "create database beta"}) + self.assertEqual(status, 200) + self.assertIn("beta", payload["reply"]) + self.assertIn("Password", payload["reply"]) + names = [row["name"] for row in self.call("GET", "/v1/databases")[1]["databases"]] + self.assertIn("beta", names) + opened = self.call("POST", "/v1/agent", {"message": "open flow"})[1] + self.assertEqual(opened["actions"][0]["href"], "/flow") + refused = self.call("POST", "/v1/agent", {"message": "write a poem about caches"})[1] + self.assertIn("OLLAMA_BASE_URL", refused["reply"]) + self.assertEqual(refused["actions"], []) + + def test_flow_is_labeled_demo(self): + status, payload = self.call("GET", "/v1/flow", token="ignored") + self.assertEqual(status, 200) + self.assertEqual(payload["source"], "demo") + self.assertGreater(payload["req"], 0) + version = self.call("GET", "/v1/integration", token="ignored")[1] + self.assertEqual(version["version"], "1.0.0") + + +class Agent(unittest.TestCase): + def test_model_cannot_request_a_shell(self): + class Store: + def __init__(self): + self.names = [] + def list(self): + return [{"name": n} for n in self.names] + def create(self, body): + self.names.append(body["name"]) + return {"name": body["name"], "port": 7001, "password": "pw"} + store = Store() + reply = agent.run(store, "do something unusual", ask=lambda _m: '{"action":"shell","cmd":"id"}') + self.assertEqual(store.names, []) + self.assertIn("something I can do", reply["reply"]) + made = agent.run(store, "please make one", ask=lambda _m: '{"action":"create","name":"from-model"}') + self.assertEqual(store.names, ["from-model"]) + self.assertIn("from-model", made["reply"]) + + def test_bad_ollama_url_does_not_run_a_command(self): + os.environ["OLLAMA_BASE_URL"] = "file:///tmp/not-a-server" + try: + reply = agent.run(object(), "tell me anything at all") + finally: + os.environ.pop("OLLAMA_BASE_URL", None) + self.assertIn("did not answer", reply["reply"]) + + def test_ollama_say_and_redirect_refused(self): + class H(BaseHTTPRequestHandler): + def do_POST(self): + n = int(self.headers.get("Content-Length") or 0) + self.rfile.read(n) + if "bounce" in self.path: + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:9/nope") + self.end_headers() + return + body = json.dumps({"message": {"content": '{"action":"say","text":"from the local model"}'}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *a): + pass + httpd = ThreadingHTTPServer(("127.0.0.1", 0), H) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + os.environ["OLLAMA_BASE_URL"] = "http://127.0.0.1:%d" % httpd.server_address[1] + try: + said = agent.run(object(), "tell me anything at all") + self.assertEqual(said["reply"], "from the local model") + os.environ["OLLAMA_BASE_URL"] = "http://127.0.0.1:%d/bounce" % httpd.server_address[1] + bounced = agent.run(object(), "tell me anything at all") + self.assertIn("did not answer", bounced["reply"]) + finally: + os.environ.pop("OLLAMA_BASE_URL", None) + httpd.shutdown() + httpd.server_close() + + +class Listen(unittest.TestCase): + def test_demo_refuses_a_public_address(self): + with self.assertRaises(SystemExit): + server.build(["--demo", "--listen", "0.0.0.0:8088"]) + + def test_real_mode_hides_the_demo_token(self): + import tempfile + previous = os.environ.get("CMO_ADMIN_TOKEN") + os.environ["CMO_ADMIN_TOKEN"] = "t" * 32 + directory = tempfile.mkdtemp() + try: + httpd, _token, mgr = start(["--config", os.path.join(directory, "p.json"), "--listen", "127.0.0.1:0"]) + self.assertFalse(getattr(mgr, "demo", False)) + base = "http://127.0.0.1:%d" % httpd.server_address[1] + with self.assertRaises(urllib.error.HTTPError) as caught: + urllib.request.urlopen(base + "/v1/demo") + self.assertEqual(caught.exception.code, 404) + httpd.shutdown() + httpd.server_close() + finally: + if previous is None: + os.environ.pop("CMO_ADMIN_TOKEN", None) + else: + os.environ["CMO_ADMIN_TOKEN"] = previous + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/flowview/flow.js b/tools/flowview/flow.js index 556478a7475..fa0490a7934 100644 --- a/tools/flowview/flow.js +++ b/tools/flowview/flow.js @@ -36,4 +36,5 @@ async function poll() { `req/s ${rates.req} near ${rates.near_hit} hit ${rates.cache_hit} miss ${rates.miss} db ${rates.db} hit ratio ${hr}`; } catch (e) { document.getElementById("stats").textContent = "feed unavailable"; } } -poll(); setInterval(poll, 1000); requestAnimationFrame(tick); +poll(); setInterval(poll, 1000); +if (!window.matchMedia("(prefers-reduced-motion: reduce)").matches) requestAnimationFrame(tick); diff --git a/tools/mgmt/cmo_mgmt.py b/tools/mgmt/cmo_mgmt.py index 77dc6fb0f05..68fc1e54a34 100644 --- a/tools/mgmt/cmo_mgmt.py +++ b/tools/mgmt/cmo_mgmt.py @@ -13,8 +13,9 @@ PATCH /v1/databases/{name} limits only DELETE /v1/databases/{name} POST /v1/databases/{name}/rotate-password + GET /v1/databases/{name}/keys read-only, prefix-locked browser GET /v1/openapi.json - GET / web console (token is typed in the page, kept in memory only) + GET / /console /flow /integrate site pages (scripts and styles are separate files) The proxy reads its config at start, so restart it after changes. @@ -32,6 +33,7 @@ import socket import tempfile import threading +from urllib.parse import parse_qs, urlsplit NAME_RE = re.compile(r"^[a-z][a-z0-9-]{1,30}$") LIMITS = ("max_conns", "ops_per_sec", "bytes_per_sec") @@ -184,6 +186,53 @@ def rotate(self, name): self.acl.set_user(name, password) return {"name": name, "password": password, "note": "Shown once. Old password stops working now."} + def open_browser(self): + """Read-only connection. Tests inject browser_conn; otherwise the admin user is used.""" + factory = getattr(self, "browser_conn", None) + if factory is not None: + return factory() + import cmo_browse + host, port = self.acl.addr + return cmo_browse.Conn(host, port, self.acl.user, self.acl.password) + + def keys(self, name, path): + """SCAN or one key, only under ':'. Writes are refused by the browser connection.""" + self.get(name) + import cmo_browse + q = parse_qs(urlsplit(path).query) + + def one(k, default=None): + v = q.get(k) + return v[0] if v else default + + match = one("match", "*") + if not isinstance(match, str) or len(match) > 128: + raise ApiError(400, "match is too long") + key = one("key") + cursor = one("cursor", "0") + conn = self.open_browser() + try: + if key is not None: + if len(key) > 512: + raise ApiError(400, "key is too long") + try: + row = cmo_browse.show(conn, name, key) + except PermissionError: + raise ApiError(403, "key outside this database") + if row is None: + raise ApiError(404, "no such key") + return row + if not str(cursor).isdigit() or len(str(cursor)) > 20: + raise ApiError(400, "cursor must be a number") + return cmo_browse.scan(conn, name, str(cursor), match) + except ValueError as e: + raise ApiError(400, str(e)) + finally: + try: + conn.close() + except OSError: + pass + OPENAPI = { "openapi": "3.0.3", @@ -198,10 +247,28 @@ def rotate(self, name): "/v1/databases/{name}": {"get": {"summary": "Get one"}, "patch": {"summary": "Change limits"}, "delete": {"summary": "Delete (keys are kept)"}}, "/v1/databases/{name}/rotate-password": {"post": {"summary": "New password, returned once"}}, + "/v1/databases/{name}/keys": {"get": {"summary": "Read-only keys under the database prefix"}}, }, } +_GATE = None + + +def _gate(): + """Load site/gate.py by path so this tool does not depend on the process cwd.""" + global _GATE + if _GATE is not None: + return _GATE + import importlib.util + path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "site", "gate.py")) + spec = importlib.util.spec_from_file_location("cmo_site_gate", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + _GATE = mod + return mod + + def make_handler(mgr, token): class H(http.server.BaseHTTPRequestHandler): server_version = "cmo-mgmt" @@ -218,19 +285,6 @@ def _send(self, status, obj): self.end_headers() self.wfile.write(body) - def _console(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "console.html"), "rb") as f: - body = f.read() - self.send_response(200) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.send_header("Content-Security-Policy", - "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'") - self.send_header("X-Frame-Options", "DENY") - self.send_header("Referrer-Policy", "no-referrer") - self.end_headers() - self.wfile.write(body) - def _body(self): n = int(self.headers.get("Content-Length") or 0) if n > 65536: @@ -244,13 +298,18 @@ def _body(self): return b def _route(self, method): - if method == "GET" and self.path.split("?")[0] in ("/", "/console"): - return self._console() + gate = _gate() + if gate.try_public(self, mgr, token, method): + return + if not self.path.split("?")[0].startswith("/v1/"): + return self._send(404, {"error": "not found"}) try: auth = self.headers.get("Authorization", "") if not hmac.compare_digest(auth.encode(), ("Bearer " + token).encode()): raise ApiError(401, "unauthorized") p = self.path.split("?")[0].rstrip("/") + if p == "/v1/agent" and method == "POST": + return self._send(200, gate.agent_run(mgr, self._body().get("message"))) parts = p.split("/")[1:] if p == "/v1/openapi.json" and method == "GET": return self._send(200, OPENAPI) @@ -269,11 +328,15 @@ def _route(self, method): return self._send(200, mgr.update(n, self._body())) if method == "DELETE": return self._send(200, mgr.delete(n)) + elif len(parts) == 4 and parts[3] == "keys" and method == "GET": + return self._send(200, mgr.keys(parts[2], self.path)) elif len(parts) == 4 and parts[3] == "rotate-password" and method == "POST": return self._send(200, mgr.rotate(parts[2])) raise ApiError(405 if len(parts) <= 4 else 404, "method not allowed") except ApiError as e: self._send(e.status, {"error": str(e)}) + except ValueError as e: + self._send(400, {"error": str(e)}) except OSError: self._send(502, {"error": "server unreachable"}) diff --git a/tools/mgmt/console.html b/tools/mgmt/console.html deleted file mode 100644 index f13f2242bff..00000000000 --- a/tools/mgmt/console.html +++ /dev/null @@ -1,79 +0,0 @@ - - -Cache Me Outside console - -

Cache Me Outside

-

-

-
-
- - - -
NamePortOps/sConns
- diff --git a/tools/mgmt/test_cmo_mgmt.py b/tools/mgmt/test_cmo_mgmt.py index a3f0b722256..54dee36cf5d 100644 --- a/tools/mgmt/test_cmo_mgmt.py +++ b/tools/mgmt/test_cmo_mgmt.py @@ -75,10 +75,17 @@ def test_console_served_without_token_with_csp(self): import urllib.request with urllib.request.urlopen(self.base + "/") as r: html = r.read().decode() + csp = r.headers["Content-Security-Policy"] self.assertEqual(r.headers["X-Frame-Options"], "DENY") - self.assertIn("connect-src 'self'", r.headers["Content-Security-Policy"]) - self.assertIn("Create database", html) + self.assertIn("connect-src 'self'", csp) + self.assertNotIn("unsafe-inline", csp) + self.assertIn('id="hero"', html) self.assertNotIn("innerHTML", html) + self.assertNotIn("