From 1ac7d3a113df06f89bd5a2ceab61dc45ad7c8200 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 8 Nov 2026 07:54:49 +0000 Subject: [PATCH] Add a Cache Me Outside site and a versioned GoDesk integration. The console key browser returned method not allowed, and the page depended on inline scripts. The site now creates, limits, browses, and deletes databases, and the agent can do those actions without a model. Co-authored-by: Admin --- .github/workflows/cmo-ci.yml | 34 ++++ .gitignore | 4 + INTEGRATION.md | 46 +++++ docs/cache-me-outside/README.md | 2 + docs/cache-me-outside/management-api.md | 13 +- integration/cache-me-outside.js | 43 ++++ integration/cache_me_outside.py | 62 ++++++ integration/manifest.json | 8 + integration/sidecar.py | 170 ++++++++++++++++ integration/test_integration.py | 181 +++++++++++++++++ integration/updater.py | 188 ++++++++++++++++++ site/agent.py | 198 +++++++++++++++++++ site/e2e/package-lock.json | 76 +++++++ site/e2e/package.json | 10 + site/e2e/playwright.config.js | 17 ++ site/e2e/ui.spec.js | 113 +++++++++++ site/gate.py | 124 ++++++++++++ site/memstore.py | 70 +++++++ site/server.py | 96 +++++++++ site/static/console.html | 69 +++++++ site/static/console.js | 253 ++++++++++++++++++++++++ site/static/favicon.svg | 5 + site/static/flow.html | 60 ++++++ site/static/flow.js | 58 ++++++ site/static/index.html | 72 +++++++ site/static/integrate.html | 73 +++++++ site/static/site.js | 86 ++++++++ site/static/style.css | 217 ++++++++++++++++++++ site/test_site.py | 190 ++++++++++++++++++ tools/flowview/flow.js | 3 +- tools/mgmt/cmo_mgmt.py | 95 +++++++-- tools/mgmt/console.html | 79 -------- tools/mgmt/test_cmo_mgmt.py | 11 +- 33 files changed, 2625 insertions(+), 101 deletions(-) create mode 100644 INTEGRATION.md create mode 100644 integration/cache-me-outside.js create mode 100644 integration/cache_me_outside.py create mode 100644 integration/manifest.json create mode 100644 integration/sidecar.py create mode 100644 integration/test_integration.py create mode 100644 integration/updater.py create mode 100644 site/agent.py create mode 100644 site/e2e/package-lock.json create mode 100644 site/e2e/package.json create mode 100644 site/e2e/playwright.config.js create mode 100644 site/e2e/ui.spec.js create mode 100644 site/gate.py create mode 100644 site/memstore.py create mode 100644 site/server.py create mode 100644 site/static/console.html create mode 100644 site/static/console.js create mode 100644 site/static/favicon.svg create mode 100644 site/static/flow.html create mode 100644 site/static/flow.js create mode 100644 site/static/index.html create mode 100644 site/static/integrate.html create mode 100644 site/static/site.js create mode 100644 site/static/style.css create mode 100644 site/test_site.py delete mode 100644 tools/mgmt/console.html diff --git a/.github/workflows/cmo-ci.yml b/.github/workflows/cmo-ci.yml index d251a5f9ad7..d7de7888333 100644 --- a/.github/workflows/cmo-ci.yml +++ b/.github/workflows/cmo-ci.yml @@ -360,3 +360,37 @@ jobs: test "$masters" -eq 4 kubectl exec cmo-cmo-data-0 -- valkey-cli -h 127.0.0.1 -c --user admin -a "$pass" --no-auth-warning \ GET cmo:kind | grep -qx before + + site: + name: Site, integration, and browser + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Unit tests for the site and the management API + run: | + python3 tools/mgmt/test_cmo_mgmt.py + python3 tools/mgmt/test_cmo_browse.py + python3 tools/flowview/test_flow_feed.py + python3 site/test_site.py + python3 integration/test_integration.py + + - name: Install Playwright + working-directory: site/e2e + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Browser tests + working-directory: site/e2e + run: npx playwright test + + - name: Upload screenshots + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: site-screenshots + path: site/e2e/screenshots/*.png + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 058adfa5bde..20830587905 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ deploy/.env deploy/.generated/ deploy/results/ dist/ +site/e2e/node_modules/ +site/e2e/test-results/ +site/e2e/playwright-report/ +site/e2e/screenshots/ diff --git a/INTEGRATION.md b/INTEGRATION.md new file mode 100644 index 00000000000..ba911527767 --- /dev/null +++ b/INTEGRATION.md @@ -0,0 +1,46 @@ +# Adopt Cache Me Outside + +The integration directory is the whole package: a Python client, a browser/Node client, a loopback sidecar, and an updater. Version `1.0.0` is recorded in `integration/manifest.json`. Copy that directory into a GoDesk app, or install a `cmo-integration-vX.Y.Z` release. No token or password belongs in the repository. + +## Python + +```python +import os +from cache_me_outside import Client + +client = Client(os.environ["CMO_API"], os.environ["CMO_ADMIN_TOKEN"]) +print(client.list_databases()) +``` + +## Browser or Node + +```html + + +``` + +`sessionToken` comes from your own sign-in. The script does not contain one. + +## Sidecar + +```bash +CMO_API=http://127.0.0.1:8088 CMO_ADMIN_TOKEN="$CMO_ADMIN_TOKEN" \ + python3 integration/sidecar.py +``` + +The app then calls `GET http://127.0.0.1:8099/health`. The sidecar binds to loopback only. Cache get and set also need `CMO_URL` and `CMO_PREFIX`; a key outside that prefix is refused before a connection opens. + +## Updates + +```bash +python3 integration/updater.py --check +python3 integration/updater.py --apply +``` + +`--apply` reads GitHub releases named `cmo-integration-vX.Y.Z`, downloads `cache-me-outside-integration.tar.gz` and `SHA256SUMS`, and writes files only when the checksum matches. It refuses other hosts, symlinks, and paths that leave the integration directory. When no release is published yet, `--check` reports that and leaves the files alone. diff --git a/docs/cache-me-outside/README.md b/docs/cache-me-outside/README.md index 7855bb3e83b..953195f7f03 100644 --- a/docs/cache-me-outside/README.md +++ b/docs/cache-me-outside/README.md @@ -22,6 +22,8 @@ Valkey and the Redis code it inherited stay under the BSD 3-Clause License. See - [Warm restart](warm-restart.md) - [DRAM + flash hybrid storage](hybrid-storage.md) - [TTL-grouped storage](ttl-segments.md) +- [Management API and site](management-api.md) +- [Adopting the cache from another GoDesk app](../../INTEGRATION.md) ## What this fork changes diff --git a/docs/cache-me-outside/management-api.md b/docs/cache-me-outside/management-api.md index 84310b1234e..1ddde5d0fcf 100644 --- a/docs/cache-me-outside/management-api.md +++ b/docs/cache-me-outside/management-api.md @@ -19,8 +19,15 @@ Not done yet: - **Terraform provider:** the plan also lists one. It needs a separate Go module that calls this API; that is a follow-up. - **Live test:** the API has not been run against a live server. The tests use a fake ACL backend. -## Web console (plan item 61) +`GET /v1/databases/{name}/keys` is the read-only browser. It scans and reads only keys under `:` and refuses a key outside that prefix. A limit of 0 means unlimited. -Open `/` on the management API to use the console. Paste the admin token, give the new database a name, and click **Create database**. The page then shows the connection URL, a `valkey-cli` command, and the key prefix the database must use. The password is shown once. Each database also has a **New password** button. +## Site -The page keeps the token in memory only; nothing is saved in the browser. It sends a strict CSP header and refuses to be framed. It inserts text with textContent only, never innerHTML. +`python3 site/server.py --demo` serves the site on `127.0.0.1` with an in-memory database, a sample key, and the agent. It will not bind a public address. The real process is the same pages on `cmo_mgmt.py` or `site/server.py --config`, with `CMO_ADMIN_TOKEN` supplied in the environment. + +- `/` is the home page. +- `/console` creates, limits, rotates, browses, and deletes databases. The token stays in memory. The password is shown once in a field, not in a link. +- `/flow` is the live data-flow picture. Demo rates are labeled as demo traffic. +- `/integrate` shows the adoption snippets from [INTEGRATION.md](../../INTEGRATION.md). + +The agent runs list, create, delete, rotate, limit, browse, explain, and open-page requests without a model. Free-form text uses `OLLAMA_BASE_URL` only. Pages are scripts and stylesheets, so the CSP does not use `unsafe-inline`. The pages refuse to be framed and insert text with textContent only. diff --git a/integration/cache-me-outside.js b/integration/cache-me-outside.js new file mode 100644 index 00000000000..1abc45acb85 --- /dev/null +++ b/integration/cache-me-outside.js @@ -0,0 +1,43 @@ +/* Cache Me Outside integration, version 1.0.0. No secrets live in this file. */ +(function (root, factory) { + var api = factory(); + if (typeof module === "object" && module.exports) module.exports = api; + else root.CacheMeOutside = api; +})(typeof globalThis !== "undefined" ? globalThis : this, function () { + var VERSION = "1.0.0"; + + function attach(opts) { + if (!opts || !opts.api || !opts.token) throw new Error("api and token are required"); + var api = String(opts.api).replace(/\/$/, ""); + if (!/^https?:\/\//.test(api)) throw new Error("api must be http or https"); + var token = String(opts.token); + + function call(method, path, body) { + return fetch(api + path, { + method: method, + headers: {"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + body: body ? JSON.stringify(body) : undefined + }).then(function (response) { + return response.json().then(function (payload) { + if (!response.ok) throw new Error(payload.error || String(response.status)); + return payload; + }); + }); + } + + return { + version: VERSION, + listDatabases: function () { return call("GET", "/v1/databases").then(function (b) { return b.databases; }); }, + createDatabase: function (name, limits) { + return call("POST", "/v1/databases", Object.assign({name: name}, limits || {})); + }, + rotatePassword: function (name) { return call("POST", "/v1/databases/" + name + "/rotate-password", {}); }, + deleteDatabase: function (name) { return call("DELETE", "/v1/databases/" + name); }, + scanKeys: function (name, match) { + return call("GET", "/v1/databases/" + encodeURIComponent(name) + "/keys?match=" + encodeURIComponent(match || "*")); + } + }; + } + + return {attach: attach, version: VERSION}; +}); diff --git a/integration/cache_me_outside.py b/integration/cache_me_outside.py new file mode 100644 index 00000000000..6a15aa4457e --- /dev/null +++ b/integration/cache_me_outside.py @@ -0,0 +1,62 @@ +"""Small client for the Cache Me Outside management API. Standard library only.""" +import json +import os +import urllib.error +import urllib.request + +_MANIFEST = os.path.join(os.path.dirname(os.path.abspath(__file__)), "manifest.json") + + +def version(): + with open(_MANIFEST) as handle: + return json.load(handle)["version"] + + +class Client: + def __init__(self, api, token, timeout=15): + if not api or not str(api).startswith(("http://", "https://")): + raise ValueError("api must be an http or https URL") + if not token or not isinstance(token, str): + raise ValueError("token is required") + self.api = api.rstrip("/") + self.token = token + self.timeout = timeout + + def _call(self, method, path, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + self.api + path, data=data, method=method, + headers={"Authorization": "Bearer " + self.token, "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + payload = json.load(exc) + raise RuntimeError(payload.get("error") or str(exc.code)) from None + + def list_databases(self): + return self._call("GET", "/v1/databases")["databases"] + + def create_database(self, name, **limits): + body = {"name": name} + body.update(limits) + return self._call("POST", "/v1/databases", body) + + def get_database(self, name): + return self._call("GET", "/v1/databases/" + name) + + def update_limits(self, name, **limits): + return self._call("PATCH", "/v1/databases/" + name, limits) + + def rotate_password(self, name): + return self._call("POST", "/v1/databases/%s/rotate-password" % name, {}) + + def delete_database(self, name): + return self._call("DELETE", "/v1/databases/" + name) + + def scan_keys(self, name, match="*", cursor="0"): + return self._call("GET", "/v1/databases/%s/keys?cursor=%s&match=%s" % (name, cursor, match)) + + def show_key(self, name, key): + from urllib.parse import quote + return self._call("GET", "/v1/databases/%s/keys?key=%s" % (name, quote(key, safe=""))) diff --git a/integration/manifest.json b/integration/manifest.json new file mode 100644 index 00000000000..6dad9193a27 --- /dev/null +++ b/integration/manifest.json @@ -0,0 +1,8 @@ +{ + "name": "cache-me-outside", + "version": "1.0.0", + "api": "v1", + "repository": "https://github.com/GoDeskio/Cache-Me-Outside", + "release_prefix": "cmo-integration-v", + "asset": "cache-me-outside-integration.tar.gz" +} diff --git a/integration/sidecar.py b/integration/sidecar.py new file mode 100644 index 00000000000..15e0e9b0a96 --- /dev/null +++ b/integration/sidecar.py @@ -0,0 +1,170 @@ +"""Loopback sidecar so another GoDesk app can call Cache Me Outside without linking a library. + + CMO_API=http://127.0.0.1:8088 CMO_ADMIN_TOKEN=... python3 integration/sidecar.py + +Cache reads and writes need CMO_URL and CMO_PREFIX as well. Keys outside the +prefix are refused before a connection is opened. The process will not bind +a wildcard address. +""" +import json +import os +import socket +import sys + +import cache_me_outside + +MAX_VALUE = 65536 +COMMANDS = {"GET", "SET", "DEL"} + + +def check_key(key, prefix): + if not isinstance(key, str) or not prefix or not key.startswith(prefix): + raise ValueError("key is outside the prefix") + if len(key) > 512 or "\r" in key or "\n" in key or "\x00" in key: + raise ValueError("key refused") + return key + + +def _send(sock, args): + enc = [a if isinstance(a, bytes) else str(a).encode() for a in args] + sock.sendall(b"*%d\r\n" % len(enc) + b"".join(b"$%d\r\n%s\r\n" % (len(x), x) for x in enc)) + + +def _read(stream): + line = stream.readline() + if not line: + raise ConnectionError("closed") + if line[:1] == b"-": + raise RuntimeError(line[1:-2].decode(errors="replace")) + if line[:1] == b"$": + n = int(line[1:]) + if n < 0: + return None + return stream.read(n + 2)[:-2] + if line[:1] == b"+": + return line[1:-2] + if line[:1] == b":": + return int(line[1:]) + raise RuntimeError("unexpected reply") + + +def cache_call(url, prefix, args): + from urllib.parse import urlsplit + if args[0] not in COMMANDS: + raise ValueError("command refused") + check_key(args[1], prefix) + if args[0] == "SET" and len(args) > 2 and len(str(args[2]).encode()) > MAX_VALUE: + raise ValueError("value is too large") + parts = urlsplit(url) + if parts.scheme not in ("redis", "rediss", "valkey", "valkeys") or not parts.hostname: + raise ValueError("CMO_URL must be a redis or valkey URL") + if parts.scheme in ("rediss", "valkeys"): + raise ValueError("the sidecar speaks plain TCP; terminate TLS in front of the cache") + sock = socket.create_connection((parts.hostname, parts.port or 6379), 5) + stream = sock.makefile("rb") + try: + if parts.password is not None: + user = parts.username or "default" + _send(sock, ["AUTH", user, parts.password]) + _read(stream) + _send(sock, args) + return _read(stream) + finally: + stream.close() + sock.close() + + +def make_handler(client, cache_url, prefix): + from http.server import BaseHTTPRequestHandler + + class H(BaseHTTPRequestHandler): + server_version = "cmo-sidecar" + + def log_message(self, *a): + pass + + def _send(self, status, obj): + body = json.dumps(obj).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _body(self): + n = int(self.headers.get("Content-Length") or 0) + if n > MAX_VALUE + 1024: + raise ValueError("body too large") + data = json.loads(self.rfile.read(n) or b"{}") + if not isinstance(data, dict): + raise ValueError("body must be an object") + return data + + def do_GET(self): + if self.path == "/health": + return self._send(200, {"ok": True, "version": cache_me_outside.version()}) + if self.path == "/v1/version": + return self._send(200, {"name": "cache-me-outside", "version": cache_me_outside.version()}) + if self.path.split("?")[0].startswith("/v1/databases"): + if client is None: + return self._send(503, {"error": "set CMO_API and CMO_ADMIN_TOKEN"}) + try: + self._send(200, client._call("GET", self.path)) + except RuntimeError as exc: + self._send(502, {"error": str(exc)}) + return + self._send(404, {"error": "not found"}) + + def do_POST(self): + path = self.path.split("?")[0] + try: + body = self._body() + if path == "/v1/cache/get": + if not cache_url: + return self._send(503, {"error": "set CMO_URL and CMO_PREFIX"}) + value = cache_call(cache_url, prefix, ["GET", body.get("key", "")]) + shown = None if value is None else value.decode("utf-8", "replace") + return self._send(200, {"key": body.get("key"), "value": shown}) + if path == "/v1/cache/set": + if not cache_url: + return self._send(503, {"error": "set CMO_URL and CMO_PREFIX"}) + args = ["SET", body.get("key", ""), body.get("value", "")] + if "ttl" in body: + ttl = body["ttl"] + if isinstance(ttl, bool) or not isinstance(ttl, int) or ttl < 1 or ttl > 8640000: + raise ValueError("ttl must be 1 to 8640000 seconds") + args.extend(["EX", ttl]) + cache_call(cache_url, prefix, args) + return self._send(200, {"stored": True}) + if client and path == "/v1/databases": + return self._send(201, client.create_database(body.get("name", ""), **{ + k: body[k] for k in ("max_conns", "ops_per_sec", "bytes_per_sec") if k in body})) + except ValueError as exc: + return self._send(400, {"error": str(exc)}) + except (OSError, RuntimeError) as exc: + return self._send(502, {"error": str(exc)}) + self._send(404, {"error": "not found"}) + + return H + + +def main(argv=None): + import argparse + from http.server import ThreadingHTTPServer + ap = argparse.ArgumentParser(description="Cache Me Outside sidecar") + ap.add_argument("--listen", default="127.0.0.1:8099") + a = ap.parse_args(argv) + host, _, port = a.listen.rpartition(":") + if host not in ("127.0.0.1", "::1"): + ap.error("the sidecar listens on 127.0.0.1 only") + api = os.environ.get("CMO_API", "") + token = os.environ.get("CMO_ADMIN_TOKEN", "") + client = cache_me_outside.Client(api, token) if api and token else None + srv = ThreadingHTTPServer((host, int(port)), make_handler(client, os.environ.get("CMO_URL", ""), os.environ.get("CMO_PREFIX", ""))) + print("sidecar on http://%s" % a.listen, file=sys.stderr) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/integration/test_integration.py b/integration/test_integration.py new file mode 100644 index 00000000000..e299a8396b6 --- /dev/null +++ b/integration/test_integration.py @@ -0,0 +1,181 @@ +import hashlib +import io +import json +import os +import socket +import subprocess +import sys +import tarfile +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import cache_me_outside +import sidecar +import updater + +ASSET = updater.ASSET + + +def archive(files): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as handle: + for name, data in files.items(): + if isinstance(data, tuple): + info, payload = data + else: + info = tarfile.TarInfo(name) + info.size = len(data) + payload = data + handle.addfile(info, io.BytesIO(payload)) + return buf.getvalue() + + +class Release(BaseHTTPRequestHandler): + blob = b"" + sums = b"" + releases = [] + + def do_GET(self): + if self.path.startswith("/releases"): + body = json.dumps(self.releases).encode() + elif self.path == "/blob": + body = type(self).blob + elif self.path == "/sums": + body = type(self).sums + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + +def serve(handler): + httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd + + +class Updater(unittest.TestCase): + def setUp(self): + self.dest = tempfile.mkdtemp() + with open(os.path.join(self.dest, "manifest.json"), "w") as handle: + json.dump({"version": "1.0.0"}, handle) + manifest = json.dumps({"name": "cache-me-outside", "version": "1.1.0", "api": "v1"}).encode() + self.blob = archive({"pkg/manifest.json": manifest, "pkg/cache_me_outside.py": b"print(1)\n"}) + Release.blob = self.blob + Release.sums = (hashlib.sha256(self.blob).hexdigest() + " " + ASSET + "\n").encode() + Release.releases = [{ + "tag_name": "cmo-integration-v1.1.0", + "assets": [ + {"name": ASSET, "browser_download_url": ""}, + {"name": "SHA256SUMS", "browser_download_url": ""}, + ], + }] + self.httpd = serve(Release) + base = "http://127.0.0.1:%d" % self.httpd.server_address[1] + Release.releases[0]["assets"][0]["browser_download_url"] = base + "/blob" + Release.releases[0]["assets"][1]["browser_download_url"] = base + "/sums" + self.api = base + "/releases" + + def tearDown(self): + self.httpd.shutdown() + self.httpd.server_close() + + def test_apply_checks_the_hash_and_rejects_traversal(self): + result = updater.apply(self.api, self.dest) + self.assertEqual(result["status"], "applied") + self.assertEqual(updater.local_version(self.dest), "1.1.0") + self.assertTrue(os.path.isfile(os.path.join(self.dest, "cache_me_outside.py"))) + Release.sums = (b"0" * 64 + b" " + ASSET.encode() + b"\n") + with open(os.path.join(self.dest, "manifest.json"), "w") as handle: + json.dump({"version": "1.0.0"}, handle) + with self.assertRaises(ValueError): + updater.apply(self.api, self.dest) + self.assertEqual(updater.local_version(self.dest), "1.0.0") + bad = archive({"../manifest.json": b"{}"}) + with self.assertRaises(ValueError): + updater.unpack(bad) + link = tarfile.TarInfo("manifest.json") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + with self.assertRaises(ValueError): + updater.unpack(archive({"manifest.json": (link, b"")})) + + def test_no_release_is_not_a_failure(self): + Release.releases = [] + found = updater.check(self.api, self.dest) + self.assertEqual(found["status"], "none") + self.assertEqual(found["version"], "1.0.0") + + def test_file_url_is_refused(self): + found = updater.check("file:///tmp/releases", self.dest) + self.assertEqual(found["status"], "unreachable") + + +class ClientAndSidecar(unittest.TestCase): + def test_package_version_and_node_client(self): + self.assertEqual(cache_me_outside.version(), "1.0.0") + with self.assertRaises(ValueError): + cache_me_outside.Client("http://127.0.0.1:9", "") + script = ( + "const c=require('./cache-me-outside.js');" + "if(c.version!=='1.0.0') process.exit(1);" + "try{c.attach({});process.exit(1)}catch(e){}" + ) + subprocess.check_call(["node", "-e", script], cwd=HERE) + + def test_prefix_is_enforced_before_connect(self): + with self.assertRaises(ValueError): + sidecar.cache_call("redis://127.0.0.1:1", "app:", ["GET", "other:x"]) + with self.assertRaises(ValueError): + sidecar.cache_call("redis://127.0.0.1:1", "app:", ["FLUSHALL", "app:x"]) + with self.assertRaises(ValueError): + sidecar.cache_call("rediss://127.0.0.1:1", "app:", ["GET", "app:x"]) + + def test_get_roundtrip_and_health(self): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(1) + + def once(): + conn, _addr = sock.accept() + conn.recv(1024) + conn.sendall(b"$5\r\nhello\r\n") + conn.close() + + threading.Thread(target=once, daemon=True).start() + port = sock.getsockname()[1] + self.assertEqual(sidecar.cache_call("redis://127.0.0.1:%d" % port, "app:", ["GET", "app:k"]), b"hello") + sock.close() + httpd = serve(sidecar.make_handler(None, "", "")) + try: + with urllib.request.urlopen("http://127.0.0.1:%d/health" % httpd.server_address[1]) as response: + self.assertEqual(json.load(response)["version"], "1.0.0") + req = urllib.request.Request( + "http://127.0.0.1:%d/v1/cache/get" % httpd.server_address[1], + data=b'{"key":"app:k"}', headers={"Content-Type": "application/json"}) + with self.assertRaises(urllib.error.HTTPError) as caught: + urllib.request.urlopen(req) + self.assertEqual(caught.exception.code, 503) + finally: + httpd.shutdown() + httpd.server_close() + + def test_sidecar_refuses_a_public_bind(self): + with self.assertRaises(SystemExit): + sidecar.main(["--listen", "0.0.0.0:8099"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/integration/updater.py b/integration/updater.py new file mode 100644 index 00000000000..095df950682 --- /dev/null +++ b/integration/updater.py @@ -0,0 +1,188 @@ +"""Update the integration directory from a published GitHub release. + + python3 integration/updater.py --check + python3 integration/updater.py --apply + +A release tag looks like cmo-integration-v1.2.3 and carries +cache-me-outside-integration.tar.gz plus SHA256SUMS. The archive is applied +only after the checksum matches, and only the known filenames are written. +Nothing in this file is a secret, and a missing release is not an error. +""" +import hashlib +import io +import json +import os +import tarfile +import urllib.request +from urllib.parse import urlsplit + +HERE = os.path.dirname(os.path.abspath(__file__)) +ASSET = "cache-me-outside-integration.tar.gz" +ALLOW = {"manifest.json", "cache_me_outside.py", "cache-me-outside.js", "sidecar.py", "updater.py"} +HOSTS = ("api.github.com", "github.com", "objects.githubusercontent.com", "release-assets.githubusercontent.com") +DEFAULT_API = "https://api.github.com/repos/GoDeskio/Cache-Me-Outside/releases?per_page=20" + + +def local_version(path=None): + with open(os.path.join(path or HERE, "manifest.json")) as handle: + return json.load(handle)["version"] + + +def semver(text): + parts = text.split(".") + if len(parts) != 3 or any(not part.isdigit() for part in parts): + raise ValueError("bad version %r" % text) + return tuple(int(part) for part in parts) + + +def allow_for(api): + base = urlsplit(api) + + def allow(url): + got = urlsplit(url) + if base.hostname in ("127.0.0.1", "localhost"): + return got.scheme == base.scheme and got.netloc == base.netloc + return got.scheme == "https" and got.hostname in HOSTS + return allow + + +class _Guard(urllib.request.HTTPRedirectHandler): + def __init__(self, allow): + self.allow = allow + + def redirect_request(self, req, fp, code, msg, headers, newurl): + if not self.allow(newurl): + raise ValueError("redirect refused") + return urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + + +def _get(url, allow, limit=2000000): + if not allow(url): + raise ValueError("refusing " + url) + opener = urllib.request.build_opener(_Guard(allow)) + with opener.open(url, timeout=20) as response: + blob = response.read(limit + 1) + if len(blob) > limit: + raise ValueError("download is too large") + return blob + + +def choose(releases, prefix="cmo-integration-v"): + best = None + best_key = None + for release in releases: + tag = release.get("tag_name") or "" + if not tag.startswith(prefix): + continue + key = semver(tag[len(prefix):]) + if best_key is None or key > best_key: + best, best_key = release, key + return best + + +def checksum(text, filename): + for line in text.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[-1].endswith(filename) and len(parts[0]) == 64: + return parts[0] + raise ValueError("checksum missing") + + +def unpack(blob): + archive = tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") + chosen = {} + for member in archive.getmembers(): + if member.isdir(): + continue + parts = member.name.split("/") + if member.issym() or member.islnk() or not member.isfile() or any(part in ("", "..") for part in parts): + raise ValueError("refusing %s" % member.name) + base = parts[-1] + if base not in ALLOW or base in chosen: + raise ValueError("unexpected file %s" % member.name) + data = archive.extractfile(member).read() + if len(data) > 200000: + raise ValueError("file too large") + chosen[base] = data + if "manifest.json" not in chosen: + raise ValueError("manifest missing") + json.loads(chosen["manifest.json"].decode()) + return chosen + + +def apply_files(files, dest): + os.makedirs(dest, exist_ok=True) + for name, data in files.items(): + target = os.path.join(dest, name) + if os.path.dirname(os.path.abspath(target)) != os.path.abspath(dest): + raise ValueError("refusing to write outside the integration directory") + tmp = target + ".tmp" + with open(tmp, "wb") as handle: + handle.write(data) + os.replace(tmp, target) + + +def check(api=DEFAULT_API, dest=None): + dest = dest or HERE + current = semver(local_version(dest)) + allow = allow_for(api) + try: + releases = json.loads(_get(api, allow).decode()) + except Exception as exc: + return {"status": "unreachable", "version": "%d.%d.%d" % current, "detail": str(exc)} + if not isinstance(releases, list): + return {"status": "unreachable", "version": "%d.%d.%d" % current, "detail": "unexpected release list"} + release = choose(releases) + if not release: + return {"status": "none", "version": "%d.%d.%d" % current} + remote = semver(release["tag_name"][len("cmo-integration-v"):]) + return { + "status": "update" if remote > current else "current", + "version": "%d.%d.%d" % current, + "remote": "%d.%d.%d" % remote, + "release": release, + } + + +def apply(api=DEFAULT_API, dest=None): + dest = dest or HERE + found = check(api, dest) + if found["status"] != "update": + return found + allow = allow_for(api) + assets = {item.get("name"): item.get("browser_download_url") for item in found["release"].get("assets") or []} + if ASSET not in assets or "SHA256SUMS" not in assets: + raise ValueError("release is missing the archive or SHA256SUMS") + blob = _get(assets[ASSET], allow) + digest = hashlib.sha256(blob).hexdigest() + if digest != checksum(_get(assets["SHA256SUMS"], allow).decode(), ASSET): + raise ValueError("checksum did not match; nothing was written") + apply_files(unpack(blob), dest) + found["status"] = "applied" + found["version"] = local_version(dest) + return found + + +def main(argv=None): + import argparse + import sys + ap = argparse.ArgumentParser(description="Cache Me Outside integration updater") + ap.add_argument("--check", action="store_true") + ap.add_argument("--apply", action="store_true") + ap.add_argument("--api", default=os.environ.get("CMO_RELEASE_API", DEFAULT_API)) + ap.add_argument("--dest", default=HERE) + a = ap.parse_args(argv) + if a.apply == a.check: + ap.error("choose --check or --apply") + try: + result = apply(a.api, a.dest) if a.apply else check(a.api, a.dest) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + print(result["status"] + " " + result.get("remote", result["version"])) + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/site/agent.py b/site/agent.py new file mode 100644 index 00000000000..ced8a9a1a13 --- /dev/null +++ b/site/agent.py @@ -0,0 +1,198 @@ +"""Deterministic site agent. OLLAMA_BASE_URL is optional and never required. + +The planner only recognizes a fixed sentence list. Anything else is sent to a +local Ollama server when that URL is set. The model may only ask for the same +actions the planner can already run. Unknown actions are dropped. +""" +import json +import os +import re +import urllib.request +from urllib.parse import urlsplit + +NAME = re.compile(r"^[a-z][a-z0-9-]{1,30}$") +NAV = {"home": "/", "console": "/console", "flow": "/flow", "integrate": "/integrate"} +TOPICS = { + "tenants": "Each database gets its own port, its own password, and keys that must start with its name.", + "browser": "The key browser is read-only. It only returns keys under that database's prefix.", + "flow": "The flow page shows requests, near-cache hits, cache hits, misses, and the database queries those misses become.", + "warm": "Warm restart can reload a checkpoint after a clean shutdown when RDB and AOF did not. It stays off until it is configured.", + "hybrid": "Hybrid storage can spill large strings to a flash log. It stays off until it is configured.", + "ttl": "TTL segments delete keys that expire in the same time bucket. They stay off until configured.", + "integration": "Other GoDesk apps adopt the integration directory. The token comes from the environment, never from the repository.", +} +HELP = ( + "I can list databases, create database , delete database , " + "rotate password , set ops for to , browse , " + "explain tenants, or open home, console, flow, or integrate. " + "Those run here with no model. Free-form chat uses OLLAMA_BASE_URL and does not call a paid provider." +) +ALLOWED = {"say", "navigate", "list", "create", "delete", "rotate", "limits", "browse", "explain"} +LIMITS = {"ops": "ops_per_sec", "connections": "max_conns", "bytes": "bytes_per_sec"} +SYSTEM = ( + "You control Cache Me Outside. Reply with one JSON object and no other text. " + 'Allowed actions: {"action":"say","text":"..."}, ' + '{"action":"navigate","href":"/"}, {"action":"list"}, ' + '{"action":"create","name":"example"}, {"action":"delete","name":"example"}, ' + '{"action":"rotate","name":"example"}, ' + '{"action":"limits","name":"example","field":"ops_per_sec","value":10}, ' + '{"action":"browse","name":"example"}, {"action":"explain","topic":"tenants"}. ' + "href is one of /, /console, /flow, /integrate. " + "field is ops_per_sec, max_conns, or bytes_per_sec. " + "topic is tenants, browser, flow, warm, hybrid, ttl, or integration. " + "Do not request any other action." +) + + +def plan(message): + low = " ".join(message.lower().split()) + if low in ("help", "what can you do", "?"): + return {"action": "say", "text": HELP} + m = re.match(r"(?:please )?open (?:the )?(home|console|flow|integrate)(?: page)?$", low) + if m: + return {"action": "navigate", "href": NAV[m.group(1)], "text": "Opening %s." % m.group(1)} + if low in ("list databases", "show databases"): + return {"action": "list"} + m = re.match(r"create database ([a-z0-9-]+)$", low) + if m: + return {"action": "create", "name": m.group(1)} + m = re.match(r"delete database ([a-z0-9-]+)$", low) + if m: + return {"action": "delete", "name": m.group(1)} + m = re.match(r"(?:rotate password|new password for) ([a-z0-9-]+)$", low) + if m: + return {"action": "rotate", "name": m.group(1)} + m = re.match(r"set (ops|connections|bytes) for ([a-z0-9-]+) to (\d+)$", low) + if m: + return {"action": "limits", "name": m.group(2), "field": LIMITS[m.group(1)], "value": int(m.group(3))} + m = re.match(r"(?:browse|show keys in) ([a-z0-9-]+)$", low) + if m: + return {"action": "browse", "name": m.group(1)} + m = re.match(r"explain ([a-z]+)$", low) + if m: + return {"action": "explain", "topic": m.group(1)} + return None + + +def _name(obj): + name = obj.get("name", "") + if not isinstance(name, str) or not NAME.match(name): + raise ValueError("name: 2-31 chars, lowercase letters, digits, dashes, starting with a letter") + return name + + +def perform(mgr, obj): + action = obj.get("action") + if action == "say": + text = obj.get("text", "") + if not isinstance(text, str): + raise ValueError("text must be a string") + return {"reply": text[:2000], "actions": []} + if action == "navigate": + href = obj.get("href") + if href not in NAV.values(): + raise ValueError("that page is not on this site") + return {"reply": obj.get("text") or "Opening that page.", "actions": [{"op": "navigate", "href": href}]} + if action == "explain": + topic = obj.get("topic") + if topic not in TOPICS: + raise ValueError("I can explain tenants, browser, flow, warm, hybrid, ttl, or integration.") + return {"reply": TOPICS[topic], "actions": []} + if action == "list": + rows = mgr.list() + names = ", ".join(d["name"] for d in rows) or "none yet" + return {"reply": "Databases: %s." % names, "actions": [{"op": "refresh"}]} + if action == "create": + out = mgr.create({"name": _name(obj)}) + reply = "Database %s is on port %s. Password, shown once: %s." % (out["name"], out["port"], out["password"]) + if out.get("sample_key"): + reply += " Sample key %s was added for the local demo." % out["sample_key"] + return {"reply": reply, "actions": [{"op": "refresh"}]} + if action == "delete": + out = mgr.delete(_name(obj)) + return {"reply": "Deleted %s. Keys under that prefix were kept." % out["deleted"], "actions": [{"op": "refresh"}]} + if action == "rotate": + out = mgr.rotate(_name(obj)) + return {"reply": "New password for %s, shown once: %s." % (out["name"], out["password"]), + "actions": [{"op": "refresh"}]} + if action == "limits": + field = obj.get("field") + value = obj.get("value") + if field not in ("ops_per_sec", "max_conns", "bytes_per_sec"): + raise ValueError("limit must be ops, connections, or bytes") + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("limit must be a non-negative integer") + out = mgr.update(_name(obj), {field: value}) + shown = "unlimited" if out[field] == 0 else str(out[field]) + return {"reply": "%s for %s is now %s." % (field, out["name"], shown), "actions": [{"op": "refresh"}]} + if action == "browse": + name = _name(obj) + found = mgr.keys(name, "/v1/databases/%s/keys" % name) + keys = found.get("keys") or [] + shown = ", ".join(keys[:20]) or "no keys" + return {"reply": "Keys in %s: %s." % (name, shown), + "actions": [{"op": "browse", "name": name}, {"op": "refresh"}]} + raise ValueError("I can't do that") + + +def parse_model(raw): + if not isinstance(raw, str): + return None + start, end = raw.find("{"), raw.rfind("}") + if start < 0 or end <= start: + return None + try: + obj = json.loads(raw[start:end + 1]) + except ValueError: + return None + if not isinstance(obj, dict) or obj.get("action") not in ALLOWED: + return None + return obj + + +class _RefuseRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("redirect refused") + + +def ask_ollama(message): + base = os.environ.get("OLLAMA_BASE_URL", "").strip().rstrip("/") + parts = urlsplit(base) + if parts.scheme not in ("http", "https") or not parts.hostname or parts.username or parts.password: + raise ValueError("OLLAMA_BASE_URL must be an http or https URL without a user") + model = os.environ.get("OLLAMA_MODEL", "llama3.2") + if not re.match(r"^[A-Za-z0-9._:-]{1,80}$", model): + raise ValueError("OLLAMA_MODEL is not a model name") + payload = json.dumps({ + "model": model, + "stream": False, + "messages": [ + {"role": "system", "content": SYSTEM}, + {"role": "user", "content": message}, + ], + }).encode() + opener = urllib.request.build_opener(_RefuseRedirect) + req = urllib.request.Request(base + "/api/chat", data=payload, headers={"Content-Type": "application/json"}) + with opener.open(req, timeout=8) as r: + data = json.loads(r.read(65536).decode()) + return data.get("message", {}).get("content", "") + + +def run(mgr, message, ask=None): + if not isinstance(message, str): + raise ValueError("message must be a string") + message = message.strip() + if not message or len(message) > 500: + raise ValueError("message must be 1-500 characters") + planned = plan(message) + if planned is None: + if ask is None and not os.environ.get("OLLAMA_BASE_URL"): + return {"reply": HELP, "actions": []} + try: + raw = (ask or ask_ollama)(message) + except Exception: + return {"reply": "The local model did not answer. " + HELP, "actions": []} + planned = parse_model(raw) + if planned is None: + return {"reply": "The local model did not ask for something I can do. " + HELP, "actions": []} + return perform(mgr, planned) diff --git a/site/e2e/package-lock.json b/site/e2e/package-lock.json new file mode 100644 index 00000000000..9ab5364448c --- /dev/null +++ b/site/e2e/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "cmo-site-e2e", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cmo-site-e2e", + "devDependencies": { + "@playwright/test": "1.55.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.55.1.tgz", + "integrity": "sha512-IVAh/nOJaw6W9g+RJVlIQJ6gSiER+ae6mKQ5CX1bERzQgbC1VSeBlwdvczT7pxb0GWiyrxH4TGKbMfDb4Sq/ig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.55.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.1.tgz", + "integrity": "sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.55.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.55.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.1.tgz", + "integrity": "sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/site/e2e/package.json b/site/e2e/package.json new file mode 100644 index 00000000000..011d8e14bba --- /dev/null +++ b/site/e2e/package.json @@ -0,0 +1,10 @@ +{ + "name": "cmo-site-e2e", + "private": true, + "scripts": { + "test": "playwright test" + }, + "devDependencies": { + "@playwright/test": "1.55.1" + } +} diff --git a/site/e2e/playwright.config.js b/site/e2e/playwright.config.js new file mode 100644 index 00000000000..65de02970e0 --- /dev/null +++ b/site/e2e/playwright.config.js @@ -0,0 +1,17 @@ +const { defineConfig } = require("@playwright/test"); + +module.exports = defineConfig({ + testDir: ".", + timeout: 30000, + workers: 1, + use: { + baseURL: "http://127.0.0.1:8091", + ...(process.env.PW_CHANNEL ? { channel: process.env.PW_CHANNEL } : {}), + }, + webServer: { + command: "python3 ../server.py --demo --listen 127.0.0.1:8091", + url: "http://127.0.0.1:8091/", + reuseExistingServer: false, + timeout: 20000, + }, +}); diff --git a/site/e2e/ui.spec.js b/site/e2e/ui.spec.js new file mode 100644 index 00000000000..aec9c1dab37 --- /dev/null +++ b/site/e2e/ui.spec.js @@ -0,0 +1,113 @@ +const { test, expect } = require("@playwright/test"); +const fs = require("fs"); + +fs.mkdirSync("screenshots", { recursive: true }); + +test("home, console, flow, and integrate", async ({ page }) => { + page.on("dialog", (dialog) => dialog.accept()); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + + await page.setViewportSize({ width: 1440, height: 900 }); + await page.goto("/"); + await expect(page.locator("#hero h1")).toHaveText("The cache that stays on your desk."); + await page.screenshot({ path: "screenshots/home-1440.png" }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.evaluate(() => window.scrollTo(0, 0)); + await page.screenshot({ path: "screenshots/home-390.png" }); + await page.setViewportSize({ width: 1440, height: 900 }); + await expect(page.getByRole("link", { name: "GitGoat" })).toHaveAttribute("href", "https://gitgoat.io"); + await expect(page.getByRole("link", { name: "RailYard" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Portkyl" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Komboe" })).toBeVisible(); + await expect(page.getByRole("link", { name: "Cerebro" })).toBeVisible(); + await page.mouse.move(180, 120); + await expect.poll(() => page.evaluate(() => getComputedStyle(document.documentElement).getPropertyValue("--mx").trim())).toBe("180px"); + await page.getByRole("button", { name: "Explain tenants" }).click(); + await expect(page.locator("#agent-out")).toContainText("password"); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.getByRole("button", { name: "Menu" }).click(); + await expect(page.locator("#nav")).toBeVisible(); + await page.locator("#nav").getByRole("link", { name: "Console", exact: true }).click(); + + await page.setViewportSize({ width: 1440, height: 900 }); + await expect(page.locator("#demo-banner")).toBeVisible(); + await page.locator("#name").fill("A"); + await page.getByRole("button", { name: "Create database" }).click(); + await expect(page.locator("#err")).toContainText("Name"); + await page.locator("#name").fill("acme"); + await page.getByRole("button", { name: "Create database" }).click(); + await expect(page.locator("#result")).toContainText("shown once"); + await expect(page.locator("#result .url")).toHaveValue(/redis:\/\/acme:/); + await page.getByRole("button", { name: "Copy URL" }).click(); + await expect(page.locator("#dbs")).toContainText("acme"); + await expect(page.locator("#dbs")).toContainText("unlimited"); + + await page.getByRole("button", { name: "Browse" }).click(); + await page.getByRole("button", { name: "acme:welcome" }).click(); + await page.getByRole("button", { name: "Show value" }).click(); + await expect(page.locator("#kv")).toContainText("hello from Cache Me Outside"); + await page.getByRole("button", { name: "Hide value" }).click(); + await expect(page.locator("#kv pre")).toHaveText("hidden"); + await page.locator("#bmatch").fill("nope*"); + await page.getByRole("button", { name: "Search keys" }).click(); + await expect(page.locator("#keys")).toContainText("No keys match."); + await page.locator("#bmatch").fill("*"); + await page.getByRole("button", { name: "Search keys" }).click(); + await expect(page.locator("#keys")).toContainText("acme:welcome"); + + await page.getByRole("button", { name: "Limits" }).click(); + await page.locator("#lim-ops").fill("25"); + await page.getByRole("button", { name: "Save limits" }).click(); + await expect(page.locator("#dbs")).toContainText("ops/s 25"); + await page.getByRole("button", { name: "New password" }).click(); + await expect(page.locator("#result")).toContainText("shown once"); + await page.getByRole("button", { name: "List databases" }).click(); + await expect(page.locator("#agent-out")).toContainText("acme"); + await page.locator("#agent-q").fill("create database beta"); + await page.getByRole("button", { name: "Ask" }).click(); + await expect(page.locator("#dbs")).toContainText("beta"); + await page.getByRole("button", { name: "Create demo" }).click(); + await expect(page.locator("#dbs")).toContainText("demo"); + await page.locator("article", { hasText: "acme" }).getByRole("button", { name: "Delete" }).click(); + await expect(page.locator("#dbs")).not.toContainText("Port 7001"); + await page.evaluate(() => { + document.getElementById("result").hidden = true; + document.getElementById("agent-out").textContent = ""; + }); + await page.locator("#dbs").scrollIntoViewIfNeeded(); + await page.screenshot({ path: "screenshots/console-1440.png" }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: "screenshots/console-390.png" }); + + await page.setViewportSize({ width: 1440, height: 900 }); + await page.locator("#agent-q").fill("open flow"); + await page.getByRole("button", { name: "Ask" }).click(); + await expect(page).toHaveURL(/\/flow$/); + await expect(page.locator("#flow-note")).toBeVisible(); + await expect(page.locator("#stats")).toContainText("req/s"); + await expect.poll(() => page.locator("#dots circle").count()).toBeGreaterThan(0); + await page.screenshot({ path: "screenshots/flow-1440.png" }); + + await page.getByRole("link", { name: "Integrate" }).click(); + await page.getByRole("button", { name: "Show version" }).click(); + await expect(page.locator("#version-out")).toHaveText("cache-me-outside 1.0.0 (api v1)"); + await page.getByRole("button", { name: "Explain integration" }).click(); + await expect(page.locator("#agent-out")).toContainText("environment"); + expect(errors).toEqual([]); +}); + +test("reduced motion keeps the flow still", async ({ browser }) => { + const context = await browser.newContext({ reducedMotion: "reduce", viewport: { width: 390, height: 844 } }); + const page = await context.newPage(); + await page.goto("/flow"); + await page.mouse.move(40, 40); + await page.waitForTimeout(900); + await expect(page.locator("#dots circle")).toHaveCount(0); + await expect(page.locator("#stats")).toContainText("req/s"); + const mx = await page.evaluate(() => getComputedStyle(document.documentElement).getPropertyValue("--mx").trim()); + expect(mx).not.toBe("40px"); + await page.screenshot({ path: "screenshots/flow-390.png" }); + await context.close(); +}); diff --git a/site/gate.py b/site/gate.py new file mode 100644 index 00000000000..d8439896ac0 --- /dev/null +++ b/site/gate.py @@ -0,0 +1,124 @@ +"""Public pages and unauthenticated product routes for the management server. + +Scripts and styles are files, so the page does not need script-src 'unsafe-inline'. +The demo token is returned only to a loopback client of a demo process. +""" +import json +import os + +CSP = ( + "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; " + "img-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'" +) +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") +HERE = os.path.dirname(os.path.abspath(__file__)) +PAGES = { + "/": ("index.html", "text/html; charset=utf-8"), + "/console": ("console.html", "text/html; charset=utf-8"), + "/flow": ("flow.html", "text/html; charset=utf-8"), + "/integrate": ("integrate.html", "text/html; charset=utf-8"), + "/assets/style.css": ("style.css", "text/css; charset=utf-8"), + "/assets/site.js": ("site.js", "text/javascript; charset=utf-8"), + "/assets/console.js": ("console.js", "text/javascript; charset=utf-8"), + "/assets/flow.js": ("flow.js", "text/javascript; charset=utf-8"), + "/assets/favicon.svg": ("favicon.svg", "image/svg+xml"), +} +_AGENT = None +_FLOW = None + + +def _load(name, filename): + import importlib.util + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, filename)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _agent(): + global _AGENT + if _AGENT is None: + _AGENT = _load("cmo_site_agent", "agent.py") + return _AGENT + + +def _flow(): + global _FLOW + if _FLOW is None: + import importlib.util + path = os.path.normpath(os.path.join(HERE, "..", "tools", "flowview", "flow_feed.py")) + spec = importlib.util.spec_from_file_location("cmo_flow_feed", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + _FLOW = mod + return _FLOW + + +def manifest(): + path = os.path.normpath(os.path.join(HERE, "..", "integration", "manifest.json")) + with open(path) as f: + data = json.load(f) + return { + "name": data["name"], + "version": data["version"], + "api": data["api"], + "adopt": "See INTEGRATION.md. Pass the API address and token from the environment.", + } + + +def _send_file(handler, filename, ctype): + with open(os.path.join(ROOT, filename), "rb") as f: + body = f.read() + handler.send_response(200) + handler.send_header("Content-Type", ctype) + handler.send_header("Content-Length", str(len(body))) + handler.send_header("Cache-Control", "no-store") + handler.send_header("X-Content-Type-Options", "nosniff") + handler.send_header("Referrer-Policy", "no-referrer") + handler.send_header("Content-Security-Policy", CSP) + handler.send_header("X-Frame-Options", "DENY") + handler.end_headers() + handler.wfile.write(body) + + +def _sample(mgr): + src = getattr(mgr, "flow_source", None) + if src is None: + addr = None if getattr(mgr, "demo", False) else mgr.store.upstream + src = _flow().Source(addr) + mgr.flow_source = src + body = src.sample() + if getattr(mgr, "demo", False): + body = dict(body, source="demo") + return body + + +def try_public(handler, mgr, token, method): + path = handler.path.split("?")[0] + if method == "GET" and path in PAGES: + filename, ctype = PAGES[path] + _send_file(handler, filename, ctype) + return True + if method != "GET": + return False + if path == "/v1/integration": + handler._send(200, manifest()) + return True + if path == "/v1/flow": + try: + handler._send(200, _sample(mgr)) + except Exception: + handler._send(503, {"error": "feed unavailable"}) + return True + if path == "/v1/demo": + loopback = handler.client_address[0] in ("127.0.0.1", "::1") + if not getattr(mgr, "demo", False) or not loopback: + handler._send(404, {"error": "not found"}) + else: + handler._send(200, {"demo": True, "token": token}) + return True + return False + + +def agent_run(mgr, message): + return _agent().run(mgr, message) diff --git a/site/memstore.py b/site/memstore.py new file mode 100644 index 00000000000..3b854b3a919 --- /dev/null +++ b/site/memstore.py @@ -0,0 +1,70 @@ +"""In-memory RESP stand-in so the local demo can browse keys without a server.""" +import fnmatch +import socket +import threading + + +def enc(v): + if v is None: + return b"$-1\r\n" + if isinstance(v, bool) or not isinstance(v, (int, str, bytes, list)): + raise TypeError("unsupported reply") + if isinstance(v, int): + return b":%d\r\n" % v + if isinstance(v, str): + raw = v.encode() + return b"+%s\r\n" % raw if b"\r" not in raw and b"\n" not in raw else b"$%d\r\n%s\r\n" % (len(raw), raw) + if isinstance(v, bytes): + return b"$%d\r\n%s\r\n" % (len(v), v) + return b"*%d\r\n" % len(v) + b"".join(enc(x) for x in v) + + +class MemStore: + def __init__(self): + self.data = {} + self.lock = threading.Lock() + + def conn(self): + import cmo_browse as browse + left, right = socket.socketpair() + threading.Thread(target=self._loop, args=(right, browse), daemon=True).start() + return browse.Conn(None, None, None, None, sock=left) + + def _loop(self, sock, browse): + stream = sock.makefile("rb") + while True: + try: + cmd = browse._read(stream) + except Exception: + return + if not isinstance(cmd, list) or not cmd: + sock.sendall(b"-ERR empty\r\n") + continue + words = [x.decode() if isinstance(x, bytes) else str(x) for x in cmd] + try: + sock.sendall(enc(self._op(words))) + except Exception: + sock.sendall(b"-ERR failed\r\n") + + def _op(self, c): + op = c[0].upper() + with self.lock: + if op == "SCAN": + pat = c[3] if len(c) > 3 else "*" + keys = [k.encode() for k in sorted(self.data) if fnmatch.fnmatch(k, pat)] + return [b"0", keys] + key = c[1] if len(c) > 1 else "" + value = self.data.get(key) + if op == "TYPE": + if isinstance(value, bytes): + return "string" + return "none" + if op == "TTL": + return -1 + if value is None: + return None + if op == "STRLEN": + return len(value) + if op == "GET": + return value + raise ValueError("unsupported") diff --git a/site/server.py b/site/server.py new file mode 100644 index 00000000000..7a041cae1de --- /dev/null +++ b/site/server.py @@ -0,0 +1,96 @@ +"""Cache Me Outside site. + + python3 site/server.py --demo + CMO_ADMIN_TOKEN=... python3 site/server.py --config proxy.json --server 127.0.0.1:6379 + +--demo listens on 127.0.0.1 only, keeps a sample key per database, and never +reads a token from the repository. The real mode uses CMO_ADMIN_TOKEN. +""" +import os +import secrets +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MGMT = os.path.normpath(os.path.join(HERE, "..", "tools", "mgmt")) +if MGMT not in sys.path: + sys.path.insert(0, MGMT) + +import cmo_mgmt as mgmt # noqa: E402 +from memstore import MemStore # noqa: E402 + + +class _Acl: + def __init__(self): + self.users = {} + + def set_user(self, name, password): + self.users[name] = password + + def delete_user(self, name): + self.users.pop(name, None) + + +class DemoManager(mgmt.Manager): + def __init__(self, store): + super().__init__(store, _Acl(), "127.0.0.1") + self.demo = True + self.mem = MemStore() + self.browser_conn = self.mem.conn + + def create(self, body): + out = super().create(body) + with self.mem.lock: + self.mem.data[out["name"] + ":welcome"] = b"hello from Cache Me Outside" + out["sample_key"] = out["name"] + ":welcome" + return out + + +def build(argv=None): + import argparse + ap = argparse.ArgumentParser(description="Cache Me Outside site") + ap.add_argument("--demo", action="store_true", help="loopback site with an in-memory database") + ap.add_argument("--listen", default=None) + ap.add_argument("--config") + ap.add_argument("--server", default="127.0.0.1:6379") + ap.add_argument("--proxy-bind", default="127.0.0.1") + a = ap.parse_args(argv) + if a.demo: + listen = a.listen or "127.0.0.1:8088" + host, _, port = listen.rpartition(":") + if host not in ("127.0.0.1", "::1"): + ap.error("demo mode listens on 127.0.0.1 only") + token = secrets.token_urlsafe(24) + directory = tempfile.mkdtemp(prefix="cmo-demo-") + store = mgmt.Store(os.path.join(directory, "proxy.json"), "127.0.0.1:6379") + mgr = DemoManager(store) + return (host, int(port)), mgr, token, a + token = os.environ.get("CMO_ADMIN_TOKEN", "") + if len(token) < 24: + ap.error("set CMO_ADMIN_TOKEN (24+ chars)") + if not a.config: + ap.error("--config is required unless --demo") + listen = a.listen or "127.0.0.1:8088" + host, _, port = listen.rpartition(":") + if host in ("0.0.0.0", "::", "*"): + ap.error("refusing a wildcard listen address; put TLS on a specific address") + sh, _, sp = a.server.rpartition(":") + acl = mgmt.Acl(sh, int(sp), os.environ.get("CMO_ADMIN_USER", "default"), os.environ.get("CMO_ADMIN_PASSWORD")) + mgr = mgmt.Manager(mgmt.Store(a.config, a.server), acl, a.proxy_bind) + return (host, int(port)), mgr, token, a + + +def main(argv=None): + import http.server + addr, mgr, token, _a = build(argv) + srv = http.server.ThreadingHTTPServer(addr, mgmt.make_handler(mgr, token)) + where = "%s:%d" % addr + if getattr(mgr, "demo", False): + print("Cache Me Outside demo on http://%s (loopback only)" % where, file=sys.stderr) + else: + print("Cache Me Outside site on http://%s" % where, file=sys.stderr) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/site/static/console.html b/site/static/console.html new file mode 100644 index 00000000000..da7b198a816 --- /dev/null +++ b/site/static/console.html @@ -0,0 +1,69 @@ + + + + + +Console · Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+

Console

+

Databases

+

The token stays in this page's memory. Passwords are shown once and are not stored by this tool.

+
+
+
+
+ +

+

+
+

Create database

+ + + + + +
+ + + +
+
+
+
+ +
Cache Me OutsideHome
+ + + + diff --git a/site/static/console.js b/site/static/console.js new file mode 100644 index 00000000000..e07403a38f7 --- /dev/null +++ b/site/static/console.js @@ -0,0 +1,253 @@ +"use strict"; +const $ = function (id) { return document.getElementById(id); }; +let current = ""; + +function limitLabel(value) { + return value ? String(value) : "unlimited"; +} + +async function api(method, path, body) { + const token = await window.CMO.ensureToken(); + const response = await fetch(path, { + method: method, + headers: {"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + body: body ? JSON.stringify(body) : undefined, + }); + const payload = await response.json(); + if (!response.ok) throw new Error(payload.error || String(response.status)); + return payload; +} + +function showSecret(db) { + const box = $("result"); + box.hidden = false; + box.textContent = ""; + const title = document.createElement("strong"); + title.textContent = "Database " + db.name + " is ready. Copy the password now. It is shown once."; + const host = location.hostname; + const url = "redis://" + encodeURIComponent(db.name) + ":" + encodeURIComponent(db.password) + "@" + host + ":" + db.port; + const input = document.createElement("input"); + input.className = "url"; + input.readOnly = true; + input.value = url; + input.setAttribute("aria-label", "Connection URL, shown once"); + const copy = document.createElement("button"); + copy.className = "btn"; + copy.type = "button"; + copy.textContent = "Copy URL"; + copy.addEventListener("click", async function () { + try { + await navigator.clipboard.writeText(url); + copy.textContent = "Copied"; + } catch (e) { + input.focus(); + input.select(); + copy.textContent = "Select the URL and copy"; + } + }); + const cli = document.createElement("p"); + cli.textContent = "CLI: valkey-cli -h " + host + " -p " + db.port + " --user " + db.name + " --askpass"; + const prefix = document.createElement("p"); + prefix.textContent = "Keys must start with " + db.name + ":"; + box.append(title, input, copy, cli, prefix); + if (db.sample_key) { + const sample = document.createElement("p"); + sample.textContent = "Local demo sample key: " + db.sample_key; + box.append(sample); + } +} + +function button(label, onClick) { + const el = document.createElement("button"); + el.className = "btn"; + el.type = "button"; + el.textContent = label; + el.addEventListener("click", onClick); + return el; +} + +async function load() { + $("err").textContent = ""; + const list = $("dbs"); + list.textContent = ""; + try { + const body = await api("GET", "/v1/databases"); + if (!body.databases.length) { + const empty = document.createElement("p"); + empty.className = "fine"; + empty.textContent = "No databases yet."; + list.append(empty); + return; + } + body.databases.forEach(function (db) { + const card = document.createElement("article"); + card.className = "card"; + const title = document.createElement("h3"); + title.textContent = db.name; + const meta = document.createElement("p"); + meta.className = "meta"; + meta.textContent = "Port " + db.port + + " · ops/s " + limitLabel(db.ops_per_sec) + + " · connections " + limitLabel(db.max_conns) + + " · bytes/s " + limitLabel(db.bytes_per_sec); + const actions = document.createElement("div"); + actions.className = "row"; + actions.append( + button("Browse", function () { browse(db.name, "0"); }), + button("New password", async function () { + if (!window.confirm("Replace the password for " + db.name + "? The old one stops working.")) return; + try { showSecret(Object.assign({}, db, await api("POST", "/v1/databases/" + db.name + "/rotate-password", {}))); } + catch (e) { $("err").textContent = e.message; } + }), + button("Limits", function () { limits(db); }), + button("Delete", async function () { + if (!window.confirm("Delete " + db.name + "? Keys under " + db.name + ": stay on the server.")) return; + try { await api("DELETE", "/v1/databases/" + db.name); await load(); } + catch (e) { $("err").textContent = e.message; } + }) + ); + card.append(title, meta, actions); + list.append(card); + }); + } catch (e) { + $("err").textContent = e.message; + } +} + +function limits(db) { + const box = $("result"); + box.hidden = false; + box.textContent = ""; + const form = document.createElement("form"); + form.id = "limits"; + const ops = field("Ops per second (0 means unlimited)", "lim-ops", db.ops_per_sec); + const conns = field("Max connections (0 means unlimited)", "lim-conns", db.max_conns); + const bytes = field("Bytes per second (0 means unlimited)", "lim-bytes", db.bytes_per_sec); + const save = document.createElement("button"); + save.className = "btn p"; + save.type = "submit"; + save.textContent = "Save limits"; + form.append(ops, conns, bytes, save); + form.addEventListener("submit", async function (e) { + e.preventDefault(); + try { + await api("PATCH", "/v1/databases/" + db.name, { + ops_per_sec: Number(ops.querySelector("input").value), + max_conns: Number(conns.querySelector("input").value), + bytes_per_sec: Number(bytes.querySelector("input").value), + }); + box.hidden = true; + await load(); + } catch (err) { + $("err").textContent = err.message; + } + }); + const heading = document.createElement("h3"); + heading.textContent = "Limits for " + db.name; + box.append(heading, form); +} + +function field(label, id, value) { + const wrap = document.createElement("label"); + wrap.className = "field"; + wrap.textContent = label; + const input = document.createElement("input"); + input.id = id; + input.type = "number"; + input.min = "0"; + input.required = true; + input.value = String(value || 0); + wrap.append(input); + return wrap; +} + +async function browse(name, cursor) { + $("err").textContent = ""; + $("browser").hidden = false; + $("bname").textContent = "Keys in " + name; + if (cursor === "0") $("keys").textContent = ""; + current = name; + try { + const body = await api("GET", "/v1/databases/" + encodeURIComponent(name) + + "/keys?cursor=" + encodeURIComponent(cursor) + + "&match=" + encodeURIComponent($("bmatch").value)); + if (!body.keys.length && cursor === "0") { + const empty = document.createElement("li"); + empty.textContent = "No keys match."; + $("keys").append(empty); + } + body.keys.forEach(function (key) { + const item = document.createElement("li"); + item.append(button(key, function () { openKey(name, key); })); + $("keys").append(item); + }); + $("bmore").hidden = body.cursor === "0"; + $("bmore").onclick = function () { browse(name, body.cursor); }; + } catch (e) { + $("err").textContent = e.message; + } +} + +async function openKey(name, key) { + try { + const body = await api("GET", "/v1/databases/" + encodeURIComponent(name) + "/keys?key=" + encodeURIComponent(key)); + const box = $("kv"); + box.hidden = false; + box.textContent = ""; + const meta = document.createElement("p"); + meta.textContent = body.key + " · " + body.type + " · TTL " + (body.ttl < 0 ? "none" : body.ttl + "s") + + (body.size !== undefined ? " · size " + body.size : "") + + (body.truncated ? " · first 100 shown" : ""); + const pre = document.createElement("pre"); + const real = JSON.stringify(body.value, null, 2); + pre.textContent = "hidden"; + const eye = document.createElement("button"); + eye.className = "btn"; + eye.type = "button"; + eye.textContent = "Show value"; + eye.setAttribute("aria-pressed", "false"); + eye.addEventListener("click", function () { + const on = eye.getAttribute("aria-pressed") !== "true"; + eye.setAttribute("aria-pressed", on ? "true" : "false"); + pre.textContent = on ? real : "hidden"; + eye.textContent = on ? "Hide value" : "Show value"; + }); + box.append(meta, eye, pre); + } catch (e) { + $("err").textContent = e.message; + } +} + +$("load").addEventListener("click", load); +$("create").addEventListener("submit", async function (e) { + e.preventDefault(); + $("err").textContent = ""; + const name = $("name").value; + if (!/^[a-z][a-z0-9-]{1,30}$/.test(name)) { + $("err").textContent = "Name: 2-31 characters, a lowercase letter, then letters, digits, or dashes."; + return; + } + const body = {name: name}; + ["ops", "conns", "bytes"].forEach(function (id) { + const raw = $(id).value; + if (raw === "") return; + const key = {ops: "ops_per_sec", conns: "max_conns", bytes: "bytes_per_sec"}[id]; + body[key] = Number(raw); + }); + try { + showSecret(await api("POST", "/v1/databases", body)); + $("name").value = ""; + await load(); + } catch (err) { + $("err").textContent = err.message; + } +}); +$("bform").addEventListener("submit", function (e) { + e.preventDefault(); + if (current) browse(current, "0"); +}); +document.addEventListener("cmo-refresh", load); +document.addEventListener("cmo-browse", function (e) { + if (e.detail) browse(e.detail, "0"); +}); +window.CMO.ensureToken().then(function (token) { if (token) load(); }); diff --git a/site/static/favicon.svg b/site/static/favicon.svg new file mode 100644 index 00000000000..d3efaf3a916 --- /dev/null +++ b/site/static/favicon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/site/static/flow.html b/site/static/flow.html new file mode 100644 index 00000000000..38ab713a7fb --- /dev/null +++ b/site/static/flow.html @@ -0,0 +1,60 @@ + + + + + +Flow · Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+

Live data flow

+

From the request to the database

+ +

waiting for the flow feed

+
+
+
+
+ + + + + + +request +near cache +near hit +cache hit +miss +DB query + + +
+
+
+ + + + + + diff --git a/site/static/flow.js b/site/static/flow.js new file mode 100644 index 00000000000..a2afca2289b --- /dev/null +++ b/site/static/flow.js @@ -0,0 +1,58 @@ +"use strict"; +const NS = "http://www.w3.org/2000/svg"; +const lanes = {req: "req", near_hit: "near", cache_hit: "hit", miss: "miss", db: "db"}; +const pathOf = {req: "p-req", near: "p-near", hit: "p-hit", miss: "p-miss", db: "p-db"}; +const reduce = window.matchMedia("(prefers-reduced-motion: reduce)").matches; +let rates = {}; +let dots = []; +const acc = {}; +const MAX_DOTS = 400; +const SCALE = 0.05; +const SPEED = 0.6; + +function spawn(cls) { + if (reduce || dots.length >= MAX_DOTS) return; + const dot = document.createElementNS(NS, "circle"); + dot.setAttribute("class", "dot " + (cls === "db" ? "dbdot" : cls)); + dot.setAttribute("r", "4"); + document.getElementById("dots").appendChild(dot); + dots.push({el: dot, p: document.getElementById(pathOf[cls]), t: 0}); +} + +let last = performance.now(); +function tick(now) { + const dt = Math.min(0.1, (now - last) / 1000); + last = now; + Object.keys(lanes).forEach(function (key) { + acc[key] = (acc[key] || 0) + (rates[key] || 0) * SCALE * dt; + while (acc[key] >= 1) { spawn(lanes[key]); acc[key] -= 1; } + }); + dots = dots.filter(function (dot) { + dot.t += SPEED * dt; + if (dot.t >= 1 || !dot.p) { dot.el.remove(); return false; } + const point = dot.p.getPointAtLength(dot.t * dot.p.getTotalLength()); + dot.el.setAttribute("cx", point.x); + dot.el.setAttribute("cy", point.y); + return true; + }); + if (!reduce) requestAnimationFrame(tick); +} + +async function poll() { + const stats = document.getElementById("stats"); + const note = document.getElementById("flow-note"); + try { + const response = await fetch("/v1/flow", {cache: "no-store"}); + if (!response.ok) throw new Error("feed unavailable"); + rates = await response.json(); + const ratio = rates.hit_ratio == null ? "none" : (rates.hit_ratio * 100).toFixed(1) + "%"; + stats.textContent = "req/s " + rates.req + " near " + rates.near_hit + " hit " + rates.cache_hit + + " miss " + rates.miss + " db " + rates.db + " hit ratio " + ratio; + if (note) note.hidden = rates.source !== "demo"; + } catch (e) { + stats.textContent = "feed unavailable"; + } +} +poll(); +setInterval(poll, 1000); +if (!reduce) requestAnimationFrame(tick); diff --git a/site/static/index.html b/site/static/index.html new file mode 100644 index 00000000000..a85b829df90 --- /dev/null +++ b/site/static/index.html @@ -0,0 +1,72 @@ + + + + + +Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+
+

Cache Me Outside

+

The cache that stays on your desk.

+

Create a database, read its keys, and watch hits and misses move. The agent on this page can do those things with you.

+
Open the consoleWatch the flow
+
+ +
+
+
+
+

What it does

+

Everything on this page is Cache Me Outside. Names of other products appear only where a card points at a similar feature.

+
+

Databases

Each database gets a port, a password that is returned once, and a key prefix. A limit of zero means unlimited.

+

Key browser

Browse is read-only. It will not return a key that sits outside the database prefix, and it will not run a write.

+

Live flow

Requests split into a near-cache hit, a cache hit, or a miss that continues to the database.

+

Warm restart

A clean shutdown can leave a checkpoint for the next process when the usual snapshot did not. It stays off until you turn it on.

+

Hybrid storage

Large strings can spill to a flash log. That path stays off until you configure it.

+

TTL segments

Keys that expire in the same time bucket can be removed together. That path stays off until you configure it.

+

Same wire protocol

Clients that already speak this protocol keep working. Comparable caches:

ValkeyRedisMemcached

+ +
+
+
+
+ + + + + diff --git a/site/static/integrate.html b/site/static/integrate.html new file mode 100644 index 00000000000..71633e57bd5 --- /dev/null +++ b/site/static/integrate.html @@ -0,0 +1,73 @@ + + + + + +Integrate · Cache Me Outside + + + + + +
+
+Cache Me Outside + + +
+
+
+
+
+

Integrate

+

Drop the cache into another GoDesk app.

+

Copy the integration directory or point a script tag at a release. The token stays in the environment.

+
+
+
+
+
+

Python

+
import os
+from cache_me_outside import Client
+
+client = Client(os.environ["CMO_API"], os.environ["CMO_ADMIN_TOKEN"])
+print(client.list_databases())
+
+
+

Browser or Node

+
const cache = CacheMeOutside.attach({
+  api: process.env.CMO_API,
+  token: process.env.CMO_ADMIN_TOKEN
+});
+const rows = await cache.listDatabases();
+
+
+

Sidecar

+
CMO_API=http://127.0.0.1:8088 \
+CMO_ADMIN_TOKEN="$CMO_ADMIN_TOKEN" \
+python3 integration/sidecar.py
+

Then GET http://127.0.0.1:8099/health from the app.

+
+
+

This build

+

The running site reports its integration version. Updating is a separate command, integration/updater.py, and it checks a published checksum before replacing files.

+ +

+
+
+
+
+ +
Cache Me OutsideHome
+ + + diff --git a/site/static/site.js b/site/static/site.js new file mode 100644 index 00000000000..7be0a4fc7f3 --- /dev/null +++ b/site/static/site.js @@ -0,0 +1,86 @@ +"use strict"; +const reduce = window.matchMedia("(prefers-reduced-motion: reduce)").matches; +const menu = document.getElementById("menu"); +const nav = document.getElementById("nav"); +if (menu && nav) { + menu.addEventListener("click", function () { + const open = nav.classList.toggle("open"); + menu.setAttribute("aria-expanded", open ? "true" : "false"); + }); +} +if (!reduce) { + window.addEventListener("pointermove", function (e) { + document.documentElement.style.setProperty("--mx", e.clientX + "px"); + document.documentElement.style.setProperty("--my", e.clientY + "px"); + }); +} + +let sessionToken = ""; +async function ensureToken() { + const field = document.getElementById("tok"); + if (field && field.value) return field.value; + if (sessionToken) return sessionToken; + const response = await fetch("/v1/demo", {cache: "no-store"}); + if (!response.ok) return ""; + const body = await response.json(); + sessionToken = body.token || ""; + if (field) field.value = sessionToken; + const banner = document.getElementById("demo-banner"); + if (banner) banner.hidden = false; + return sessionToken; +} + +async function agentAsk(message) { + const out = document.getElementById("agent-out"); + const token = await ensureToken(); + if (!token) { + out.textContent = "Paste the admin token on the console. This page does not keep a token of its own."; + return; + } + const response = await fetch("/v1/agent", { + method: "POST", + headers: {"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + body: JSON.stringify({message: message}), + }); + const body = await response.json(); + if (!response.ok) { + out.textContent = body.error || "The agent could not do that."; + return; + } + out.textContent = body.reply || ""; + (body.actions || []).forEach(function (action) { + if (action.op === "navigate" && action.href) window.location.assign(action.href); + if (action.op === "refresh") document.dispatchEvent(new Event("cmo-refresh")); + if (action.op === "browse") document.dispatchEvent(new CustomEvent("cmo-browse", {detail: action.name})); + }); +} + +const agentForm = document.getElementById("agent-form"); +if (agentForm) { + agentForm.addEventListener("submit", function (e) { + e.preventDefault(); + agentAsk(document.getElementById("agent-q").value); + }); + document.querySelectorAll("[data-ask]").forEach(function (button) { + button.addEventListener("click", function () { + document.getElementById("agent-q").value = button.getAttribute("data-ask"); + agentAsk(button.getAttribute("data-ask")); + }); + }); +} + +const versionButton = document.getElementById("show-version"); +if (versionButton) { + versionButton.addEventListener("click", async function () { + const slot = document.getElementById("version-out"); + try { + const response = await fetch("/v1/integration", {cache: "no-store"}); + const body = await response.json(); + slot.textContent = body.name + " " + body.version + " (api " + body.api + ")"; + } catch (e) { + slot.textContent = "The version endpoint did not answer."; + } + }); +} + +window.CMO = {ensureToken: ensureToken}; diff --git a/site/static/style.css b/site/static/style.css new file mode 100644 index 00000000000..c3cfaa6fa83 --- /dev/null +++ b/site/static/style.css @@ -0,0 +1,217 @@ +:root { + --bg: #101614; + --tx: #f6f1e7; + --mu: #b7c2ba; + --ac: #f0c48a; + --gold: #e7b56a; + --mint: #9be7c4; + --line: rgba(255, 255, 255, .14); + --mx: 50vw; + --my: 18vh; + --shadow: 8px 8px 18px #070b09, -6px -6px 16px #24302b; + --inset: inset 5px 5px 10px #070b09, inset -4px -4px 10px #2a3832; +} +* { box-sizing: border-box; } +html { scroll-behavior: smooth; } +body { + margin: 0; + min-height: 100vh; + font-family: "Avenir Next", "Segoe UI", sans-serif; + background: var(--bg); + color: var(--tx); + line-height: 1.55; +} +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + z-index: 0; + background: + radial-gradient(480px circle at var(--mx) var(--my), rgba(240, 196, 138, .16), transparent 42%), + radial-gradient(820px circle at 100% -10%, rgba(155, 231, 196, .12), transparent 40%); +} +body > * { position: relative; z-index: 1; } +a { color: var(--ac); } +.wrap { max-width: 1120px; margin: 0 auto; padding: 0 24px; } +.skip { position: absolute; left: -999px; } +.skip:focus { left: 12px; top: 12px; z-index: 20; background: #fff; color: #111; padding: 8px 12px; } +header { + position: sticky; + top: 0; + z-index: 8; + background: rgba(16, 22, 20, .62); + backdrop-filter: blur(18px); + border-bottom: 1px solid var(--line); +} +.nav { display: flex; align-items: center; gap: 22px; min-height: 68px; } +.brand { display: flex; align-items: center; gap: 10px; color: var(--tx); text-decoration: none; font-weight: 700; font-size: 18px; } +.mark { display: block; filter: drop-shadow(0 6px 12px rgba(0, 0, 0, .35)); } +.links { display: flex; gap: 18px; margin-left: auto; align-items: center; flex-wrap: wrap; } +.links a { color: var(--mu); text-decoration: none; } +.links a:hover, .links a[aria-current="page"], .links a:focus-visible { color: var(--tx); } +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + min-height: 44px; + padding: 12px 18px; + border: 0; + border-radius: 14px; + background: #17211d; + color: var(--tx); + font: inherit; + font-weight: 650; + text-decoration: none; + cursor: pointer; + box-shadow: var(--shadow); +} +.btn.p { + background: linear-gradient(165deg, #e7b56a, #c47a3a); + color: #1a1208; + box-shadow: 7px 7px 16px #070b09, -3px -3px 10px rgba(155, 231, 196, .28); +} +.btn:hover { transform: translateY(-1px); } +.btn:focus-visible, a:focus-visible, input:focus-visible, button:focus-visible, select:focus-visible { + outline: 2px solid var(--mint); + outline-offset: 3px; +} +.btn:disabled { opacity: .6; cursor: progress; transform: none; } +#menu { + display: none; + margin-left: auto; + min-width: 44px; + min-height: 44px; + border: 0; + border-radius: 12px; + background: #17211d; + color: var(--tx); + font: inherit; + box-shadow: var(--shadow); + cursor: pointer; +} +.hero { padding: 72px 0 36px; } +.hero h1 { font-size: clamp(36px, 6vw, 64px); line-height: 1.05; letter-spacing: -.03em; margin: 0 0 14px; } +.hero p { color: var(--mu); font-size: clamp(17px, 2vw, 20px); max-width: 640px; } +.eyebrow { color: var(--gold); font-size: 13px; letter-spacing: .14em; text-transform: uppercase; margin-bottom: 8px; } +.cta { display: flex; gap: 12px; flex-wrap: wrap; } +.hero-grid { display: grid; grid-template-columns: minmax(0, 1.2fr) minmax(260px, .8fr); gap: 28px; align-items: center; } +.page-head { padding: 36px 0 8px; } +.page-head h1 { font-size: clamp(32px, 4vw, 48px); margin: 0 0 8px; letter-spacing: -.02em; } +section { padding: 28px 0 48px; } +h2 { font-size: clamp(26px, 3vw, 36px); margin: 0 0 8px; letter-spacing: -.02em; } +.lead { color: var(--mu); max-width: 680px; } +.grid { display: grid; gap: 16px; grid-template-columns: repeat(auto-fit, minmax(min(100%, 240px), 1fr)); } +.card { + background: rgba(255, 255, 255, .06); + border: 1px solid var(--line); + border-radius: 22px; + padding: 22px; + backdrop-filter: blur(18px); + box-shadow: 0 16px 40px rgba(0, 0, 0, .28); +} +.card h3 { margin: 0 0 8px; font-size: 18px; } +.card p, .fine { color: var(--mu); } +.srcs { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 12px; } +.src { + display: inline-block; + padding: 4px 10px; + border-radius: 999px; + background: rgba(240, 196, 138, .12); + color: var(--ac); + text-decoration: none; + font-size: 13px; + font-weight: 650; +} +.field { display: block; margin: 12px 0; font-size: 14px; } +.field input, .field select { + display: block; + width: 100%; + margin-top: 6px; + min-height: 44px; + padding: 10px 12px; + border: 0; + border-radius: 14px; + background: #0c1210; + color: var(--tx); + font: inherit; + box-shadow: var(--inset); +} +.row { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; } +.err { + color: #ffd0d0; + background: rgba(120, 24, 36, .45); + border: 1px solid rgba(255, 180, 180, .35); + border-radius: 12px; + padding: 10px 12px; + margin: 8px 0; +} +.err:empty { display: none; } +.banner { color: var(--gold); margin: 0 0 12px; } +.db { display: grid; gap: 12px; } +.db article { display: grid; gap: 8px; } +.meta { color: var(--mu); font-variant-numeric: tabular-nums; } +.keys { list-style: none; padding: 0; margin: 12px 0; display: grid; gap: 8px; } +.keys button { text-align: left; } +pre, code { font-family: ui-monospace, "SFMono-Regular", monospace; } +pre { + white-space: pre-wrap; + background: rgba(0, 0, 0, .35); + border: 1px solid var(--line); + border-radius: 12px; + padding: 12px; + overflow: auto; +} +.url { + width: 100%; + min-height: 44px; + border: 0; + border-radius: 12px; + padding: 10px 12px; + background: #0c1210; + color: var(--tx); + box-shadow: var(--inset); +} +.assistant { margin: 0 auto 40px; } +.out { min-height: 3em; } +footer { border-top: 1px solid var(--line); padding: 28px 0 40px; color: var(--mu); } +footer .wrap { display: flex; gap: 16px; flex-wrap: wrap; justify-content: space-between; } +footer a { color: var(--mu); text-decoration: none; } +svg.flow { width: 100%; height: auto; display: block; } +.node rect { fill: #17211d; stroke: #e7b56a; stroke-width: 2; } +.node text { fill: var(--tx); text-anchor: middle; font-size: 14px; } +.path { fill: none; stroke: #2c4038; stroke-width: 3; } +.dot { r: 4; } +.req { fill: #f0c48a; } +.near { fill: #9be7c4; } +.hit { fill: #7ddec0; } +.miss { fill: #e07a7a; } +.dbdot { fill: #e7b56a; } +@media (max-width: 860px) { + .hero-grid { grid-template-columns: 1fr; } + .hero-copy { text-align: center; } + .hero-copy p, .hero-copy .cta { margin-left: auto; margin-right: auto; } + .hero-copy .cta { justify-content: center; } + .links { + display: none; + position: absolute; + top: 68px; + left: 0; + right: 0; + flex-direction: column; + align-items: stretch; + gap: 8px; + padding: 12px 20px 18px; + background: rgba(16, 22, 20, .94); + backdrop-filter: blur(16px); + border-bottom: 1px solid var(--line); + } + .links.open { display: flex; } + #menu { display: inline-flex; align-items: center; justify-content: center; } +} +@media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + body::before { background: none; } + .btn:hover { transform: none; } +} diff --git a/site/test_site.py b/site/test_site.py new file mode 100644 index 00000000000..fc78a13aa32 --- /dev/null +++ b/site/test_site.py @@ -0,0 +1,190 @@ +import json +import os +import sys +import threading +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "..", "tools", "mgmt")) + +import agent +import cmo_mgmt as mgmt +import server + + +def start(argv): + addr, mgr, token, _args = server.build(argv) + httpd = ThreadingHTTPServer(addr, mgmt.make_handler(mgr, token)) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd, token, mgr + + +def get(url): + with urllib.request.urlopen(url) as response: + return response.status, dict(response.headers), response.read() + + +class Site(unittest.TestCase): + def setUp(self): + self.httpd, self.token, self.mgr = start(["--demo", "--listen", "127.0.0.1:0"]) + self.base = "http://127.0.0.1:%d" % self.httpd.server_address[1] + + def tearDown(self): + self.httpd.shutdown() + self.httpd.server_close() + + def call(self, method, path, body=None, token=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + self.base + path, data=data, method=method, + headers={"Authorization": "Bearer " + (token if token is not None else self.token), + "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as exc: + return exc.code, json.load(exc) + + def test_pages_and_csp(self): + status, headers, body = get(self.base + "/") + html = body.decode() + self.assertEqual(status, 200) + self.assertIn('id="hero"', html) + self.assertNotIn("unsafe-inline", headers["Content-Security-Policy"]) + self.assertNotIn("innerHTML", html) + self.assertIn("GitGoat", html) + self.assertNotIn("GitGoat", html.split('id="hero"')[1].split("")[0]) + console = get(self.base + "/console")[2].decode() + self.assertIn("Create database", console) + self.assertNotIn('pattern="[a-z][a-z0-9-]', console) + + def test_demo_token_and_database_roundtrip(self): + demo = self.call("GET", "/v1/demo", token="ignored") + # /v1/demo is public. The helper still sends a token; that is fine. + status, payload = demo + self.assertEqual(status, 200) + self.assertEqual(payload["token"], self.token) + created = self.call("POST", "/v1/databases", {"name": "acme"})[1] + self.assertEqual(created["sample_key"], "acme:welcome") + scanned = self.call("GET", "/v1/databases/acme/keys?match=w*")[1] + self.assertEqual(scanned["keys"], ["acme:welcome"]) + shown = self.call("GET", "/v1/databases/acme/keys?key=acme%3Awelcome")[1] + self.assertEqual(shown["value"], "hello from Cache Me Outside") + self.assertEqual(self.call("GET", "/v1/databases/acme/keys?key=other%3Ax")[0], 403) + self.assertEqual(self.call("PATCH", "/v1/databases/acme", {"ops_per_sec": 25})[1]["ops_per_sec"], 25) + self.assertEqual(self.call("DELETE", "/v1/databases/acme")[0], 200) + + def test_agent_creates_without_a_model(self): + os.environ.pop("OLLAMA_BASE_URL", None) + status, payload = self.call("POST", "/v1/agent", {"message": "create database beta"}) + self.assertEqual(status, 200) + self.assertIn("beta", payload["reply"]) + self.assertIn("Password", payload["reply"]) + names = [row["name"] for row in self.call("GET", "/v1/databases")[1]["databases"]] + self.assertIn("beta", names) + opened = self.call("POST", "/v1/agent", {"message": "open flow"})[1] + self.assertEqual(opened["actions"][0]["href"], "/flow") + refused = self.call("POST", "/v1/agent", {"message": "write a poem about caches"})[1] + self.assertIn("OLLAMA_BASE_URL", refused["reply"]) + self.assertEqual(refused["actions"], []) + + def test_flow_is_labeled_demo(self): + status, payload = self.call("GET", "/v1/flow", token="ignored") + self.assertEqual(status, 200) + self.assertEqual(payload["source"], "demo") + self.assertGreater(payload["req"], 0) + version = self.call("GET", "/v1/integration", token="ignored")[1] + self.assertEqual(version["version"], "1.0.0") + + +class Agent(unittest.TestCase): + def test_model_cannot_request_a_shell(self): + class Store: + def __init__(self): + self.names = [] + def list(self): + return [{"name": n} for n in self.names] + def create(self, body): + self.names.append(body["name"]) + return {"name": body["name"], "port": 7001, "password": "pw"} + store = Store() + reply = agent.run(store, "do something unusual", ask=lambda _m: '{"action":"shell","cmd":"id"}') + self.assertEqual(store.names, []) + self.assertIn("something I can do", reply["reply"]) + made = agent.run(store, "please make one", ask=lambda _m: '{"action":"create","name":"from-model"}') + self.assertEqual(store.names, ["from-model"]) + self.assertIn("from-model", made["reply"]) + + def test_bad_ollama_url_does_not_run_a_command(self): + os.environ["OLLAMA_BASE_URL"] = "file:///tmp/not-a-server" + try: + reply = agent.run(object(), "tell me anything at all") + finally: + os.environ.pop("OLLAMA_BASE_URL", None) + self.assertIn("did not answer", reply["reply"]) + + def test_ollama_say_and_redirect_refused(self): + class H(BaseHTTPRequestHandler): + def do_POST(self): + n = int(self.headers.get("Content-Length") or 0) + self.rfile.read(n) + if "bounce" in self.path: + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:9/nope") + self.end_headers() + return + body = json.dumps({"message": {"content": '{"action":"say","text":"from the local model"}'}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *a): + pass + httpd = ThreadingHTTPServer(("127.0.0.1", 0), H) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + os.environ["OLLAMA_BASE_URL"] = "http://127.0.0.1:%d" % httpd.server_address[1] + try: + said = agent.run(object(), "tell me anything at all") + self.assertEqual(said["reply"], "from the local model") + os.environ["OLLAMA_BASE_URL"] = "http://127.0.0.1:%d/bounce" % httpd.server_address[1] + bounced = agent.run(object(), "tell me anything at all") + self.assertIn("did not answer", bounced["reply"]) + finally: + os.environ.pop("OLLAMA_BASE_URL", None) + httpd.shutdown() + httpd.server_close() + + +class Listen(unittest.TestCase): + def test_demo_refuses_a_public_address(self): + with self.assertRaises(SystemExit): + server.build(["--demo", "--listen", "0.0.0.0:8088"]) + + def test_real_mode_hides_the_demo_token(self): + import tempfile + previous = os.environ.get("CMO_ADMIN_TOKEN") + os.environ["CMO_ADMIN_TOKEN"] = "t" * 32 + directory = tempfile.mkdtemp() + try: + httpd, _token, mgr = start(["--config", os.path.join(directory, "p.json"), "--listen", "127.0.0.1:0"]) + self.assertFalse(getattr(mgr, "demo", False)) + base = "http://127.0.0.1:%d" % httpd.server_address[1] + with self.assertRaises(urllib.error.HTTPError) as caught: + urllib.request.urlopen(base + "/v1/demo") + self.assertEqual(caught.exception.code, 404) + httpd.shutdown() + httpd.server_close() + finally: + if previous is None: + os.environ.pop("CMO_ADMIN_TOKEN", None) + else: + os.environ["CMO_ADMIN_TOKEN"] = previous + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/flowview/flow.js b/tools/flowview/flow.js index 556478a7475..fa0490a7934 100644 --- a/tools/flowview/flow.js +++ b/tools/flowview/flow.js @@ -36,4 +36,5 @@ async function poll() { `req/s ${rates.req} near ${rates.near_hit} hit ${rates.cache_hit} miss ${rates.miss} db ${rates.db} hit ratio ${hr}`; } catch (e) { document.getElementById("stats").textContent = "feed unavailable"; } } -poll(); setInterval(poll, 1000); requestAnimationFrame(tick); +poll(); setInterval(poll, 1000); +if (!window.matchMedia("(prefers-reduced-motion: reduce)").matches) requestAnimationFrame(tick); diff --git a/tools/mgmt/cmo_mgmt.py b/tools/mgmt/cmo_mgmt.py index 77dc6fb0f05..68fc1e54a34 100644 --- a/tools/mgmt/cmo_mgmt.py +++ b/tools/mgmt/cmo_mgmt.py @@ -13,8 +13,9 @@ PATCH /v1/databases/{name} limits only DELETE /v1/databases/{name} POST /v1/databases/{name}/rotate-password + GET /v1/databases/{name}/keys read-only, prefix-locked browser GET /v1/openapi.json - GET / web console (token is typed in the page, kept in memory only) + GET / /console /flow /integrate site pages (scripts and styles are separate files) The proxy reads its config at start, so restart it after changes. @@ -32,6 +33,7 @@ import socket import tempfile import threading +from urllib.parse import parse_qs, urlsplit NAME_RE = re.compile(r"^[a-z][a-z0-9-]{1,30}$") LIMITS = ("max_conns", "ops_per_sec", "bytes_per_sec") @@ -184,6 +186,53 @@ def rotate(self, name): self.acl.set_user(name, password) return {"name": name, "password": password, "note": "Shown once. Old password stops working now."} + def open_browser(self): + """Read-only connection. Tests inject browser_conn; otherwise the admin user is used.""" + factory = getattr(self, "browser_conn", None) + if factory is not None: + return factory() + import cmo_browse + host, port = self.acl.addr + return cmo_browse.Conn(host, port, self.acl.user, self.acl.password) + + def keys(self, name, path): + """SCAN or one key, only under ':'. Writes are refused by the browser connection.""" + self.get(name) + import cmo_browse + q = parse_qs(urlsplit(path).query) + + def one(k, default=None): + v = q.get(k) + return v[0] if v else default + + match = one("match", "*") + if not isinstance(match, str) or len(match) > 128: + raise ApiError(400, "match is too long") + key = one("key") + cursor = one("cursor", "0") + conn = self.open_browser() + try: + if key is not None: + if len(key) > 512: + raise ApiError(400, "key is too long") + try: + row = cmo_browse.show(conn, name, key) + except PermissionError: + raise ApiError(403, "key outside this database") + if row is None: + raise ApiError(404, "no such key") + return row + if not str(cursor).isdigit() or len(str(cursor)) > 20: + raise ApiError(400, "cursor must be a number") + return cmo_browse.scan(conn, name, str(cursor), match) + except ValueError as e: + raise ApiError(400, str(e)) + finally: + try: + conn.close() + except OSError: + pass + OPENAPI = { "openapi": "3.0.3", @@ -198,10 +247,28 @@ def rotate(self, name): "/v1/databases/{name}": {"get": {"summary": "Get one"}, "patch": {"summary": "Change limits"}, "delete": {"summary": "Delete (keys are kept)"}}, "/v1/databases/{name}/rotate-password": {"post": {"summary": "New password, returned once"}}, + "/v1/databases/{name}/keys": {"get": {"summary": "Read-only keys under the database prefix"}}, }, } +_GATE = None + + +def _gate(): + """Load site/gate.py by path so this tool does not depend on the process cwd.""" + global _GATE + if _GATE is not None: + return _GATE + import importlib.util + path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "site", "gate.py")) + spec = importlib.util.spec_from_file_location("cmo_site_gate", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + _GATE = mod + return mod + + def make_handler(mgr, token): class H(http.server.BaseHTTPRequestHandler): server_version = "cmo-mgmt" @@ -218,19 +285,6 @@ def _send(self, status, obj): self.end_headers() self.wfile.write(body) - def _console(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "console.html"), "rb") as f: - body = f.read() - self.send_response(200) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.send_header("Content-Security-Policy", - "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; connect-src 'self'") - self.send_header("X-Frame-Options", "DENY") - self.send_header("Referrer-Policy", "no-referrer") - self.end_headers() - self.wfile.write(body) - def _body(self): n = int(self.headers.get("Content-Length") or 0) if n > 65536: @@ -244,13 +298,18 @@ def _body(self): return b def _route(self, method): - if method == "GET" and self.path.split("?")[0] in ("/", "/console"): - return self._console() + gate = _gate() + if gate.try_public(self, mgr, token, method): + return + if not self.path.split("?")[0].startswith("/v1/"): + return self._send(404, {"error": "not found"}) try: auth = self.headers.get("Authorization", "") if not hmac.compare_digest(auth.encode(), ("Bearer " + token).encode()): raise ApiError(401, "unauthorized") p = self.path.split("?")[0].rstrip("/") + if p == "/v1/agent" and method == "POST": + return self._send(200, gate.agent_run(mgr, self._body().get("message"))) parts = p.split("/")[1:] if p == "/v1/openapi.json" and method == "GET": return self._send(200, OPENAPI) @@ -269,11 +328,15 @@ def _route(self, method): return self._send(200, mgr.update(n, self._body())) if method == "DELETE": return self._send(200, mgr.delete(n)) + elif len(parts) == 4 and parts[3] == "keys" and method == "GET": + return self._send(200, mgr.keys(parts[2], self.path)) elif len(parts) == 4 and parts[3] == "rotate-password" and method == "POST": return self._send(200, mgr.rotate(parts[2])) raise ApiError(405 if len(parts) <= 4 else 404, "method not allowed") except ApiError as e: self._send(e.status, {"error": str(e)}) + except ValueError as e: + self._send(400, {"error": str(e)}) except OSError: self._send(502, {"error": "server unreachable"}) diff --git a/tools/mgmt/console.html b/tools/mgmt/console.html deleted file mode 100644 index f13f2242bff..00000000000 --- a/tools/mgmt/console.html +++ /dev/null @@ -1,79 +0,0 @@ - - -Cache Me Outside console - -

Cache Me Outside

-

-

-
-
- - - -
NamePortOps/sConns
- diff --git a/tools/mgmt/test_cmo_mgmt.py b/tools/mgmt/test_cmo_mgmt.py index a3f0b722256..54dee36cf5d 100644 --- a/tools/mgmt/test_cmo_mgmt.py +++ b/tools/mgmt/test_cmo_mgmt.py @@ -75,10 +75,17 @@ def test_console_served_without_token_with_csp(self): import urllib.request with urllib.request.urlopen(self.base + "/") as r: html = r.read().decode() + csp = r.headers["Content-Security-Policy"] self.assertEqual(r.headers["X-Frame-Options"], "DENY") - self.assertIn("connect-src 'self'", r.headers["Content-Security-Policy"]) - self.assertIn("Create database", html) + self.assertIn("connect-src 'self'", csp) + self.assertNotIn("unsafe-inline", csp) + self.assertIn('id="hero"', html) self.assertNotIn("innerHTML", html) + self.assertNotIn("