Read-only data browser in the console (item 62)

docs/ci/cmo-browse-test.patch

@@ -0,0 +1,12 @@
+--- a/.github/workflows/cmo-ci.yml
++++ b/.github/workflows/cmo-ci.yml
+@@ -28,6 +28,9 @@
+       - name: Build server
+         run: make -j"$(nproc)" SERVER_CFLAGS='-Werror'
+
++      - name: Data browser tests
++        run: cd tools/mgmt && python3 test_cmo_browse.py
++
+       - name: Run a fast Valkey test subset
+         run: |
+           # introspection is omitted: its forkless-bgsave case is timing-sensitive

tools/mgmt/cmo_browse.py

@@ -0,0 +1,92 @@
+"""Read-only key browser for one database (REA plan item 62).
+
+Only reads keys under "<name>:", only with read commands, and caps sizes so
+a big key can't stall the server or the page.
+"""
+import socket
+
+MAX_ITEMS = 100
+MAX_BYTES = 4096
+READ_CMDS = {"SCAN", "TYPE", "TTL", "GET", "STRLEN", "LRANGE", "LLEN", "SMEMBERS", "SCARD",
+             "SSCAN", "HSCAN", "HLEN", "ZRANGE", "ZCARD", "XRANGE", "XLEN", "AUTH", "MEMORY"}
+
+
+def _read(f):
+    line = f.readline()
+    if not line:
+        raise ConnectionError("closed")
+    t, rest = line[:1], line[1:-2]
+    if t == b"+":
+        return rest.decode()
+    if t == b"-":
+        raise ValueError(rest.decode(errors="replace"))
+    if t == b":":
+        return int(rest)
+    if t == b"$":
+        n = int(rest)
+        return None if n < 0 else f.read(n + 2)[:-2]
+    if t == b"*":
+        n = int(rest)
+        return None if n < 0 else [_read(f) for _ in range(n)]
+    raise ValueError("unexpected reply")
+
+
+class Conn:
+    def __init__(self, host, port, user, password, timeout=5.0, sock=None):
+        self.s = sock or socket.create_connection((host, port), timeout=timeout)
+        self.f = self.s.makefile("rb")
+        if password:
+            self.cmd("AUTH", user, password)
+
+    def cmd(self, *a):
+        if str(a[0]).upper() not in READ_CMDS:
+            raise PermissionError("browser is read-only")
+        enc = [x if isinstance(x, bytes) else str(x).encode() for x in a]
+        self.s.sendall(b"*%d\r\n" % len(enc) + b"".join(b"$%d\r\n%s\r\n" % (len(x), x) for x in enc))
+        return _read(self.f)
+
+    def close(self):
+        self.f.close(); self.s.close()
+
+
+def _txt(b):
+    if b is None:
+        return None
+    t = b[:MAX_BYTES].decode("utf-8", errors="replace")
+    return t + ("…" if len(b) > MAX_BYTES else "")
+
+
+def scan(conn, name, cursor="0", pattern="*", count=MAX_ITEMS):
+    if "\\" in pattern:
+        raise ValueError("backslash not allowed in pattern")
+    cur, keys = conn.cmd("SCAN", cursor, "MATCH", "%s:%s" % (name, pattern), "COUNT", min(int(count), 1000))
+    return {"cursor": cur.decode() if isinstance(cur, bytes) else str(cur), "keys": [_txt(k) for k in keys]}
+
+
+def show(conn, name, key):
+    if not key.startswith(name + ":"):
+        raise PermissionError("key outside this database")
+    typ = conn.cmd("TYPE", key)
+    out = {"key": key, "type": typ, "ttl": conn.cmd("TTL", key)}
+    n = MAX_ITEMS - 1
+    if typ == "none":
+        return None
+    if typ == "string":
+        out["size"] = conn.cmd("STRLEN", key); out["value"] = _txt(conn.cmd("GET", key))
+    elif typ == "list":
+        out["size"] = conn.cmd("LLEN", key); out["value"] = [_txt(x) for x in conn.cmd("LRANGE", key, 0, n)]
+    elif typ == "set":
+        out["size"] = conn.cmd("SCARD", key); out["value"] = [_txt(x) for x in conn.cmd("SSCAN", key, 0, "COUNT", MAX_ITEMS)[1]]
+    elif typ == "hash":
+        out["size"] = conn.cmd("HLEN", key); r = conn.cmd("HSCAN", key, 0, "COUNT", MAX_ITEMS)[1]
+        out["value"] = {_txt(r[i]): _txt(r[i + 1]) for i in range(0, len(r), 2)}
+    elif typ == "zset":
+        out["size"] = conn.cmd("ZCARD", key); r = conn.cmd("ZRANGE", key, 0, n, "WITHSCORES")
+        out["value"] = [[_txt(r[i]), float(r[i + 1])] for i in range(0, len(r), 2)]
+    elif typ == "stream":
+        out["size"] = conn.cmd("XLEN", key); r = conn.cmd("XRANGE", key, "-", "+", "COUNT", MAX_ITEMS)
+        out["value"] = [[_txt(e[0]), [_txt(x) for x in e[1]]] for e in r]
+    else:
+        out["value"] = None
+    out["truncated"] = isinstance(out.get("size"), int) and out["size"] > MAX_ITEMS
+    return out

tools/mgmt/console.html

@@ -15,6 +15,9 @@ <h1>Cache Me Outside</h1>
 <button>Create database</button></form>
 <div id="result" class="box" hidden role="status"></div>
 <p id="err" class="err" role="alert"></p>
+<section id="browser" hidden><h2 id="bname"></h2>
+<form id="bform"><label>Match <input id="bmatch" value="*"></label> <button>Search keys</button> <button type="button" id="bmore" hidden>More</button></form>
+<ul id="keys"></ul><div id="kv" class="box" hidden></div></section>
 <table><thead><tr><th>Name</th><th>Port</th><th>Ops/s</th><th>Conns</th><th></th></tr></thead><tbody id="rows"></tbody></table>
 <script>
 "use strict";
@@ -42,10 +45,34 @@ <h1>Cache Me Outside</h1>
       const tr=tb.insertRow();for(const v of [d.name,d.port,d.ops_per_sec||"∞",d.max_conns||"∞"])tr.insertCell().textContent=v;
       const rot=document.createElement("button");rot.textContent="New password";
       rot.onclick=async()=>{if(confirm(`Replace the password for ${d.name}? The old one stops working.`)){try{show({...d,...await api("POST",`/v1/databases/${d.name}/rotate-password`,{})});}catch(e){$("err").textContent=e.message;}}};
-      tr.insertCell().append(rot);
+      const br=document.createElement("button");br.textContent="Browse";br.onclick=()=>browse(d.name,"0");
+      tr.insertCell().append(rot," ",br);
     }
   }catch(e){$("err").textContent=e.message;}
 }
+let cur={name:"",cursor:"0"};
+async function browse(name,cursor){
+  $("err").textContent="";$("browser").hidden=false;$("bname").textContent="Keys in "+name;
+  if(cursor==="0")$("keys").textContent="";cur={name,cursor};
+  try{
+    const r=await api("GET",`/v1/databases/${encodeURIComponent(name)}/keys?cursor=${encodeURIComponent(cursor)}&match=${encodeURIComponent($("bmatch").value)}`);
+    for(const k of r.keys){const li=document.createElement("li");const a=document.createElement("button");a.textContent=k;a.onclick=()=>openKey(name,k);li.append(a);$("keys").append(li);}
+    cur.cursor=r.cursor;$("bmore").hidden=r.cursor==="0";
+  }catch(e){$("err").textContent=e.message;}
+}
+async function openKey(name,key){
+  try{
+    const r=await api("GET",`/v1/databases/${encodeURIComponent(name)}/keys?key=${encodeURIComponent(key)}`);
+    const box=$("kv");box.hidden=false;box.textContent="";
+    const meta=document.createElement("p");meta.textContent=`${r.key} · ${r.type} · TTL ${r.ttl<0?"none":r.ttl+"s"}${r.size!==undefined?" · size "+r.size:""}${r.truncated?" · first 100 shown":""}`;
+    const pre=document.createElement("pre");const real=JSON.stringify(r.value,null,2);pre.textContent="•••••• hidden";
+    const eye=document.createElement("button");eye.textContent="👁 Show value";eye.setAttribute("aria-pressed","false");
+    eye.onclick=()=>{const on=eye.getAttribute("aria-pressed")!=="true";eye.setAttribute("aria-pressed",String(on));pre.textContent=on?real:"•••••• hidden";eye.textContent=on?"🙈 Hide value":"👁 Show value";};
+    box.append(meta,eye,pre);
+  }catch(e){$("err").textContent=e.message;}
+}
+$("bform").onsubmit=e=>{e.preventDefault();browse(cur.name,"0");};
+$("bmore").onclick=()=>browse(cur.name,cur.cursor);
 $("load").onclick=load;
 $("create").onsubmit=async e=>{e.preventDefault();$("err").textContent="";
   try{show(await api("POST","/v1/databases",{name:$("name").value}));$("name").value="";load();}catch(x){$("err").textContent=x.message;}};

tools/mgmt/test_cmo_browse.py

@@ -0,0 +1,101 @@
+import os, socket, sys, threading, unittest
+sys.path.insert(0, os.path.dirname(__file__))
+import cmo_browse as b
+
+
+def enc(v):
+    if v is None: return b"$-1\r\n"
+    if isinstance(v, int): return b":%d\r\n" % v
+    if isinstance(v, str): return b"+%s\r\n" % v.encode()
+    if isinstance(v, bytes): return b"$%d\r\n%s\r\n" % (len(v), v)
+    return b"*%d\r\n" % len(v) + b"".join(enc(x) for x in v)
+
+
+class Fake:
+    def __init__(self, data):
+        self.data, self.log = data, []
+        self.a, self.s = socket.socketpair()
+        threading.Thread(target=self.loop, daemon=True).start()
+
+    def loop(self):
+        f = self.s.makefile("rb")
+        while True:
+            try: c = b._read(f)
+            except Exception: return
+            c = [x.decode() for x in c]; self.log.append(c); op, k = c[0], (c[1] if len(c) > 1 else None)
+            v = self.data.get(k)
+            if op == "SCAN":
+                pat = c[3].rstrip("*")
+                r = [b"0", [x.encode() for x in self.data if x.startswith(pat)]]
+            elif op == "TYPE":
+                r = "none" if v is None else {str: "string", list: "list", dict: "hash"}[type(v)]
+            elif op == "TTL": r = -1
+            elif op == "STRLEN": r = len(v)
+            elif op == "GET": r = v.encode()
+            elif op == "LLEN": r = len(v)
+            elif op == "LRANGE": r = [x.encode() for x in v[: int(c[3]) + 1]]
+            elif op == "HLEN": r = len(v)
+            elif op == "HSCAN": r = ["0", [y.encode() for kv in v.items() for y in kv]]
+            else: r = "OK"
+            self.s.sendall(enc(r))
+
+
+class T(unittest.TestCase):
+    def setUp(self):
+        self.f = Fake({"acme:s": "x" * 5000, "acme:l": [str(i) for i in range(150)],
+                       "acme:h": {"a": "1"}, "other:s": "secret"})
+        self.c = b.Conn(None, None, None, None, sock=self.f.a)
+
+    def test_scan_is_scoped(self):
+        r = b.scan(self.c, "acme")
+        self.assertNotIn("other:s", r["keys"])
+        self.assertEqual(self.f.log[-1][3], "acme:*")
+
+    def test_cannot_read_other_db(self):
+        with self.assertRaises(PermissionError):
+            b.show(self.c, "acme", "other:s")
+
+    def test_string_truncated(self):
+        r = b.show(self.c, "acme", "acme:s")
+        self.assertEqual(r["size"], 5000)
+        self.assertTrue(r["value"].endswith("…"))
+
+    def test_list_capped(self):
+        r = b.show(self.c, "acme", "acme:l")
+        self.assertEqual(len(r["value"]), 100)
+        self.assertTrue(r["truncated"])
+
+    def test_hash_and_missing(self):
+        self.assertEqual(b.show(self.c, "acme", "acme:h")["value"], {"a": "1"})
+        self.assertIsNone(b.show(self.c, "acme", "acme:nope"))
+
+    def test_read_only(self):
+        with self.assertRaises(PermissionError):
+            self.c.cmd("DEL", "acme:s")
+
+    def test_backslash_rejected(self):
+        with self.assertRaises(ValueError):
+            b.scan(self.c, "acme", pattern="\\*")
+
+    def test_api_route(self):
+        import cmo_mgmt as m, tempfile, http.server, cmoctl
+        sys.path.insert(0, os.path.dirname(__file__))
+        from test_cmo_mgmt import FakeAcl, TOKEN
+        d = tempfile.mkdtemp()
+        mgr = m.Manager(m.Store(os.path.join(d, "p.json"), "x:1"), FakeAcl())
+        mgr.browser_conn = lambda: b.Conn(None, None, None, None, sock=Fake({"acme:h": {"a": "1"}}).a)
+        srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), m.make_handler(mgr, TOKEN))
+        threading.Thread(target=srv.serve_forever, daemon=True).start()
+        base = "http://127.0.0.1:%d" % srv.server_address[1]
+        self.assertEqual(cmoctl.call("GET", "/v1/databases/acme/keys", None, base, TOKEN)[0], 404)
+        cmoctl.call("POST", "/v1/databases", {"name": "acme"}, base, TOKEN)
+        st, r = cmoctl.call("GET", "/v1/databases/acme/keys?match=h*", None, base, TOKEN)
+        self.assertEqual((st, r["keys"]), (200, ["acme:h"]))
+        st, r = cmoctl.call("GET", "/v1/databases/acme/keys?key=acme%3Ah", None, base, TOKEN)
+        self.assertEqual(r["value"], {"a": "1"})
+        self.assertEqual(cmoctl.call("GET", "/v1/databases/acme/keys?key=x%3Ay", None, base, TOKEN)[0], 403)
+        srv.shutdown(); srv.server_close()
+
+
+if __name__ == "__main__":
+    unittest.main()