From 44d8af985326f2a01a7be287b1e7df0c926ea4d5 Mon Sep 17 00:00:00 2001 From: Mickael Farina Date: Thu, 2 Jul 2026 16:13:15 +0200 Subject: [PATCH] feat(pilot): land the Operator-style Pilot implementation (June 10 work) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the complete Pilot build that sat uncommitted in the working tree since 2026-06-10 — HEAD was a non-functional skeleton (no auth token injection → every pilot-runner call 401'd; MJPEG buffered → viewport hung forever; bare-domain navigation broken). - routes/pilot_proxy.py: injects x-pilot-token from ~/.codec/pilot_token server-side (PP-1 — token never reaches the browser), relays the MJPEG stream via StreamingResponse chunk passthrough, 503/502 on runner-down. - codec_tasks.html: Operator-style layout (1280×800 live viewport + 400px mission rail), LIVE / CODEC DRIVING badges, incremental action log, pause / resume / take-over / hand-back HITL controls, voice I/O, https:// prepend for bare domains. - tests/test_pilot_proxy.py: 4 tests (token read/strip, missing-file, header injection, MJPEG path coverage). Side effect: clears the dirty working tree that had been blocking the 6 AM auto-pull cron since ~Jun 10. Co-Authored-By: Claude Fable 5 --- codec_tasks.html | 302 +++++++++++++++++++++++++++++--------- routes/pilot_proxy.py | 73 ++++++++- tests/test_pilot_proxy.py | 41 ++++++ 3 files changed, 343 insertions(+), 73 deletions(-) create mode 100644 tests/test_pilot_proxy.py diff --git a/codec_tasks.html b/codec_tasks.html index 556163a..363a0d9 100644 --- a/codec_tasks.html +++ b/codec_tasks.html @@ -78,6 +78,8 @@ /* ── Container ── */ .container { max-width: 900px; margin: 0 auto; padding: 24px 16px; } +/* Pilot tab gets the full runway — the live viewport deserves the space */ +.container:has(#panel-pilot.active) { max-width: 1560px; } /* ── Tabs ── */ .tabs { display: flex; gap: 4px; margin-bottom: 24px; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 4px; } @@ -430,11 +432,11 @@

Task Reports

Checking pilot… - pilot.lucyvpa.com ↗ + 🔒 local-only -
+
Teach mode — drive the browser yourself and CODEC will compile your clicks into a reusable skill. @@ -444,40 +446,82 @@

Task Reports

- -
+ +
- -
+ +
+
+ Live browser +
+ LIVE +
+ +
+
+ +
+ + + +
+
+ + +
- +
-
Navigate
-
- - +
Give CODEC a mission
+ +
+ + +
- + + + +
-
DOM Snapshot
- +
Recent Runs
+ +
+
+
No runs yet.
-
—
- +
+
+ + +
+ ⚙ Advanced — DOM snapshot · manual actions · skill review +
+ +
-
Run Agent Task
- -
- - - - +
+
DOM Snapshot
+
+
—
@@ -496,14 +540,6 @@

Task Reports

- -
-
Recent Runs
-
-
No runs yet.
-
-
-
@@ -528,33 +564,18 @@

Task Reports


         

 
-      

-
-      
-      

-        

-          
Screenshot

-          
-        

-        Browser screenshot
-        

-      

-
-    

-
-    
-    
 
@@ -1190,9 +1211,11 @@ 
Task Reports

 });
 /* ═══════════════════════════════════════════════════════════════
    CODEC PILOT — tab JS
-   All calls go to localhost:8094 (pilot-runner FastAPI)
+   All calls go through the same-origin dashboard proxy /api/pilot/*
+   (routes/pilot_proxy.py injects x-pilot-token server-side; pilot-runner
+   itself stays loopback-only on :8094). Works locally AND over the tunnel.
 ═══════════════════════════════════════════════════════════════ */
-var _pilotBase = 'http://localhost:8094';
+var _pilotBase = '/api/pilot';
 var _pilotPollTimer = null;
 var _pilotSelectedRun = null;
 var _pilotStreamTimer = null;   // fallback poll when MJPEG stream unavailable
@@ -1233,37 +1256,59 @@ 
Task Reports

       if (dot)  dot.style.background = '#22c55e';
       if (text) text.textContent = 'Online  ·  ' + (d.url || 'about:blank') + '  ·  CDP :' + (d.cdp_port || 9223);
       document.getElementById('pilotCurrentUrl').textContent = d.url || '';
+      var pulse = document.getElementById('pilotLivePulse');
+      if (pulse) { pulse.style.background = '#22c55e'; pulse.style.boxShadow = '0 0 8px rgba(34,197,94,.8)'; }
     })
     .catch(function() {
       var dot  = document.getElementById('pilotDot');
       var text = document.getElementById('pilotStatusText');
       if (dot)  dot.style.background = '#ef4444';
       if (text) text.textContent = 'Pilot Runner offline  ·  pm2 restart pilot-runner';
+      var pulse = document.getElementById('pilotLivePulse');
+      if (pulse) { pulse.style.background = '#666'; pulse.style.boxShadow = 'none'; }
     });
 }
 
-/* ── Screenshot / live stream ── */
+/* ── Screenshot / live stream ──
+   The MJPEG  stream dies silently (sleep, dashboard restart, network
+   blip) and freezes on its last frame. Contract:
+   - pilotStartStream(): (re)connect the LIVE stream; on error, poll frames.
+   - pilotRefreshScreenshot(): "refresh" = RECONNECT the live stream — it
+     must never replace the stream with a static frame (the old freeze bug).
+   - _pilotPollFrame(): static-frame fallback used only while the stream is down. */
 function pilotStartStream() {
   var img = document.getElementById('pilotScreenshot');
   if (!img) return;
-  // Point at MJPEG stream — browser keeps it alive as a live feed
-  img.src = _pilotBase + '/screenshot/stream?' + Date.now();
+  if (_pilotStreamTimer) { clearInterval(_pilotStreamTimer); _pilotStreamTimer = null; }
   img.onerror = function() {
-    // Stream failed (pilot offline?) — fall back to 2-second polling
+    // Stream failed (pilot offline?) — fall back to 2-second frame polling
     img.onerror = null;
     if (!_pilotStreamTimer) {
-      _pilotStreamTimer = setInterval(pilotRefreshScreenshot, 2000);
+      _pilotStreamTimer = setInterval(_pilotPollFrame, 2000);
     }
   };
+  img.src = _pilotBase + '/screenshot/stream?' + Date.now();
 }
 
-function pilotRefreshScreenshot() {
+function _pilotPollFrame() {
   var img = document.getElementById('pilotScreenshot');
   if (!img) return;
-  // Only used as fallback when stream is unavailable
   img.src = _pilotBase + '/screenshot?' + Date.now();
 }
 
+function pilotRefreshScreenshot() {
+  pilotStartStream();   // refresh = revive the live feed
+}
+
+/* Reconnect the stream whenever the tab regains focus — img streams don't
+   survive sleep/navigation-away and there's no JS event when they die. */
+document.addEventListener('visibilitychange', function() {
+  if (!document.hidden) {
+    var panel = document.getElementById('panel-pilot');
+    if (panel && panel.classList.contains('active')) pilotStartStream();
+  }
+});
+
 function pilotOpenScreenshot() {
   window.open(_pilotBase + '/screenshot', '_blank');
 }
@@ -1405,12 +1450,133 @@ 
Task Reports

       document.getElementById('pilotTaskInput').value = '';
       setTimeout(pilotRefreshRuns, 500);
       _pilotSelectedRun = runId;
-      setTimeout(function(){ pilotShowRunDetail(runId); }, 1000);
+      _pilotWatchRun(runId, task);   // Operator view: live action log + HITL
     });
   })
   .catch(function(e){ showToast('Run error: ' + e, true); });
 }
 
+/* ── Operator live-run watcher: status chip + streaming action log + HITL ── */
+var _pilotWatchTimer = null;
+var _pilotLogCount = 0;
+
+function _agoFmt(epochSecs) {
+  if (!epochSecs) return '';
+  var s = Math.max(0, Math.round(Date.now() / 1000 - epochSecs));
+  if (s < 60)    return s + 's ago';
+  if (s < 3600)  return Math.round(s / 60) + 'm ago';
+  if (s < 86400) return Math.round(s / 3600) + 'h ago';
+  return Math.round(s / 86400) + 'd ago';
+}
+
+function _pilotStepLine(s, i) {
+  var icons = {navigate:'🌐', click:'🖱', type:'⌨️', scroll:'↕️', wait:'⏲', done:'✅', error:'❌'};
+  var a = (s && (s.action || s.act || s.type)) || '';
+  var icon = icons[a] || '•';
+  var det = '';
+  if (s) {
+    if (s.url) det = s.url;
+    else if (s.text) det = '"' + String(s.text).substring(0, 40) + '"';
+    else if (s.index !== undefined && s.index !== null) det = '[' + s.index + ']';
+    else if (s.result) det = String(s.result).substring(0, 60);
+    else if (s.reason) det = String(s.reason).substring(0, 60);
+  }
+  var thought = s && (s.summary || s.thought)
+    ? ' — ' + escHtml(String(s.summary || s.thought).substring(0, 70)) + '' : '';
+  return '
' + (i + 1) + '. ' + icon +
+         ' ' + escHtml(a) + ' ' +
+         '' + escHtml(det) + '' + thought + '
';
+}
+
+function _pilotSetHitl(state) {
+  var p = document.getElementById('pilotPauseBtn'),
+      r = document.getElementById('pilotResumeBtn'),
+      t = document.getElementById('pilotTakeoverBtn'),
+      h = document.getElementById('pilotHandbackBtn');
+  if (!p) return;
+  if (state === 'running')       { p.style.display='';     r.style.display='none'; t.style.display='';     h.style.display='none'; }
+  else if (state === 'paused')   { p.style.display='none'; r.style.display='';     t.style.display='';     h.style.display='none'; }
+  else if (state === 'takeover') { p.style.display='none'; r.style.display='none'; t.style.display='none'; h.style.display='';     }
+  else                           { p.style.display='none'; r.style.display='none'; t.style.display='none'; h.style.display='none'; }
+}
+
+function _pilotWatchRun(runId, taskText) {
+  if (_pilotWatchTimer) clearInterval(_pilotWatchTimer);
+  _pilotSelectedRun = runId;
+  _pilotLogCount = 0;
+  document.getElementById('pilotRunPanel').style.display = '';
+  document.getElementById('pilotRunTask').textContent = taskText || '';
+  document.getElementById('pilotActionLog').innerHTML = '
starting…
';
+  document.getElementById('pilotAgentBadge').style.display = '';
+  _pilotSetHitl('running');
+  _pilotWatchTimer = setInterval(function(){ _pilotPollRun(runId); }, 1200);
+  _pilotPollRun(runId);
+}
+
+function _pilotPollRun(runId) {
+  _pilotFetch('/run/' + runId + '/status')
+    .then(function(r){ return r.json(); })
+    .then(function(d) {
+      var steps = d.steps || [];
+      var log = document.getElementById('pilotActionLog');
+      if (_pilotLogCount === 0 && steps.length) log.innerHTML = '';
+      if (steps.length > _pilotLogCount) {
+        for (var i = _pilotLogCount; i < steps.length; i++) {
+          log.insertAdjacentHTML('beforeend', _pilotStepLine(steps[i], i));
+        }
+        _pilotLogCount = steps.length;
+        log.scrollTop = log.scrollHeight;
+      }
+      var chipMap = {running:'⏳ running', done:'✅ done', error:'❌ error',
+                     budget_exhausted:'⚠️ budget', paused:'⏸ paused', paused_timeout:'⏸ pause timeout'};
+      document.getElementById('pilotRunChip').textContent = chipMap[d.status] || ('📊 ' + (d.status || ''));
+      // Speak terminal transitions once (shares the guard map with pilotShowRunDetail)
+      if (_pilotLastSpokenStatus[runId] !== d.status) {
+        var prev = _pilotLastSpokenStatus[runId];
+        _pilotLastSpokenStatus[runId] = d.status;
+        if (prev && (d.status === 'done' || d.status === 'error' || d.status === 'budget_exhausted')) {
+          pilotSay(d.status === 'done'
+            ? 'Run complete. ' + (d.result ? String(d.result).slice(0,160) : '')
+            : d.status === 'error' ? 'Run failed. ' + (d.error || '')
+            : 'Step budget exhausted.');
+        }
+      }
+      if (d.status !== 'running' && d.status !== 'paused') {
+        clearInterval(_pilotWatchTimer); _pilotWatchTimer = null;
+        document.getElementById('pilotAgentBadge').style.display = 'none';
+        _pilotSetHitl('terminal');
+        if (d.result) log.insertAdjacentHTML('beforeend',
+          '
🏁 ' + escHtml(String(d.result).substring(0, 200)) + '
');
+        pilotRefreshRuns();
+      }
+    })
+    .catch(function(){});
+}
+
+function pilotTakeover() {
+  if (!_pilotSelectedRun) return;
+  _pilotFetch('/hitl/' + _pilotSelectedRun + '/takeover',
+              {method:'POST', headers:{'Content-Type':'application/json'}, body:'{}'})
+    .then(function(){
+      showToast('You have the wheel — drive with the URL bar / Advanced tools, then Hand back');
+      document.getElementById('pilotAgentBadge').style.display = 'none';
+      _pilotSetHitl('takeover');
+    })
+    .catch(function(e){ showToast('Takeover failed: ' + e, true); });
+}
+
+function pilotHandback() {
+  if (!_pilotSelectedRun) return;
+  _pilotFetch('/hitl/' + _pilotSelectedRun + '/handback',
+              {method:'POST', headers:{'Content-Type':'application/json'}, body:'{}'})
+    .then(function(){
+      showToast('CODEC has the wheel again');
+      document.getElementById('pilotAgentBadge').style.display = '';
+      _pilotSetHitl('running');
+    })
+    .catch(function(e){ showToast('Handback failed: ' + e, true); });
+}
+
 /* ── Runs list ── */
 function pilotRefreshRuns() {
   _pilotFetch('/runs')
@@ -1426,7 +1592,7 @@ 
Task Reports

       var statusIcon = {done:'✅', error:'❌', running:'⏳', budget_exhausted:'⚠️', recording:'⏺️'};
       list.innerHTML = runs.slice(0,12).map(function(r) {
         var icon = statusIcon[r.status] || '📊';
-        var ago  = r.started_at ? Math.round((Date.now()/1000 - r.started_at)) + 's ago' : '';
+        var ago  = _agoFmt(r.started_at);
         var rid  = r.run_id;
         return '
' +
           '' + icon + '' +
@@ -1622,7 +1788,7 @@ 
Task Reports

         return;
       }
       list.innerHTML = items.map(function(s) {
-        var ago = s.mtime ? Math.round((Date.now()/1000 - s.mtime)) + 's ago' : '';
+        var ago = _agoFmt(s.mtime);
         return '
' +
           '📝' +
           '' + escHtml(s.filename) + '' +
diff --git a/routes/pilot_proxy.py b/routes/pilot_proxy.py
index 4e8d025..579ada9 100644
--- a/routes/pilot_proxy.py
+++ b/routes/pilot_proxy.py
@@ -7,25 +7,88 @@
 over Cloudflare-tunneled HTTPS — Pilot Runner is HTTP-localhost — so
 the PWA needs this same-origin proxy to reach it without a CORS
 preflight or a mixed-content block.
+
+Auth (PP-1): pilot-runner requires `x-pilot-token` (from ~/.codec/pilot_token,
+0600) on every request. The proxy injects it server-side so the token never
+reaches the browser; the dashboard's own AuthMiddleware gates who can reach
+/api/pilot/* in the first place. Pilot stays loopback-only (P-1 — never
+tunnel :8094 directly).
+
+Streaming: the MJPEG live view (`screenshot/stream`) is proxied via
+StreamingResponse chunk passthrough — a buffered request would hang forever
+on the endless multipart stream.
 """
 from __future__ import annotations
 
+import os
+
 import httpx
 from fastapi import APIRouter, Request
-from fastapi.responses import JSONResponse, Response
+from fastapi.responses import JSONResponse, Response, StreamingResponse
 
 router = APIRouter()
 
+_TOKEN_PATH = os.path.expanduser("~/.codec/pilot_token")
+_PILOT_BASE = "http://localhost:8094"
+
+# Endpoints whose responses never end (multipart streams) — must be chunk-proxied.
+_STREAM_PATHS = {"screenshot/stream"}
+
+
+def _pilot_token() -> str:
+    """Read the shared pilot token (never cached — supports rotation)."""
+    try:
+        with open(_TOKEN_PATH) as f:
+            return f.read().strip()
+    except OSError:
+        return ""
+
+
+def _build_headers(content_type: str | None) -> dict:
+    headers = {"x-pilot-token": _pilot_token()}
+    if content_type:
+        headers["content-type"] = content_type
+    return headers
+
 
 @router.api_route("/api/pilot/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
 async def pilot_proxy(path: str, request: Request):
     """Proxy /api/pilot/* → localhost:8094/* so the HTTPS dashboard can reach the local runner."""
-    target = f"http://localhost:8094/{path}"
+    target = f"{_PILOT_BASE}/{path}"
     params = dict(request.query_params)
+    headers = _build_headers(request.headers.get("content-type"))
+
+    # ── live MJPEG stream: chunk passthrough, no buffering ──
+    if path in _STREAM_PATHS:
+        client = httpx.AsyncClient(timeout=None)
+        try:
+            req = client.build_request("GET", target, params=params, headers=headers)
+            upstream = await client.send(req, stream=True)
+        except httpx.ConnectError:
+            await client.aclose()
+            return JSONResponse({"error": "Pilot Runner offline — pm2 restart pilot-runner"},
+                                status_code=503)
+        except Exception as exc:
+            await client.aclose()
+            return JSONResponse({"error": str(exc)}, status_code=502)
+
+        async def _relay():
+            try:
+                async for chunk in upstream.aiter_bytes():
+                    yield chunk
+            finally:
+                await upstream.aclose()
+                await client.aclose()
+
+        return StreamingResponse(
+            _relay(),
+            status_code=upstream.status_code,
+            media_type=upstream.headers.get("content-type",
+                                            "multipart/x-mixed-replace; boundary=frame"),
+        )
+
+    # ── normal request/response ──
     body = await request.body()
-    headers = {}
-    if request.headers.get("content-type"):
-        headers["content-type"] = request.headers["content-type"]
     try:
         async with httpx.AsyncClient(timeout=30.0) as client:
             r = await client.request(
diff --git a/tests/test_pilot_proxy.py b/tests/test_pilot_proxy.py
new file mode 100644
index 0000000..24839f9
--- /dev/null
+++ b/tests/test_pilot_proxy.py
@@ -0,0 +1,41 @@
+"""Tests for the Pilot proxy token injection (PP-1 auth handshake).
+
+The dashboard proxy must inject x-pilot-token (read server-side from
+~/.codec/pilot_token) into every upstream request — the browser never
+sees the token. Missing/unreadable token file → empty header (pilot-runner
+fail-closes with 401, proxy must not crash).
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from routes import pilot_proxy  # noqa: E402
+
+
+def test_pilot_token_read_and_stripped(tmp_path, monkeypatch):
+    tok = tmp_path / "pilot_token"
+    tok.write_text("  secret-token-value\n")
+    monkeypatch.setattr(pilot_proxy, "_TOKEN_PATH", str(tok))
+    assert pilot_proxy._pilot_token() == "secret-token-value"
+
+
+def test_pilot_token_missing_file_is_empty(tmp_path, monkeypatch):
+    monkeypatch.setattr(pilot_proxy, "_TOKEN_PATH", str(tmp_path / "nope"))
+    assert pilot_proxy._pilot_token() == ""
+
+
+def test_headers_always_include_token(tmp_path, monkeypatch):
+    tok = tmp_path / "pilot_token"
+    tok.write_text("tok123")
+    monkeypatch.setattr(pilot_proxy, "_TOKEN_PATH", str(tok))
+    h = pilot_proxy._build_headers(None)
+    assert h["x-pilot-token"] == "tok123"
+    assert "content-type" not in h
+    h2 = pilot_proxy._build_headers("application/json")
+    assert h2["x-pilot-token"] == "tok123"
+    assert h2["content-type"] == "application/json"
+
+
+def test_stream_paths_cover_mjpeg():
+    assert "screenshot/stream" in pilot_proxy._STREAM_PATHS