diff --git a/Cargo.lock b/Cargo.lock index 998bc4cc2..6be96d12e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,6 +540,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -969,6 +996,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -1750,6 +1783,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -3421,6 +3465,7 @@ dependencies = [ "quarto-error-catalog", "quarto-error-reporting", "quarto-hub", + "quarto-hub-provider", "quarto-lsp", "quarto-mcp-launcher", "quarto-preview", @@ -3440,6 +3485,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "url", "walkdir", ] @@ -3712,6 +3758,32 @@ dependencies = [ "walkdir", ] +[[package]] +name = "quarto-hub-provider" +version = "0.7.0" +dependencies = [ + "automerge", + "ciborium", + "flate2", + "futures", + "pollster", + "quarto-core", + "quarto-hub", + "quarto-mcp-launcher", + "quarto-system-runtime", + "quarto-trace", + "samod", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite 0.27.0", + "tracing", + "tungstenite 0.27.0", + "url", +] + [[package]] name = "quarto-lsp" version = "0.7.0" diff --git a/claude-notes/hub-execution-e2e/README.md b/claude-notes/hub-execution-e2e/README.md new file mode 100644 index 000000000..07ce32854 --- /dev/null +++ b/claude-notes/hub-execution-e2e/README.md @@ -0,0 +1,163 @@ +# End-to-end test: run code from hub-client via a `q2` executor + +A hands-on test of the remote code-execution feature (bd-sfet3264, Phases +4a + 4b): you click **Run** in the hub-client editor and a connected `q2` +process executes the document's code and streams the output back into the +preview — the path a real collaborator would use. + +You run hub-client locally (`npm run dev`) plus a `q2 provide-hub` executor. +Both talk to an automerge **sync server**, which stores and relays the project +docs (index, files, captures) and the ephemeral run requests/beacons. There are +two ways to provide that sync server — pick one: + +``` + ┌───────────────┐ automerge sync (index+file+capture docs, ephemeral msgs) + │ hub-client │◄─────────────────────┐ + │ (npm run dev) │ │ + │ Run button ──┼──── exec/request ─────┤ ┌──────────────────────┐ + │ shows output │◄─── capture doc ──────┤◄─►│ sync server │ + └───────────────┘ │ │ (A: sync.automerge.org + ┌──────────────────────┐│ │ B: local q2 hub) │ + │ q2 provide-hub ├┘ └──────────────────────┘ + │ --allow-all │ runs engines, writes captures + └──────────────────────┘ +``` + +## Option A — public sync server (simplest; verified) + +Uses `wss://sync.automerge.org` (hub-client's built-in default) as the sync +server. **No `q2 hub` needed** — the public server stores and relays every doc +any peer creates, so a project you make in hub-client is visible to the +provider. Requires internet, and note that **your document text + code are +pushed to a public server** (fine for throwaway test code; use Option B if you +care). + +### Prerequisites + +- A built `q2` (the commands use `cargo run`, which builds on demand). +- `npm install` from the **repo root**. +- **A real engine matching the document:** `engine: jupyter` needs + `python3` + `jupyter` with a `python3` kernel; `engine: knitr` needs `R`. +- **Build the hub-client WASM once** (the dev server doesn't): + `cd hub-client && npm run build:wasm`. Without it the preview won't render. + +### Steps + +1. **hub-client** (terminal 1), on its default sync server: + ```bash + cd hub-client + npm run dev # do NOT set VITE_DEFAULT_SYNC_SERVER + ``` + Open http://localhost:5173, **create a new project**, and add a document + with an executable cell — front matter **must** declare an engine: + ``` + --- + title: demo + engine: knitr # or: jupyter + --- + + ```{r} + cat(1, 2, 3) + ``` + ``` + > **The `engine:` key is required.** Without it the cell renders as source, + > no Run button appears, and nothing executes. + +2. Get the project's **index-document id**: click **Share** in hub-client; the + id is the part after `#/share/` in the URL it shows. (Shortcut: you can pass + the *whole* Share URL to the provider — it extracts the id.) + +3. **provider** (terminal 2): + ```bash + # from the repo root + cargo run --bin q2 -- provide-hub --server wss://sync.automerge.org --allow-all --token dev + ``` + - `--token dev` skips the interactive OAuth bridge (the public server ignores + the bearer). **Local testing only.** + - `--allow-all` opts this machine in to running code for anyone in the + session. Without it the provider is *fail-closed* (connect + list + exit). + + You should see `Connected. N file(s)…` (N > 0) and `Execution ENABLED…`. + +4. In the browser: open the document, switch to the **preview**, and click + **Run**. The button shows **Executing…**; within a moment the preview shows + the executed output in place of the source. Edit the cell and **Re-run** to + see it update; a staleness note appears when the code changed since the last + run. + +## Option B — fully local (offline / private): `q2 hub` + +Runs a local sync server so nothing leaves your machine. The example project +(`project/hello.qmd`, `engine: jupyter`) and the `start-local-hub.sh` helper +are for this path. + +**Important:** `q2 hub --project ` (project mode) only serves *its own* +watched project. So you must **open that project via the Share URL** — do **not** +create a new project in hub-client (a fresh hub-client project isn't stored by a +project-mode hub, and the provider will see `0 file(s)`). + +```bash +cd claude-notes/hub-execution-e2e +./start-local-hub.sh # builds q2, starts the no-auth hub, prints the URLs +``` + +It prints the project's index id (from `GET /health`) and the exact commands +for the other two terminals: + +- **hub-client**, pointed at the local hub: + ```bash + cd hub-client + VITE_DEFAULT_SYNC_SERVER=ws://127.0.0.1:3031 npm run dev + ``` + Then open the printed **Share URL** (`…/#/share/?server=ws://127.0.0.1:3031&file=hello.qmd&name=Local%20demo`) — this opens the hub-owned project, not a new one. +- **provider**: + ```bash + cargo run --bin q2 -- provide-hub --server ws://127.0.0.1:3031 --allow-all --token dev + ``` + +`q2 hub` runs with **no auth** (no `--oidc-client-id`), and hub-client runs with +auth off because `VITE_GOOGLE_CLIENT_ID` is unset — so there's no login screen. + +(If you'd rather create projects freely in hub-client while staying local, run a +general relay instead of project mode: `q2 hub --no-project --port 3031`. It +stores/relays arbitrary docs the way `sync.automerge.org` does.) + +## Troubleshooting + +- **Provider prints `0 file(s)` / `project discovery failed: … No such file`:** + the provider can't see the project's files. In Option B this means you created + a *new* project instead of opening the hub-owned one via the Share URL (a + project-mode hub only serves its own project). Use the Share URL, or switch to + Option A / `q2 hub --no-project`. +- **No Run bar, only "Executor online" (or nothing):** the document has no + executable cell — check the `engine:` front-matter key and that the fence is + ```` ```{r} ```` / ```` ```{python} ````, not ```` ```r ```` or ```` ```{.r} ````. +- **No "Executor online" at all:** the provider isn't connected to the *same* + sync server as hub-client, or against a different index id. Confirm the + provider printed `Execution ENABLED` and `N file(s)` with N > 0. +- **Run does nothing / error in the bar:** the engine isn't installed or failed; + the provider terminal logs `exec request failed …`. Sanity-check the engine + standalone: `cargo run --bin q2 -- render .qmd --to html` should contain + the computed output. +- **Preview blank / won't render:** build the WASM — `cd hub-client && npm run build:wasm`. + +## Notes & caveats + +- **Single executor.** Two `provide-hub` processes on one project both execute + every request (v1 limitation; see the plan's "Known limitations"). +- **`--allow-all` is the only mode wired today.** The safer provider-only + default (only your own requests run) needs Phase 5; here any peer can trigger + execution — fine for a solo test. +- **Every run creates a fresh capture doc** (always-fresh execution); old + capture docs accumulate until server-side GC exists (Phase 5/6). +- Design + phase details: `claude-notes/plans/2026-06-29-remote-execution-provider.md`. + +### Verified directly while writing this + +- Provider dials `wss://sync.automerge.org` (TLS) and syncs; an absent doc + returns a fast "not found" (no hang). +- Provider against a hub-owned local project lists its files; the Jupyter/knitr + engines execute on this machine (`q2 render`, same registry the provider uses). +- The provider execute loop (receive `exec/request` → run engine → write capture + back) is covered by `crates/quarto-hub-provider/tests/integration/execute.rs`. +- The browser Run-click is the manual last mile this harness drives. diff --git a/claude-notes/hub-execution-e2e/project/.gitignore b/claude-notes/hub-execution-e2e/project/.gitignore new file mode 100644 index 000000000..e86a93bcb --- /dev/null +++ b/claude-notes/hub-execution-e2e/project/.gitignore @@ -0,0 +1,4 @@ +# Hub project-mode data (index doc + synced file docs) is generated at runtime. +.quarto/ +*.html +*_files/ diff --git a/claude-notes/hub-execution-e2e/project/hello.qmd b/claude-notes/hub-execution-e2e/project/hello.qmd new file mode 100644 index 000000000..26293fccc --- /dev/null +++ b/claude-notes/hub-execution-e2e/project/hello.qmd @@ -0,0 +1,24 @@ +--- +title: Local execution demo +engine: jupyter +--- + +This document has an executable Python cell. When a `q2 provide-hub` executor +is connected, the preview shows a **Run** button; clicking it runs the code on +the executor's machine and splices the output back into every collaborator's +preview. + +The `engine: jupyter` line in the front matter above is what makes the cell +executable — without an `engine:` key the cell renders as source only. (For R, +use `engine: knitr` and an `{r}` cell instead.) + +```{python} +2 + 3 +``` + +You can add more cells and re-run: + +```{python} +import sys +f"python {sys.version_info.major}.{sys.version_info.minor} on the executor" +``` diff --git a/claude-notes/hub-execution-e2e/project/r-demo.qmd b/claude-notes/hub-execution-e2e/project/r-demo.qmd new file mode 100644 index 000000000..f01ed0bdd --- /dev/null +++ b/claude-notes/hub-execution-e2e/project/r-demo.qmd @@ -0,0 +1,17 @@ +--- +title: R execution demo +engine: knitr +--- + +An R cell executed via the connected `q2` provider (engine: knitr). + +```{r} +1 + 1 +``` + +A second cell to confirm output splicing: + +```{r} +cat("hello from", R.version.string, "\n") +sum(1:10) +``` diff --git a/claude-notes/hub-execution-e2e/start-local-hub.sh b/claude-notes/hub-execution-e2e/start-local-hub.sh new file mode 100755 index 000000000..1c21877c1 --- /dev/null +++ b/claude-notes/hub-execution-e2e/start-local-hub.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Start a local, no-auth Quarto hub watching the example project, then print +# the project's index-document id and the exact URLs/commands for the other +# two processes (hub-client dev server + q2 provide-hub executor). +# +# See README.md in this directory for the full walkthrough. +# +# Usage: +# ./start-local-hub.sh # hub on port 3031 +# PORT=4000 ./start-local-hub.sh + +set -euo pipefail + +PORT="${PORT:-3031}" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +PROJECT="$HERE/project" +Q2="$REPO/target/debug/q2" + +echo "Building q2 (if needed)…" +( cd "$REPO" && cargo build --bin q2 ) + +echo "Starting hub on 127.0.0.1:$PORT (no auth), watching $PROJECT …" +"$Q2" hub --project "$PROJECT" --port "$PORT" & +HUB_PID=$! +trap 'kill "$HUB_PID" 2>/dev/null || true' EXIT + +# Wait for the hub to answer /health, then read the project's index doc id. +until curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; do + sleep 0.3 +done +ID="$(curl -s "http://127.0.0.1:$PORT/health" \ + | sed 's/.*"index_document_id":"\([^"]*\)".*/\1/')" + +cat <` + (`EngineCapture { engine_name, input_qmd, result }`, + `crates/quarto-trace/src/lib.rs:186`). +2. It serializes them to gzipped JSON and stores them as a **separate + samod/automerge binary document** (`create_binary_document(.., + CAPTURE_MIME_TYPE)`, written by `write_capture_doc` in + `crates/quarto-preview/src/capture_driver.rs:326` and + `re_execute.rs:337`). One binary doc per `.qmd` with code cells. +3. It writes a **pointer** into the project IndexDocument's `captures` + sidecar: `CaptureRef { capture_doc_id, staleness, state, + last_error }` keyed by the source file's relative path + (`crates/quarto-hub/src/index.rs:7-17,68-74,272`; TS mirror in + `ts-packages/quarto-automerge-schema/src/index.ts:40-63`). +4. Both docs sync to the browser over `/ws`. The SPA observes the + sidecar via `onCapturesChange` + (`ts-packages/quarto-sync-client/src/client.ts:352-377`), fetches + the binary doc by id (`getBinaryDocById`, `client.ts:1102`), and + threads the gzipped JSON into the WASM renderer + (`render_page_for_preview(path, grammars, captureGzJson)`, + `q2-preview-spa/src/PreviewApp.tsx:1030-1059`). +5. WASM splices the captured output into the live-edited AST via + `CaptureSpliceStage` (`crates/quarto-core/src/engine/capture_splice.rs`, + wired in `crates/wasm-quarto-hub-client/src/lib.rs:1201-1309`). + Engines never run in the browser; the capture is treated as an + AST-level "recipe" matched by `(content-hash, occurrence-index)` + so it survives live prose edits (see + `claude-notes/plans/2026-05-18-q2-preview-project-replay-engine.md`). + +**So "deposit results into automerge in a way visible to other +players" is exactly what the CaptureRef sidecar + capture binary doc +already do.** The hub relays both to every connected peer. The +document-bloat instinct is also already handled at the *file* level: +captures live in their own binary docs, never inside the (frequently +edited) file documents. + +What is **not** reusable as-is, and is the real work of this feature: + +- **The execution *trigger*.** In `q2 preview` the SPA POSTs + `/api/preview/re-execute` to the *same loopback process* that owns + the repo (`crates/quarto-preview/src/re_execute.rs:102`). In a + shared hub the executor is a *remote peer*; there is no loopback + HTTP between a player's browser and the volunteer's `q2`. The + trigger must travel **through automerge** (ephemeral message or a + persisted request entry). +- **The executor's *role*.** In `q2 preview` the `q2` process *is* + the hub: it owns the samod `Repo` and accepts inbound connections. + The new executor must instead be a samod **client peer** that + *dials out* to a remote hub and `find()`s the existing index doc. +- **Auth.** The remote hub (quarto-hub.com) requires a Bearer JWT on + the WS upgrade. Today only the TS sync client can attach it; the + Rust samod dialer cannot (see "Auth" below). +- **hub-client capture consumption.** hub-client currently ignores + the `captures` sidecar and renders source-only. It must learn to + consume captures the way q2-preview-spa does. +- **Capture retention.** An ephemeral preview repo can leak orphaned + capture docs freely. A persistent project cannot — and samod + exposes **no document-delete API** (`Repo` has only `create`, + `find`, `dial_websocket`). See "Capture retention" below. + +## Current architecture (verified against the code, 2026-06-29) + +### Two React apps, shared runtime + +| | `hub-client/` | `q2-preview-spa/` | +|---|---|---| +| Role | Full collaborative editor (Monaco, sidebar, presence, auth) | Minimal preview embedded in `q2` binary | +| Hub | persistent `quarto-hub.com` | local ephemeral hub from `q2 preview` | +| Storage | IndexedDB | memory | +| Auth | HttpOnly cookie (browser) | none (loopback) | +| Execution | **none — source-only render** | consumes captures, splices output | +| WASM | `crates/wasm-quarto-hub-client` (shared) | same | +| Shared TS | `@quarto/preview-runtime`, `@quarto/preview-renderer`, `@quarto/quarto-sync-client`, `@quarto/quarto-automerge-schema` | same | + +### automerge document model + +- `IndexDocument { files: Record, version, identities, + captures: Record }` + (`ts-packages/quarto-automerge-schema/src/index.ts:58-63`; + `CURRENT_SCHEMA_VERSION = 2`). The "VFS" at the automerge layer is + just `files: path → docId`; each file is its own document + (`TextDocumentContent { text }` or `BinaryDocumentContent { + content, mimeType, hash }`). +- The Rust hub uses **samod** (quarto-dev fork of automerge-repo, + `samod 0.10`). `samod::Repo` is symmetric: it can `accept` inbound + *and* `dial_websocket(url, backoff)` outbound + (`crates/quarto-hub/src/peer.rs:18-50`, used by `q2 hub --peer`). + +### Engines are Rust-native (this shapes everything) + +`KnitrEngine`/`JupyterEngine` are `#[cfg(not(target_arch = +"wasm32"))]` (`crates/quarto-core/src/engine/registry.rs:52-65`). +`record_capture` (`crates/quarto-core/src/engine/preview_record.rs:130`) +and the disk cache `record_capture_cached` +(`crates/quarto-preview/src/cache.rs:151`) are Rust. **Real execution +can only happen in a native Rust process.** The browser/WASM side only +*replays* captures. + +### Auth (TS-only today) + +OAuth 2.0 Authorization-Code + PKCE with an RFC 8252 loopback +redirect, against Google; implemented entirely in +`ts-packages/quarto-hub-mcp/src/auth/`. Tokens live in the **OS +keyring** (`@napi-rs/keyring`, service `dev.quarto.hub-mcp`, account +`:`), **shared across the `q2 mcp` and `npx` +channels**. The TS sync client attaches the Bearer on every (re)connect +via `NodeWebSocketClientAdapter` (`getBearer()` → +`Authorization: Bearer `). The Rust samod dialer **cannot set +headers** — joining an *authenticated* hub from Rust needs a custom +`BearerDialer` on samod's public `Dialer` trait (the +`2026-06-11-q2-mcp-hub-auth.md` plan judged this "feasible, no fork +changes" but deliberately chose the TS launcher to avoid a second +auth/threat-model surface). + +### `q2 mcp` is the closest precedent + +`q2 mcp` is a **thin Rust launcher** (`crates/quarto-mcp-launcher`) +that discovers Node, extracts an embedded esbuild bundle, injects +compiled-in OAuth client id/secret + default server, and `exec`s a +Node process that reuses `@quarto/quarto-sync-client` + +`@quarto/hub-mcp` auth to join a project's automerge session. It does +**not** execute code; it manipulates files. + +## Target architecture + +``` + ┌─────────────┐ automerge sync over /ws (Bearer auth) ┌──────────────────────┐ + │ Player A │◄──────────────── quarto-hub.com ──────────────────►│ q2 execution provider │ + │ (hub-client │ index doc + file docs + capture docs │ (NEW subcommand) │ + │ browser) │ │ │ + │ │ ──(1) "execute foo.qmd" request──────────────────► │ watches for requests │ + │ │ ◄─(2) capability: "executor online: knitr,jupyter" │ runs engines (Rust) │ + │ consumes │ ◄─(3) CaptureRef sidecar update + capture binary ─ │ writes capture doc + │ + │ captures │ doc (EXISTING transport) │ sidecar (EXISTING) │ + └─────────────┘ └──────────────────────┘ +``` + +Steps (2) and (3) reuse existing machinery. Step (1) and the executor's +client role are new. + +### Reuse map + +| Concern | Reuse | New | +|---|---|---| +| Serialize execution result | `EngineCapture`, `write_capture_doc`, `create_binary_document` | — | +| Deposit result in automerge | `CaptureRef` sidecar + capture binary doc | retention/GC policy | +| Run engines | `record_capture` / `record_capture_cached`, `EngineRegistry` | engine availability handshake (announce which engines this host has) | +| Join the session | samod `Repo` + `dial_websocket` | `BearerDialer` for auth; client-peer bootstrap (no inbound acceptor) | +| Auth | `q2 mcp` OAuth/keyring (TS) | token bridge to Rust *or* Rust OAuth port (decision below) | +| Trigger execution | — | ephemeral request channel + capability announcement | +| Consume captures in the editor | q2-preview-spa's `onCapturesChange`→`getBinaryDocById`→`render_page_for_preview` | port the same into hub-client | +| Request execution from the editor | q2-preview-spa `StaleCaptureOverlay` POST | replace POST with an automerge ephemeral send | +| Clear results (D6) | Rust `IndexDocument::remove_capture` (map-key delete) | TS `SyncClient.clearCapture`/`clearAllCaptures` + hub-client affordance | + +## Decisions locked (2026-06-30) + +All of D1–D6 and open questions 6–8 were resolved with the user on +2026-06-30. Per-decision detail is inline below; the short form: + +- **D1 — Hybrid (C).** Node owns auth; Rust owns sync + execution + + capture-writing via a `BearerDialer`. +- **D2 — Hybrid request channel.** Ephemeral "execute now" + ephemeral + capability beacon; persisted `CaptureRef.state`/`staleness` for + durable status. Beacon liveness timeout = **1.5 × refresh interval**. +- **D3 — Content-addressed + dedup** now; capture docs **excluded from + zip export** (treated as cache); a real GC design follows + immediately after. +- **D4 — Surface to all, allow all.** Providing execution implicitly + extends it to every player. Optional **owner-only** locked-down mode + (only the providing user's actor id may request) as a follow-on. +- **D5 — Heartbeat claims + cooperative `--force` takeover.** Stale + (no-heartbeat) claims auto-reclaim; a live claim needs `--force`, and + the displaced executor stands back. +- **D6 — Per-doc clear first.** Clear-all deferred. Confirmation UX + + in-flight race handling settle when the executor lands (Phase 4). +- **Q6 — quarto-hub.com only for v1**; mechanism is engine-agnostic. +- **Q7 — working name `q2 provide-hub`** (avoid "connect": Posit + Connect collision). Final naming TBD; alternatives below. +- **Q8 — per-doc clearing first.** + +## Key design decisions + +### D1 — Where does the executor live: native Rust, TS launcher, or hybrid? — DECIDED: hybrid (C) + +**Decision (2026-06-30): Option C (hybrid).** Node owns auth only; +Rust owns sync + execution + capture-writing via a `BearerDialer`. + +This is the central decision; everything else flows from it. + +- **Option A — Native Rust provider.** New `q2` subcommand opens a + local samod `Repo` (memory/temp storage), dials the remote hub with + a `BearerDialer`, `find()`s the index doc, watches for requests, + runs engines in-process, writes capture doc + sidecar with the + *existing Rust functions verbatim*. Self-contained single binary; no + Node. Cost: must obtain/refresh the Bearer token in Rust (port the + OAuth loopback+PKCE+keyring+refresh, or read the existing keyring + entry — fragile w.r.t. refresh) and implement `BearerDialer`. +- **Option B — TS launcher (like `q2 mcp`).** Node holds the + authenticated connection (reuse everything), detects requests, then + shells out to `q2` to run engines and produce a capture, and writes + the capture doc + sidecar **in TS** (duplicating Rust's + `write_capture_doc`/`set_capture` against the TS schema). Cost: + splits capture-writing logic across two languages; the engine + invocation becomes an awkward subprocess boundary. +- **Option C — Hybrid (recommended starting point).** Node owns + *auth only* (reuse OAuth/keyring/refresh untouched) and mints/hands + a fresh Bearer to a Rust process; Rust owns *sync + execution + + capture-writing* natively via a `BearerDialer`. Minimizes new code: + no Rust OAuth port, no TS duplication of capture-writing. New Rust: + `BearerDialer` + request-watch loop + capability announce. New + glue: token hand-off (env var / fd / short-lived local socket) and + refresh propagation. + +Recommendation: **C** for the first cut — it keeps execution and the +already-working capture transport entirely in Rust while reusing the +entire TS auth surface. Revisit a fully-native A later if the Node +dependency is unwanted. + +### D2 — Execution-request channel: ephemeral vs persisted + +**Decision (2026-06-30): hybrid.** Ephemeral "execute now" nudge + +ephemeral capability beacon; persisted `CaptureRef.state`/`staleness` +for durable status (as preview already does). + +The user specifically asked about automerge **ephemeral messages**. +Findings: + +- Ephemeral is **proven** (hub-client presence: + `handle.broadcast(msg)` + `handle.on('ephemeral-message')`, + relayed by samod to all peers subscribed to that doc, **including + the server peer**). It is **per-DocHandle**, **best-effort**, and + **not persisted**. +- The sync client currently exposes only **per-file** handles + (`getFileHandle(path)`); the **index** `DocHandle` is internal + (`state.indexHandle`, `client.ts:174`) and would need a new exposed + method to broadcast on. + +Trade-off: + +- **Ephemeral request** ("please execute foo.qmd now"): low latency, + zero document churn — but **lost if no executor is connected at + send time** (no durability). Good for "Run" button semantics where + the user can retry. +- **Persisted request** (write an intent into the index doc, e.g. + bump a `CaptureRef.requestedAt` or a `requests` map): survives + executor reconnect, naturally deduped by CRDT — but adds index-doc + history churn and needs cleanup. + +Likely answer: **hybrid** — ephemeral for the live "execute now" +nudge and for **capability/liveness** ("an executor is online with +engines X, Y"), plus the *existing persisted* `CaptureRef.state` +(`idle`/`running`/`error`) and `staleness` for durable status the way +preview already does. Capability announcement is inherently ephemeral +(presence-like): the executor periodically broadcasts +`{ executor: true, engines: [...], actorId }`; editors show "Run" +affordances only while a capability beacon is live. + +**Capability beacon liveness (decided 2026-06-30).** The executor +re-broadcasts the beacon every `BEACON_INTERVAL`; an editor marks the +executor offline if no beacon arrives within +`BEACON_TIMEOUT = 1.5 × BEACON_INTERVAL`. The 1.5× factor absorbs CRDT +propagation latency and avoids flicker, while staying tight enough that +a genuinely-disconnected provider disappears quickly (it tolerates a +late beacon, not a fully-dropped one — by design, per the user: a dead +provider should not linger as "online"). Proposed starting values +**`BEACON_INTERVAL = 3 s`, `BEACON_TIMEOUT = 4.5 s`** (both tunable; +the invariant `TIMEOUT = 1.5 × INTERVAL` is the contract, the absolute +numbers are not). The beacon carries `{ actorId, engines: [...], +generation }` so editors can both (a) gate "Run" affordances on +liveness and (b) show *which* engines are serviceable (D-Q6: +engine-agnostic mechanism, engine-specific *availability*). Open: which +DocHandle carries the beacon — the index handle (needs the new exposed +broadcast method) is the natural project-scoped channel; alternatively +a convention on a well-known per-file handle. Lean index-handle. + +### D3 — Capture retention in long-lived projects — DECIDED: content-addressed dedup, GC next + +**Decision (2026-06-30): content-addressed + dedup now**, with capture +docs **excluded from the project zip export** (treated as cache, never +"real" project content). A proper GC design is the **immediate +follow-on** (the user expects to want it right after) — file it as a +linked strand once this lands. + +`q2 preview` creates a **new** capture binary doc on every +re-execute and orphans the previous `DocumentId` +(`re_execute.rs:144-156`). Fine for an ephemeral repo; a persistent +project would accumulate orphans **and samod has no public +document-delete API**. Options: + +- **Content-addressed captures.** Key the capture doc by a hash of + its bytes (or of `input_qmd`); reuse the existing doc when the hash + matches (dedup), so re-running an unchanged doc creates nothing new. + Bounds growth to "one live capture per distinct result," but stale + results still linger. +- **In-place mutation of one capture doc per file.** Keep a single + `captureDocId` per path and overwrite its `content` on re-execute. + Avoids orphans, but an automerge binary doc's **history** grows + with each overwrite (the very bloat we want to avoid). Mitigate + only if samod gains history compaction. +- **Server-side GC.** quarto-hub.com (the persistent server, which + *does* own its storage) prunes capture docs not referenced by any + index `CaptureRef`. Needs a server feature + a safe "unreferenced" + definition across branches/versions. + +Recommendation: start with **content-addressed + dedup** (no +orphan-on-unchanged, simple, client-only), and file a follow-up for +server-side GC of truly-unreferenced capture docs. Confirm with the +user whether capture docs should be **excluded from the project zip +export / treated as cache** so they never become "real" project +content. + +**Separate two concerns that this plan previously conflated.** There +is (a) *removing the reference* — taking a document back to its +pre-captures effective state — and (b) *reclaiming storage* — actually +freeing the orphaned binary doc bytes. (a) is a user-facing affordance +(see D6) and is cheap and fully supported. (b) is the hard +server/samod-level GC problem above. The user-facing "clear" solves +(a); it does **not** require (b) to work — a cleared document renders +source-only immediately regardless of whether the bytes are ever +reclaimed. + +### D4 — Authorization model (who may execute, who may request) — DECIDED: surface to all, allow all; owner-only as follow-on + +Running arbitrary `{r}`/`{python}` from a shared document is **remote +code execution on the volunteer's machine**. + +**Decision (2026-06-30):** + +- The executor **opts in per project/session** (explicit `provide-hub` + invocation, never automatic). +- **All players may request execution by default.** Providing + execution implicitly extends that capability to everyone in the + session — a user who runs `provide-hub` is understood to be offering + their machine to the whole room. We **surface to all players** that + "code from this document runs on ``'s machine" (a visible + trust banner / indicator, not buried). +- **Optional owner-only locked-down mode (follow-on).** A flag (e.g. + `--owner-only`) restricts requests to *the providing user's own actor + id*. **This is knowable:** under auth the hub derives a per-project + actor id `HMAC-SHA256(server_secret, sub ‖ project_id)`, exposed at + `GET /auth/actor?project=` (`crates/quarto-hub/src/server.rs:781-806`, + `auth.rs:729`). It is **stable across the user's devices/sessions** + and unique per project. So the executor knows its own actor id, every + request carries the requester's actor id, and owner-only is simply + `request.actorId == self.actorId`. Same user on a second device still + matches (same actor id) — the intended semantics. + +Default-open is the v1 posture; owner-only is a small additive gate +once the request channel carries actor ids (it does, for D5's claim +model anyway). + +### D5 — hub-client capture-consumption UX + +hub-client renders source-only today. Bringing in captures means: + +- Wire `onCapturesChange` → `getBinaryDocById` → + `render_page_for_preview(captureGzJson)` into hub-client's + `ReactPreview` (port from `q2-preview-spa/PreviewApp.tsx`). +- Add a "Run" / "Re-execute" affordance that emits the D2 request and + reflects `CaptureRef.state` (`running`/`error`) + `staleness` and + the D2 capability beacon (disabled when no executor is online). +- Decide multi-executor behavior (more than one volunteer online): + see the claim model below. + +**Multi-executor claim model (decided 2026-06-30): heartbeat claims + +cooperative `--force` takeover.** First-claim-wins alone has the +failure the user flagged — if the claiming executor dies mid-run, +everyone else is blocked indefinitely. The fix has two parts: + +- **Heartbeat staleness handles the offline case automatically.** A + claim is a CRDT entry (per-doc or per-request) carrying + `{ actorId, generation, claimedAt, heartbeatAt }`. The owning + executor refreshes `heartbeatAt` on the same cadence as the + capability beacon. Any executor that sees a claim whose `heartbeatAt` + is older than `CLAIM_TIMEOUT` (reuse the `1.5 × interval` rule) + treats it as abandoned and may re-claim — **no `--force` needed for + the common "provider went offline" case.** This directly answers the + user's "long delays if an executor goes offline" worry. +- **`--force` is the escape hatch for an *alive-but-stuck* executor.** + A new executor invoked with `--force` writes a claim with a higher + `generation`. Existing executors **watch their own claim** and, on + seeing a live claim with a higher generation for the same doc, + **voluntarily stand back** (abort/skip the run, stop heartbeating + that claim). This is a *cooperative* yield — safe because every + executor is our own trusted binary, not a security boundary. A + malicious/stale peer ignoring the yield is out of scope (the trust + model is D4: whoever provides execution is trusted by the room). +- **Feasibility:** yes. It is the same shape as the existing + process-wide `IN_FLIGHT` in-flight guard (`re_execute.rs:49`), + lifted from one process into a CRDT claim every executor observes, + plus a generation counter for force and a heartbeat for liveness. +- **Open detail:** claim granularity — per-document (simpler; one + executor "owns" a doc at a time) vs per-request (finer; lets two + executors service two different docs concurrently). Lean + **per-request claim, keyed by (path, request-generation)** so + independent docs never block each other, with the doc-level beacon + separate from the per-request claim. + +### D6 — User-facing "clear execution results" affordance — DECIDED: per-doc first + +**Decision (2026-06-30): ship per-document clear first** (sub-decision +1); project-wide "clear all" deferred. Sub-decisions 2 (confirmation +UX) and 3 (in-flight race) **settle when the executor lands (Phase 4)** +— until then there is no executor to race with, and a hand-injected +capture can simply be cleared. Recommended landing positions when we +get there: confirmation prompt = **yes** (clearing affects all +players); in-flight race = **write-if-not-cleared** on the executor +side (the principled fix), falling back to "accept + document" if it +proves fiddly. + +A user must be able to **remove** the executed output from a +document's preview *without replacing it* — returning the document to +its pre-`captures` effective state. This is semantically distinct from +the two existing operations: re-execute *replaces* a capture, and +`staleness` *keeps* the capture but flags it outdated. "Clear" removes +it entirely; the editor then falls back to source-only rendering (the +splice has nothing to apply). + +**Data-level mechanism (small, already half-built).** Clearing is a +single automerge **map-key delete** on the IndexDocument's `captures` +map — *not* a document deletion. This sidesteps samod's missing +doc-delete API entirely: we never delete the binary doc, only the +`CaptureRef` reference pointing at it. + +- Rust: `IndexDocument::remove_capture(path)` already exists + (`crates/quarto-hub/src/index.rs:307`, `tx.delete(captures_obj, + path)`), used only by tests today. +- TS: the sync client currently has **no** capture-mutation API at all + (it only *reads* via `onCapturesChange`). The one new piece is a + typed `SyncClient` method, e.g. `clearCapture(path)` / + `clearAllCaptures()`, that mutates the already-held internal index + handle: `state.indexHandle.change(d => { delete d.captures?.[path] })`. + +**Key property: clearing needs no executor and no server round-trip.** +Unlike re-execute (which requires a native engine), clearing is a pure +CRDT mutation any peer can perform directly — it works even when no +volunteer `q2` is connected. This makes the affordance a low-risk, +executor-independent deliverable that can ship **before** the executor +work (it pairs naturally with Phase 1's capture-consumption port). + +**Sub-decisions to settle:** + +1. **Granularity.** Per-document "clear results" and a project-wide + "clear all results"? (Recommend both — per-doc in the preview + toolbar, clear-all in a project menu.) +2. **Collaborative semantics.** The `captures` sidecar is shared, so + one user clearing removes executed output **for every player** and + for every open tab. Likely desired ("clean up the document"), but + warrants a confirmation prompt naming that it affects collaborators. +3. **In-flight interplay.** If a capture is `state: running` when a + user clears it, the executor may finish afterward and re-create the + entry — silently re-adding output the user just removed. Options: + (a) accept + document the race for v1; (b) clearing also broadcasts + a cancel on the request channel (D2); (c) the executor re-checks + the sidecar still wants results before writing (write-if-not-cleared). + Recommend (c) as the principled fix, (a) acceptable for a first cut. +4. **History caveat (be honest in the UX copy).** Automerge history is + append-only: the old `captureDocId` remains in the index doc's + history and the binary-doc bytes remain in storage until D3's + server GC runs. "Clear" restores the *effective/visible* state, not + the byte-level history. The user sees a clean document; true + reclamation is the separate (b) concern in D3. + +## Open questions + +D1–D6 and Q6–Q8 are **resolved** (see "Decisions locked" above). What +remains genuinely open and needs settling *during* the relevant phase: + +1. **Auth bridge (D1=C) [Phase 3]**: token hand-off mechanism from + Node→Rust — env var (simple, but token in process env), inherited + fd / pipe, or a short-lived local socket. Plus how refresh + propagates (Node refreshes; Rust must pick up the new token before + the old one expires on reconnect). Lean fd/pipe for the secret. +2. **Beacon/claim channel [Phase 2]**: carry the beacon + claim on the + index `DocHandle` (needs a new exposed broadcast/subscribe method on + `SyncClient`, since only per-file handles are exposed today) vs a + convention on a well-known per-file handle. Lean index handle. +3. **Claim granularity [Phase 2/4]**: per-document vs per-request + (lean per-request keyed by `(path, generation)`; see D5). +4. **Naming (Q7) [Phase 0/3]**: working name `q2 provide-hub`; + alternatives `q2 provide`, `q2 provide-execution`, `q2 hub-provide`. + Avoid `connect` (Posit Connect collision). Final pick before the + subcommand is user-visible. +5. **Clear in-flight race (D6 sub-3) [Phase 4]**: confirm + write-if-not-cleared vs accept-and-document once the executor exists. + +## Phased plan (TDD) + +> No implementation starts until the user gives the explicit go-ahead. +> Phases below are a skeleton; each phase ends green and is verified +> end-to-end per CLAUDE.md. **Scope (Q6): quarto-hub.com only for v1; +> whole-project; mechanism engine-agnostic** (verify E2E with whichever +> engine the test host has — knitr per prior precedent). + +- **Phase 0 — Decisions.** ✅ D1–D6 + Q6–Q8 resolved 2026-06-30 (see + "Decisions locked"). Remaining to lock *in-phase*: the items in + "Open questions" (auth-bridge mechanism, beacon/claim channel + wire + format, final subcommand name). Working name: **`q2 provide-hub`**. +- **Phase 1 — Editor consumes captures + clear affordance (hub-client).** + Port capture-consumption from q2-preview-spa into hub-client; verify a + *hand-injected* capture doc + sidecar renders executed output in a + real browser session against a local hub. (No executor yet — proves + the consumption half end-to-end.) **Also lands D6's per-doc "clear + results":** add the `SyncClient.clearCapture` mutation + + toolbar affordance; verify clearing a hand-injected capture + returns the preview to source-only. Clearing is executor-independent, + so it is fully shippable in this phase. (Clear-all deferred.) +### Phase 1 — detailed checklist (TDD) + +Findings that shaped this (verified 2026-06-30): +- The inner WASM helpers `render_single_doc_to_response` / + `render_project_active_page_to_response` **already accept both + `captures` and `attribution_json`** and attach both on the same + `RenderToPreviewAstRenderer` (`lib.rs:1572-1577`). The capture-aware + entry `render_page_for_preview` hardcodes `attribution=None`; the + attribution entry `render_page_in_project_with_attribution` hardcodes + `captures=Vec::new()`. hub-client's main path uses the *latter*, so + it cannot consume captures today. +- preview-runtime's `setSyncHandlers` **already supports + `onCapturesChange`** and `getBinaryDocById` is already exported + (`ts-packages/preview-runtime/src/automergeSync.ts:42,68,116,200`). + hub-client's `App.tsx` simply never registers the handler. +- `IndexDocument::remove_capture` exists in Rust but there is **no TS + capture-mutation API** on `SyncClient` yet. + +- **1A — WASM: capture-aware attribution entry (Rust).** ✅ done. + - [x] Added `capture_gz_json: Option>` to + `render_page_in_project_with_attribution`; parses via + `parse_capture_from`; threads `captures` into both inner helpers + (replaced the two `Vec::new()` literals). Fixed the internal + `render_page_in_project` caller (4th `None`). + - [x] RED→GREEN via WASM vitest + `hub-client/src/services/captureSplice.wasm.test.ts`: single-file + q2-preview doc with one `{r}` cell + a hand-built capture whose + post-engine markdown carries a capture-only marker; asserts the + marker appears in the rendered AST. RED confirmed on current + code (3-arg entry ignores the 4th arg → no marker); GREEN after. + - [x] Coexistence test: capture splices even when an attribution + payload (`{}`) is also supplied (both attach on the same renderer). +- **1B — TS wrapper + binding type.** ✅ done. + - [x] Extended the `render_page_in_project_with_attribution` binding + type and the `renderPageInProjectWithAttribution` wrapper in + `ts-packages/preview-runtime/src/wasmRenderer.ts` with + `captureGzJson?: Uint8Array`. +- **1C — hub-client capture state (`App.tsx`).** ✅ done. + - [x] Added `captures: Record` state; registered + `onCapturesChange` in `setSyncHandlers`; passes `captures` to + ``. (Reset relies on `onCapturesChange` firing on + connect, matching the existing `identities` pattern.) +- **1D — hub-client ReactPreview consumes captures.** ✅ done. + - [x] Threaded `captures` App→Editor→PreviewRouter→ReactPreview + (mirroring `identities`). ReactPreview fetches the active doc's + capture bytes via `getBinaryDocById`, keyed on its `captureDocId` + (not content), and passes them to both + `renderPageInProjectWithAttribution` (4th arg) and + `renderPageForPreview` (slides). `captureBytes` added to the + render-trigger deps so a freshly-arrived/cleared capture + re-renders. + - [x] Integration test `ReactPreview.capture.integration.test.tsx`: + a capture in props is fetched by id and its exact bytes reach + the render call as the 4th arg; no-capture ⇒ no fetch, 4th arg + undefined. +- **1E — D6 clearCapture (sync client).** ✅ done. + - [x] Added `SyncClient.clearCapture(path)` → + `indexHandle.change(d => { delete d.captures[path] })`; exported + through preview-runtime (`automergeSync.ts`). + - [x] RED→GREEN unit tests in `client.test.ts`: `clearCapture` + removes the entry leaving siblings intact; no-op (no throw, no + sibling change) when the path has no capture. +- **1F — hub-client clear affordance (UI).** ✅ done. + - [x] `ClearCaptureControl` (presentational, injected `onClear`) shown + in the preview pane only when `captures[activeFile]` exists; a + two-step inline confirmation naming the collaborator-wide effect + (chosen over `window.confirm` for stylability + testability). + Wired into `Editor.tsx` → `clearCapture(path)`; minimal CSS in + `Editor.css`. + - [x] RED→GREEN component test + `ClearCaptureControl.integration.test.tsx` (5 cases: hidden + without capture / without path; shown with capture; confirm + calls `onClear(path)`; cancel does not). +- **1G — E2E (Playwright). ⏸ Deferred to Phase 4 (rationale below).** + - A faithful browser E2E needs a capture to exist in the live + automerge session. In the target architecture **only the Rust + executor writes captures** (`set_capture`); the browser solely + *reads* and *clears* them — there is deliberately no TS + `setCapture`. Injecting a capture in a Phase-1 Playwright test would + therefore require throwaway test-only scaffolding (a + `_seedCaptureForTesting` that fabricates a binary doc + sidecar), + which `claude-notes/instructions/coding.md` discourages. **Phase 4's + real executor makes this exact E2E faithful with zero scaffolding** + (executor writes a real capture → hub-client shows it → clear), so + the browser E2E is folded into Phase 4. + - [ ] (Phase 4) Browser E2E: executor writes a capture → hub-client + iframe shows `.cell-output` → "Clear results" → source-only. + - Phase-1 verification standing in for it (see "End-to-end evidence + (Phase 1)" below): the WASM test drives hub-client's **actual** + render entry with a **real** capture and asserts spliced output; + the React integration test covers the browser wiring + (props→fetch→render-call); the component test covers clear. +- **1H — verify.** ✅ TS suites green (hub-client unit 662 / integration + 83 / wasm 124; sync-client 107; preview-runtime 74). Rust workspace + untouched (only the out-of-workspace `wasm-quarto-hub-client` crate + + TS changed; `npm run build:wasm` + `npm run typecheck` green). Full + `cargo xtask verify` to run before the push request. + +### End-to-end evidence (Phase 1, 2026-06-30) + +Per CLAUDE.md's end-to-end rule. The strongest currently-faithful check +drives hub-client's **actual** render entry +(`render_page_in_project_with_attribution`) through the **real** WASM +pipeline with a **real** gzipped capture, and inspects the output AST. + +Invocation: +``` +cd hub-client && npx vitest run --config vitest.wasm.config.ts \ + src/services/captureSplice.wasm.test.ts +``` +Observed (spliced AST region around the capture-only marker +`SPLICEDOUTPUT42`, console-dumped during a one-off run): +``` +…["code-copy-outer-scaffold"],…["code-with-copy"],…"SPLICEDOUTPUT42"],"t":"CodeBlock"… +``` +The marker exists **only** in the capture's post-engine markdown, never +in the source doc — so its presence in the rendered AST proves the +capture's engine output was spliced in through the entry hub-client +uses. The no-capture baseline asserts the marker is absent, and a third +case asserts capture+attribution coexist. Browser wiring +(props→`getBinaryDocById`→render-call 4th arg) is covered by +`ReactPreview.capture.integration.test.tsx`; the clear affordance by +`ClearCaptureControl.integration.test.tsx`. The full in-browser E2E is +deferred to Phase 4 (see 1G). + +- **Phase 2 — Request + capability channel.** Implement the D2 channel + (ephemeral capability beacon + execute request). + +### Phase 2 — decisions locked (2026-06-30) + checklist (TDD) + +Resolved with the user: +- **Q-A — scope:** channel + `SyncClient` API + capability detection + + stub-responder tests. The user-facing **Run** affordance is deferred + to **Phase 4** (no executor exists yet, so a Run button would be + dormant). hub-client *does* consume beacons into a live-executor state + in this phase. +- **Q-B — claims (D5 heartbeat/`--force`/generation):** deferred to + **Phase 4** (claims only matter once a real executor can collide). +- **Q-C — carrier:** the **index `DocHandle`** (project-scoped), + exposed via `SyncClient.getIndexHandle()` (mirrors the existing + `getFileHandle()` ephemeral surface). Per-file handles would fragment + the channel. +- **Q-D — wire format** (cross-language contract; Rust executor mirrors + it in Phase 4): `kind`-discriminated, `exec/`-namespaced JSON on the + index handle's ephemeral channel: + - beacon → `{ kind: 'exec/beacon', actorId, engines: string[], generation }` + - request → `{ kind: 'exec/request', path, requestId, requesterActorId }` +- **Q-E — timing:** `BEACON_INTERVAL_MS = 3000`, + `BEACON_TIMEOUT_MS = 4500` (the locked `TIMEOUT = 1.5 × INTERVAL`). + +- **2A — `SyncClient.getIndexHandle()` + preview-runtime export.** ✅ done. + - [x] Returns `state.indexHandle` (or null); re-exported from + `automergeSync.ts`. RED→GREEN unit test: index handle null + before connect, the handle after. +- **2B — execution-channel wire format + pure helpers.** ✅ done. + - [x] `hub-client/src/services/executionChannel.ts`: message types + + `makeBeacon`/`makeExecuteRequest`, `parseExecMessage` + (validate/discriminate untrusted payloads), `applyBeacon`/ + `pruneExecutors` (live-executor map keyed on actorId, `1.5×` + staleness). RED→GREEN pure unit tests (13). +- **2C — execution-channel service (stateful).** ✅ done. + - [x] `createExecutionChannel({ getIndexHandle, onExecutorsChange, + now, ... })`: subscribes to index ephemeral messages → + live-executor set + prune timer; `requestExecution(path)` + broadcasts an `exec/request`. RED→GREEN stub-responder tests (5) + against a fake DocHandle (records broadcasts; injects messages): + beacon appears/expires; request shape round-trips through + `parseExecMessage`; self-beacon ignored; null when not connected. +- **2D — wire into hub-client (capability state, no Run UI).** ✅ done. + - [x] `useExecutionChannel(isOnline, indexDocId)` hook starts/stops the + channel with the connection/project and returns `liveExecutors` + (integration test: beacon→executor, teardown). App holds it and + passes `executorsOnline` to Editor, which shows a minimal + read-only "Executor online" bar (no Run button). Minimal CSS. +- **2E — verify.** ✅ hub-client unit 680 / integration 86; sync-client + 108; preview-runtime 74; `tsc -b` + vite build green; typecheck green. +- **Phase 3 — Rust client peer + BearerDialer (D1=C).** New subcommand + opens a samod client `Repo`, dials a remote hub with a `BearerDialer` + fed a token from the auth bridge, `find()`s the index doc. Verify it + joins an authenticated session and reads the file list. + +### Phase 3 — BearerDialer feasibility spike ✅ (2026-06-30) + +De-risked **before** committing to the plan (per user). **Verified +against the actual dependency** — note it's the **quarto-dev git fork +`samod@q2` (v0.9.0)**, not crates.io 0.10 — and the +`BearerDialer` approach is viable with **zero fork changes**: + +- `Repo::dial(backoff, Arc)` is **public** + (`samod/src/lib.rs:671`); `dial_websocket` is just a convenience + wrapper that constructs a `TungsteniteDialer` and calls it + (`samod/src/websocket.rs:223`). +- The `Dialer` trait (`samod/src/dialer.rs`) is public and its + `connect()` is documented as called "on the initial dial and on each + reconnection attempt after backoff" → a `BearerDialer::connect()` + can fetch a **fresh** token from the auth bridge on every (re)connect + (exactly the per-connect refresh D2/D1 want). +- `Transport` + `Transport::new(stream, sink)` are public + (`pub use transport::Transport`; `transport.rs:32`). `new` accepts a + `Stream, E>>` + `Sink, Error=E>`. +- The only non-public helper is the ~25-line `ws_to_bytes` + (`websocket.rs:91`) mapping tungstenite `Message` ↔ bytes; we + replicate it in our crate (filter Binary, drop Close/Ping/Pong, error + on Text). Trivial. +- Header injection is standard tokio-tungstenite: + `url.into_client_request()` → insert `AUTHORIZATION: Bearer ` → + `tokio_tungstenite::connect_async(request)`. Our crate uses its own + tokio-tungstenite internally (it only has to *produce* a samod + `Transport`), so there's **no tungstenite/http version coupling** + with samod beyond the `Transport` boundary. + +Recipe (mirrors `TungsteniteDialer::connect`, + auth header + fresh +token): +```rust +impl Dialer for BearerDialer { + fn url(&self) -> Url { self.url.clone() } + fn connect(&self) -> BoxFuture<'static, Result> { + let url = self.url.clone(); + let token_source = self.token_source.clone(); // async fresh-token getter + Box::pin(async move { + let token = token_source.fresh_bearer().await?; + let mut req = url.into_client_request()?; + req.headers_mut().insert(AUTHORIZATION, format!("Bearer {token}").parse()?); + let (ws, _resp) = tokio_tungstenite::connect_async(req).await?; + let (stream, sink) = ws_to_bytes_local(ws); // our replica + Ok(Transport::new(stream, sink)) + }) + } +} +// repo.dial(BackoffConfig::default(), Arc::new(BearerDialer::new(url, token_source))) +``` +Optional future cleanup: upstream a `dial_websocket_with_request` (or +`TungsteniteDialer::with_request`) into our fork to avoid the +`ws_to_bytes` replica — but that needs a fork bump, so not for v1. + +**Verdict: no plan change. Proceed with D1=C as designed.** + +### Phase 3 — decisions locked (2026-06-30) + checklist + +- **Auth-bridge hand-off (cross-platform):** **Node child's stdout/stdin + pipes**, not a Unix-only extra inherited fd. Topology: the Rust + `q2 provide-hub` process is the long-running parent; it spawns the + Node auth helper as a child (same direction as the `q2 mcp` + launcher). Node emits newline-delimited JSON Bearer tokens on + **stdout** (`{"type":"token","bearer":…,"expiresAt":…}`) on initial + auth + every refresh; logs/auth-URL go to **stderr**; Rust may write + `{"type":"refresh"}` to Node **stdin** to pull a token before a + reconnect. stdio pipes are identical on Windows/macOS/Linux, + process-private (not in argv/env), and stream refreshes. Reuses + `quarto-mcp-launcher` (Node discovery + bundle extraction) to spawn + the helper; the helper is thin (OAuth loopback + RefreshManager from + `quarto-hub-mcp`'s `auth/`, streaming Bearers — it does **not** + connect to the hub; that's Rust's job). +- **File materialization (for Phase 4 execution):** the executor + materializes the project to a **fresh temp dir from the VFS each + run** (clean, self-contained, safer). Large-file/perf optimizations + deferred until usage patterns are known. (Implemented in Phase 4; the + decision is recorded here.) +- **Subcommand name:** working name `q2 provide-hub` (final TBD; avoid + `connect` — Posit Connect collision). + +Phase 3 deliverable is **narrow**: join an *authenticated* hub, `find()` +the index doc, list the project files. No execution / beacon / temp-dir +materialization yet (Phase 4). Decomposed to de-risk the Rust sync path +before the Node auth helper: + +- **3A — `BearerDialer` + `TokenSource` (Rust).** ✅ done. + - [x] New crate `crates/quarto-hub-provider` with `BearerDialer` + (impl `samod::Dialer`; the spike recipe + an in-crate + `ws_to_bytes` replica via `inbound_to_bytes`/`outbound_to_ws`) + and a `TokenSource` trait (`fresh_bearer() -> BearerFuture`) + + `StaticTokenSource`. Uses tokio-tungstenite 0.27 (matches the + samod fork). + - [x] Unit tests (6): mapping (Binary↔bytes / drop Close-Ping-Pong / + error on Text) + the request carries `Authorization: Bearer + ` and rejects a token with illegal header bytes. +- **3B — Rust client peer joins + lists (dev token source).** ✅ done. + - [x] `join_and_list_files(JoinConfig, Arc)`: memory + `Repo`, `repo.dial(backoff, BearerDialer)`, `handle.established()` + with timeout, `IndexDocument::load` → sorted `get_all_files()`. + - [x] Integration test (`tests/integration/join.rs`): a bare samod + acceptor behind a real tungstenite ws server + a seeded index + doc; the provider connects **over the real `BearerDialer` + transport**, syncs, and lists the files. (No auth — the header + is sent and ignored; the authenticated-acceptance path is + `quarto-hub`'s `auth_bearer` tests + the Phase 3C real-binary + run.) Clippy `-D warnings` clean. +- **3C — `q2 provide-hub` subcommand + Node auth bridge.** + + Implementation notes (verified 2026-06-30): + - **Bearer = the OIDC `id_token`** (a JWT) from + `RefreshManager.getValidIdToken()` — byte-identical to what + `connection-manager.ts` sends and the hub validates. + - **Reuse** `auth/{credential-store,refresh-manager,oauth-config}.ts` + + `AuthToolsState` (the OAuth-loopback flow lives there, not in the + MCP shim; URL/logs already go to stderr). Boot order mirrors + `index.ts:185-265`: `resolveIssuer` → `authServer = () => + discoverAuthorizationServer(issuer)` → `loadOAuthConfigFromEnv` → + `new CredentialStore({issuer, clientId})` → `new RefreshManager` → + `new AuthToolsState({…, connectionManager: })`. `handleAuthenticate()` signs in; then stream + `getValidIdToken()`; on stdin `{"type":"refresh"}` call + `forceRefresh()`. + - **Env**: the auth code reads `QUARTO_HUB_MCP_CLIENT_ID` / + `QUARTO_HUB_MCP_CLIENT_SECRET` (issuer defaults to Google). The + launcher's `defaults::injections` already injects exactly those + (compiled-in when not in user env) — so spawning Node with the + injected env supplies the creds; dev sets them in their shell. + - **Bundle**: `scripts/bundle.mjs` is imperative esbuild (single entry + `src/index.ts` → `dist-bundle/index.mjs`). Add `src/auth-stream.ts` + as a **second** esbuild entry → `dist-bundle/auth-stream.mjs`; it + rides the existing `$QUARTO_HUB_MCP_EMBED_DIR` include_dir. + - **Spawn**: the provider reuses `quarto-mcp-launcher` + (`bundle::embedded_files`/`content_hash`, `cache::extract_and_lock`, + `node::find_node`, `defaults::injections`) to get a Node + the + extracted dir, then `tokio::process::Command` runs + `node /auth-stream.mjs` with piped stdin/stdout (holding the + `ExtractedBundle` keeps the lifetime lock — we spawn, not exec). + + Steps: + - [x] Testable TS protocol core `runTokenStream` (`auth-stream/protocol.ts`) + + 7 vitest cases (initial token, refresh, ignore junk, fatal initial + error, non-fatal refresh error). `auth-stream.ts` wires it to the real + auth boot (reuses `auth/*`; stub `connectionManager`; sign-in on + `ReauthRequired`). + - [x] `scripts/bundle.mjs`: shared esbuild options + a **second entry** → + `dist-bundle/auth-stream.mjs`. Bundle builds; the helper boots and + emits the correct error frame on stdout when creds are absent. + - [x] Rust `token_bridge.rs` (`NodeBridge`): reuses the launcher + (`embedded_files`/`content_hash`/`extract_and_lock`/`find_node`/ + `injections` — exposed via small new `pub use`s) to spawn + `node auth-stream.mjs` with piped stdio (stderr inherited for the + sign-in URL); a stdout-reader task parses frames into a token cache; + `impl TokenSource`. 4 unit tests on the frame parser. **Integration + test** spawns the *real* bundled helper with no creds and asserts the + bridge surfaces the error frame (ran, not skipped). `await`-holding-lock + fixed (async mutex on stdin). + - [x] `q2 provide-hub` clap subcommand: share-URL/index-doc-id + `--server` + → `NodeBridge` → `join_and_list_files`. 3 unit tests (share-URL parse, + server resolution); help renders; wired in `main.rs`. + - [ ] **E2E (real binary) — manual, can't automate interactive Google + OAuth.** `q2 hub` (auth on) + `q2 provide-hub ` → + browser sign-in → prints the file list. Coverage standing in for it: + `join.rs` (real BearerDialer sync+list) + `auth_bridge.rs` (real + Rust↔Node helper plumbing) + the protocol unit tests. (Note: like + `q2 mcp`, `q2 provide-hub` needs the hub-mcp bundle built — `cargo + xtask build-hub-mcp-bundle`; otherwise it errors at runtime and the + gated test skips.) +- **3D — verify.** ✅ `cargo xtask verify` green (Phase 3A/3B checkpoint); + 3C committed (`5b31827f`). Phase 3 complete. + +### Phase 4 — Execute-on-request (the payoff) — DESIGN (2026-06-30) + +The connected `q2 provide-hub` finally runs code. End-to-end target: a +player clicks **Run** → the editor broadcasts `exec/request` → the +provider materializes the project, runs the engines, writes the capture +doc + sidecar → every player's preview shows the executed output (the +Phase 1 consumption path). + +**Architecture (provider side, all native Rust — reuses Phases 1–3):** +1. After `join`, subscribe to the index `DocHandle` ephemeral channel + (the Phase 2 channel) and **broadcast the `exec/beacon`** every + `BEACON_INTERVAL`, advertising the host's *available* engines + (probe via the registry's `is_available` — `KnitrEngine`/`JupyterEngine`). +2. On an `exec/request { path }`: set `CaptureRef.state = running` + (durable status, Phase 1), then on a blocking worker: + a. **Materialize the project to a fresh temp dir from the VFS** — for + each path in `index.files`, `repo.find(doc_id)` → read `text` + (`TextDocumentContent`) or `content` (`resource::read_binary_content`) + → write under `/`. (A dedicated read-only materializer; + `sync_all_documents` is bidirectional + sync-state-heavy, so we + write a lean one reusing `resource::*`.) + b. `ProjectContext::discover(/)` → + `record_capture_cached(cache_dir, abs, project, runtime, registry)` + → `Vec` (the **same native engine path `q2 preview` + uses**). + c. Write the capture binary doc (`create_binary_document` + + `CAPTURE_MIME_TYPE` + `repo.create`) and `index.set_capture(path, + CaptureRef{ capture_doc_id, staleness:false, state:idle })` — the + exact functions `quarto-preview`'s `re_execute.rs` uses. A client + peer's `repo.create` + index mutation **sync to the hub and every + peer**, so the editor sees it via `onCapturesChange`. + d. On error: `CaptureRef.state = error` + `last_error` (Phase 1 + surfaces it). +3. **Retention (D3):** content-address the capture doc (hash its bytes / + `input_qmd`) and reuse an existing doc when unchanged, so repeated + runs of an unchanged doc don't orphan docs. (Server-GC of truly + unreferenced docs is a separate follow-up.) + +**hub-client (editor side):** +4. **Run UI** (deferred from Phases 1–2): a "Run" affordance — shown when + an executor beacon is live (`useExecutionChannel`) and the active doc + has executable cells — that calls `channel.requestExecution(path)`. + Reflect `CaptureRef.state` (running/error) + staleness (reuse the + q2-preview-spa `StaleCaptureOverlay` pattern). + +**Decisions locked (2026-06-30):** +- **D4 — provider-only by default, `--allow-all` to open up.** Default: + only the **providing user's own per-project actor id** may trigger + execution (same user across their devices — the per-project actor id + is stable per `(sub, project)`). The provider fetches its own actor id + from the hub's `GET /auth/actor?project=` (with its Bearer) and + honors an `exec/request` only when `requesterActorId == self.actorId`. + An affirmative **`--allow-all`** flag on `q2 provide-hub` opens + execution to **everyone with access to the document**. Noun is + **"provider"**, not "owner" (avoid confusion with document ownership). + This is the *safe default* — running `provide-hub` does not silently + expose your machine to the whole room. Consent is still surfaced (the + provider terminal states the mode; the editor's "Executor online" + badge shows availability). +- **D5 — single-executor v1; document the double-execution behavior.** + No claim/heartbeat/`--force` yet. **Two providers on one project would + *both* execute every request** (duplicate side effects), with the + CRDT sidecar converging to one capture and the other orphaned. True + single-execution mutual exclusion is *not* achievable peer-to-peer + (no atomic CAS across peers) — it needs the hub server as arbiter, + which is **deferred to Phase 6**. v1 assumes one provider per project; + the editor shows multiple beacons so the situation is visible. +- **#3 — split 4a/4b.** 4a = provider executes + writes the capture + (verifiable by a *scripted* request — no UI, fully testable against a + local hub) → 4b = hub-client Run button. +- **#4 — always force a fresh run.** Use the uncached `record_capture` + (never the staleness cache): code may have side effects, and we will + not be in the business of proving code is side-effect-free. (Note: this + means every Run creates a new capture doc and orphans the prior one — + reinforcing that D3 server-side GC is needed for long-lived projects; + Phase 5/6.) + +#### Phase 4a — authz scope (DECIDED 2026-07-01: mechanism-first, fail closed) + +The full D4 default (provider-only, gated on the provider's own +per-project actor id) needs the provider to learn that actor id from +`GET /auth/actor?project=` — but that endpoint is **cookie-only** +today (`server.rs:796` reads `cookie_token` exclusively) and the provider +authenticates with a **Bearer** id_token. Making provider-only real +therefore needs a hub-side change (teach `/auth/actor` to accept Bearer) +plus an HTTPS fetch from the provider. Per the phase list, "authorization +(D4)" already lives in **Phase 5**. + +**Decision (user, 2026-07-01): 4a is mechanism-first and fails closed.** +- 4a ships the execute loop + capability beacon + Rust wire-format mirror + + an `--allow-all` flag on `q2 provide-hub`. +- The execute loop takes a pluggable `AuthzPolicy`. In 4a it is one of + `AllowAll` (opened by `--allow-all`) or `Deny` (the default): **without + `--allow-all` the provider refuses every request and prints guidance** + (safe default; does nothing rather than exposing the machine). +- The scripted E2E runs with `AllowAll`. +- Real provider-only actor-gating (hub `/auth/actor` Bearer support + + provider HTTPS actor-id fetch + `ProviderOnly { self_actor_id }` policy) + lands in **Phase 5** with the consent UX. The `AuthzPolicy` enum is the + seam it slots into. + +#### Phase 4a — implementation checklist (TDD), crate `quarto-hub-provider` + +- [x] **4a-1 — exec wire-format mirror + CBOR (`exec_channel.rs`).** ✅ done. + `ExecMessage` internally-tagged enum + `to_cbor`/`parse_exec_message` + via ciborium; 6 unit tests green, incl. a CBOR-shape assertion proving + the bytes are a standard map with the exact camelCase keys the TS + `parseExecMessage` checks (ciborium ↔ cbor-x interop confirmed). + `ExecMessage` enum mirroring the TS contract (internally tagged on + `kind`; camelCase field renames). CBOR encode/decode via `ciborium` + (the browser's `DocHandle.broadcast` CBOR-encodes the payload with + cbor-x `{useRecords:false}` → standard CBOR maps, so ciborium + interops). `BEACON_INTERVAL`/`BEACON_TIMEOUT` consts. Unit tests: + round-trip; CBOR shape is a map with the exact keys the TS + `parseExecMessage` checks; junk/unknown-kind → None. +- [x] **4a-2 — VFS→temp-dir materializer (`materialize.rs`).** ✅ done. + `materialize_project(repo, index, dest)` reads each file doc (text via + `doc.text`, binary via `resource::read_binary_content`), `safe_join` + guards against `..`/absolute traversal, writes under `/`. + 2 unit tests (safe_join) + 1 integration test (text + nested binary → + on-disk bytes) green. +- [x] **4a-3 — execute loop + capability beacon (`execute.rs`).** ✅ done. + `AuthzPolicy` (`AllowAll`/`Deny`, seam for Phase 5 `ProviderOnly`), + `Provider` (Arc-shared) with `run` = concurrent beacon-broadcast + + ephemeral request-listen until a shutdown future fires; + `execute_document` (materialize → `ProjectContext::discover` → + **uncached** `record_capture` → `write_capture_doc` gzip+binary-doc → + `set_capture` idle; running/error status on an existing capture; + in-flight dedup; path-safety guard). Local `CAPTURE_MIME_TYPE` with a + cross-ref comment (avoids the heavy quarto-preview dep). `join` + refactored to return the live `(Repo, IndexDocument)`. 4 unit tests + (authz gating, path safety, engine list). +- [x] **4a-4 — integration test (`tests/integration/execute.rs`).** ✅ done. + Bare samod acceptor + passthrough-engine qmd; the editor side + re-broadcasts an `exec/request` on the index handle; the provider + (`AllowAll`) materializes, runs the passthrough engine, and writes a + capture binary doc + `idle` sidecar that **syncs back to the server** + (verified by gunzipping the synced capture doc → `EngineCapture` with + `engine_name == "test-passthrough"`). Second test: `Deny` writes no + capture over a 3 s broadcast window. Both green. +- [x] **4a-5 — wire `--allow-all` into `q2 provide-hub` + verify.** ✅ done. + `--allow-all` clap flag; fail-closed default (connect, list files, print + guidance, exit); with the flag the command builds a `Provider` (beacon + actorId = samod peer id) and runs the serve loop until Ctrl-C + (`tokio::signal`). Help renders; 3 unit tests green. **Full + `cargo xtask verify` green** (all 14 steps, incl. WASM rebuild + hub-client + tests). Two pre-existing environment issues surfaced and were fixed en + route — both unrelated to this work: `npm install` (the branch had added + uninstalled tiptap/prosemirror deps for `preview-renderer`) and a stale + WASM artifact (`captureSplice.wasm.test.ts` needed `npm run build:wasm`). + +### Phase 4a — end-to-end evidence (2026-07-01) + +Per CLAUDE.md's end-to-end rule. The scripted integration test +(`tests/integration/execute.rs`) drives the **real** provider loop against a +**real** samod acceptor: + +``` +cargo nextest run -p quarto-hub-provider --test integration execute +``` + +Observed: `provider_executes_an_allowed_request_and_writes_a_capture` — the +editor side broadcasts an `exec/request` on the index handle; the provider +(joined over the real `BearerDialer` transport) materializes the project, runs +the passthrough engine via the uncached `record_capture`, and writes a capture +binary doc + `idle` `CaptureRef`. The test then reads the capture doc back +**from the server repo** (proving it synced to peers), gunzips it, and asserts +a `Vec` with `engine_name == "test-passthrough"`. The `Deny` +companion asserts no capture is written over a 3 s broadcast window. + +The interactive `q2 provide-hub --allow-all ` against a real +quarto-hub.com session is **manual** (same as Phase 3C — can't automate Google +OAuth). Coverage standing in for it: the scripted execute loop above + +`join.rs` (real authenticated sync path) + `auth_bridge.rs` (real Node↔Rust +token plumbing). + +**Phase 4a complete.** Remaining in Phase 4: **4b** — the hub-client Run +button (gated on a live beacon, reflecting `CaptureRef.state`/staleness). + +#### Phase 4b — implementation checklist (TDD), hub-client + +The trigger is the ephemeral `channel.requestExecution(path)` (not q2-preview's +HTTP POST), so we reuse the *pattern* of `q2-preview-spa`'s +`StaleCaptureOverlay` (state-reflecting label, disable-while-running, inline +error) but not the component. + +- [x] **4b-1 — `hasExecutableCells(content)` helper.** Pure detector of + executable fenced code cells (```` ```{lang} ````) to gate the Run + affordance. Unit-tested. +- [x] **4b-2 — expose `requestExecution` from `useExecutionChannel`.** Return + `{ executors, requestExecution }` (hold the channel in a ref so the + callback is stable across renders). Update the hook's integration test + + the two App.tsx call sites. +- [x] **4b-3 — `RunControl` component (presentational).** Run/Re-run button → + `onRun(path)`; disabled + "Executing…" while a local pending flag or + `state === 'running'`; `state === 'error'` surfaces `lastError`; + `staleness` shows a "code changed" note. Local pending clears when the + `captureDocId` changes (new capture arrived), on `error`, or after a + timeout (ephemeral request may find no executor). Component test for the + states. +- [x] **4b-4 — wire into App + Editor.** App destructures the hook and passes + `requestExecution` to Editor; Editor computes `hasExecutableCells(content)` + and renders `RunControl` in the preview pane when + `executorsOnline && hasExecutableCells`, keeping the plain + "Executor online" bar for non-executable docs. +- [x] **4b-5 — build + changelog.** ✅ done. `npm run build:all` green (strict + tsc -b + vite); full hub-client suite green (unit 685 / integration 95 / + wasm 124). Committed `76a01167` (code) + `6e279c8f` (changelog). + +**Phase 4b complete → Phase 4 (execute-on-request) complete.** The full loop +works: a collaborator clicks Run → the editor broadcasts `exec/request` → a +connected `q2 provide-hub --allow-all` executes and writes the capture back → +every peer's preview shows the executed output. Remaining is Phase 5 +(retention/dedup + real provider-only authz via `/auth/actor` Bearer support) +and Phase 6 (hardening: reconnect/refresh, multi-executor claims). + +#### Local e2e harness + dev-token escape hatch (2026-07-01) + +To make the browser click-through testable without quarto-hub.com or Google +OAuth, added a **dev escape hatch** to `q2 provide-hub`: `--token ` (or +`QUARTO_HUB_TOKEN`) uses a `StaticTokenSource` instead of spawning the Node +OAuth bridge. Intended for a **local, no-auth `q2 hub`**, which ignores the +bearer entirely (`server.rs:944` — no `auth_config` ⇒ no credential check). The +interactive OAuth path is unchanged when `--token` is absent. + +This unblocks a fully-local end-to-end test: `q2 hub --project` (no auth, +`/health` returns the index-doc id) + hub-client dev (`VITE_DEFAULT_SYNC_SERVER` ++ no `VITE_GOOGLE_CLIENT_ID` ⇒ anonymous) + `q2 provide-hub --allow-all --token +dev`. The runnable harness (example project + helper script + walkthrough) lives +in **`claude-notes/hub-execution-e2e/`**. Verified directly: no-auth hub + +`/health` id, provider connect over `--token`, and the Jupyter engine executing +`2+3`→`5` (via `q2 render` with `engine: jupyter`); the browser Run-click is the +manual last mile the harness drives. + +**Note (no faithful browser E2E in 4b, same as 1G):** a real end-to-end — +click Run → provider executes → preview shows output — needs both a browser +*and* a live provider against a shared hub, which can't be automated here +(interactive OAuth + a running executor). The scripted Phase 4a integration +test already proves the provider half end-to-end; 4b's component/hook/unit tests +prove the editor wiring (request broadcast, state reflection, gating). The full +click-through is the manual verification the user can run with +`q2 provide-hub --allow-all`. + +- **Phase 5 — Retention (D3) + authorization (D4).** Content-addressed +- **Phase 5 — Retention (D3) + authorization (D4).** Content-addressed + capture dedup; per-project opt-in + (if chosen) requester gating + + consent UX. File server-GC follow-up. +- **Phase 6 — Hardening.** Reconnect/refresh, multi-executor claim, + diagnostics surface, `cargo xtask verify` full green, push approval. + +## Key source references + +- Capture transport (result side, fully reusable): + - `crates/quarto-preview/src/capture_driver.rs:57,326` (record + write doc) + - `crates/quarto-preview/src/re_execute.rs:102,291,337` (trigger + perform) + - `crates/quarto-core/src/engine/preview_record.rs:130` (`record_capture`) + - `crates/quarto-preview/src/cache.rs:151` (`record_capture_cached`) + - `crates/quarto-trace/src/lib.rs:186` (`EngineCapture`) + - `crates/quarto-hub/src/index.rs:68,272` (`CaptureRef`, `set_capture`) + - `crates/quarto-hub/src/resource.rs:18,116` (binary doc schema) + - `crates/quarto-core/src/engine/capture_splice.rs` (replay/splice) + - `crates/wasm-quarto-hub-client/src/lib.rs:1201-1309` (WASM consume) +- Editor consumption (port target): + - `q2-preview-spa/src/PreviewApp.tsx:758,1030-1059` + - `ts-packages/quarto-sync-client/src/client.ts:352-377,1102,174` + - `q2-preview-spa/src/components/StaleCaptureOverlay.tsx:59-77` +- Clear results (D6): + - `crates/quarto-hub/src/index.rs:307` (`remove_capture`, map-key delete) + - `ts-packages/quarto-sync-client/src/client.ts:174` (internal `indexHandle` to expose for mutation) +- Ephemeral channel (request/capability model): + - `hub-client/src/services/presenceService.ts:333-439` + - `ts-packages/quarto-sync-client/src/client.ts:1335-1346` (`getFileHandle`) +- samod client peer + auth: + - `crates/quarto-hub/src/peer.rs:18-50` (`dial_websocket`) + - `crates/quarto-hub/src/server.rs:385-395,781-806,1051-1057` (Bearer, actor) + - `ts-packages/quarto-hub-mcp/src/auth/*` (OAuth/keyring/refresh) + - `ts-packages/quarto-sync-client/src/NodeWebSocketClientAdapter.ts` (Bearer WS) + - `crates/quarto-mcp-launcher/src/lib.rs` (`q2 mcp` launcher pattern) +- Prior plans: + - `claude-notes/plans/2026-06-11-q2-mcp-hub-auth.md` (auth + launcher; native-Rust findings) + - `claude-notes/plans/2026-05-18-q2-preview-project-replay-engine.md` (capture splice) + - `claude-notes/plans/2026-05-27-multi-engine-execution.md` (Vec) + - `claude-notes/plans/2026-01-06-execution-engine-infrastructure.md` (engine trait) diff --git a/claude-notes/plans/2026-07-01-html-format-capture-display.md b/claude-notes/plans/2026-07-01-html-format-capture-display.md new file mode 100644 index 000000000..4b491363b --- /dev/null +++ b/claude-notes/plans/2026-07-01-html-format-capture-display.md @@ -0,0 +1,229 @@ +# Display executed code output in the default `format: html` preview + +**Strand:** bd-uy4uygha (bug, P1; discovered-from bd-sfet3264). +**Date:** 2026-07-01. +**Status:** Planned — awaiting review. No implementation started. + +## Overview + +hub-client shows server-recorded engine output (executed code results) **only** +for documents whose resolved format is `q2-preview` (or `q2-slides` / `q2-debug` +/ `revealjs`). For the **default `format: html`** — plain documents and every +website page — the preview renders code cells as *source* even when a capture +exists: the "Executor online" and "Showing executed output" bars appear, the +provider runs the engine and writes the capture, but the output never lands in +the preview. + +Discovered while verifying the remote-execution feature (bd-sfet3264) +end-to-end. Switching a document to `format: q2-preview` immediately shows the +output — that is the current **workaround**. + +**Why it never surfaced before:** `q2 preview` only ever renders in the +`q2-preview` (AST-splice) format, so its capture path is always exercised. +hub-client additionally supports the plain-`html` render (with full website +chrome), which `q2 preview` does not — an untested mode. The Phase 1 capture +tests only covered `ReactPreview` (the `q2-*`/revealjs component), not the +default `Preview`. + +## Root cause (three layers, verified 2026-07-01) + +The capture bytes already flow **into Rust** — `render_page_in_project_with_attribution` +parses them (`crates/wasm-quarto-hub-client/src/lib.rs:1148`) — but every HTML +branch drops them, and the TS default-HTML path never sends them. + +1. **WASM — the HTML branch ignores captures.** `render_single_doc_to_response` + (`lib.rs:1351`) dispatches on `format.pipeline_kind`: the `Some("preview")` + branch folds captures through the splice; the `_ =>` HTML branch + (`lib.rs:1438`) calls `render_qmd_to_html(...)` and never references + `captures` (explicit comment at `lib.rs:1357-1362`). The website path + `render_project_active_page_to_response` (`lib.rs:1519`) is the same: its + HTML `_ =>` arm (`lib.rs:1616`) builds a `RenderToHtmlRenderer` with no + captures. Both HTML branches funnel through `render_qmd_to_html` + (`crates/quarto-core/src/pipeline.rs` ~833; the Pass-2 site is + `crates/quarto-core/src/project/pass2_renderer.rs:825`), which has **no + captures channel**. + +2. **preview-runtime — `renderToHtml` doesn't thread `captureGzJson`.** + `RenderToHtmlOptions` (`ts-packages/preview-runtime/src/wasmRenderer.ts:959`) + has no `captureGzJson`; `renderToHtmlInner` (`:1189`) calls + `renderPageInProject(documentPath, grammarsHandle)` and `renderPageInProject` + (`:459`) hard-codes `renderPageInProjectWithAttribution(path, ug, null)` with + no captures. (The lowest wrapper `renderPageInProjectWithAttribution` **does** + already accept `captureGzJson` — `:499` — so only the two convenience + wrappers above it drop it.) + +3. **hub-client — `` never receives captures.** `PreviewRouter` + routes non-`q2-*` formats to `` (`PreviewRouter.tsx:157,163`) and + destructures `captures` **out** of the props it forwards there + (`PreviewRouter.tsx:148`, comment `:146-147`). `Preview.tsx` calls + `renderToHtml({ documentPath, userGrammars })` (`Preview.tsx:106`) — it never + sees `captures`. + +## Key finding that makes this tractable + +`CaptureSpliceStage` (`crates/quarto-core/src/stage/stages/capture_splice.rs`) +is a **plain `DocumentAst → DocumentAst` pipeline stage** with no +q2-preview-specific coupling — empty captures = pass-through. The q2-preview +builder inserts it **immediately before `EngineExecutionStage`** and rebuilds +that stage with `.with_spliced_engines(names)` to suppress the spurious +"(no execution)" warning (`pipeline.rs:406-433`). We insert it into the HTML +stage list the exact same way. + +Two properties make this low-risk: + +- **Cell alignment is guaranteed.** The server records captures via + `build_capture_pipeline_stages` = the **HTML** stage list truncated at + `engine-execution` (`crates/quarto-core/src/engine/preview_record.rs:109-117`). + So the HTML render's pre-engine stages produce byte-identical cell input to + what the capture recorded — the `(content-hash, occurrence-index)` match the + splice relies on holds by construction. +- **No phase-ordering contract to satisfy.** `CaptureSpliceStage` is a top-level + *pipeline stage*, not a member of `build_transform_pipeline`, so the + `TransformPhase` contract (which only governs transforms inside + `AstTransformsStage`) does not apply. The only invariant — splice **before** + engine execution — is the same one the preview builder already relies on. + +## Design decision + +**Insert `CaptureSpliceStage` into the HTML pipeline at the stage level, +immediately before `EngineExecutionStage`** — a captures-aware sibling of +`build_html_pipeline_stages_with_options`, mirroring +`build_q2_preview_pipeline_stages`. Rejected alternative: adding a transform +inside `build_transform_pipeline` (would re-implement splice logic, run +post-engine, and drag in the phase-ordering contract). + +**Plumbing surface for `render_qmd_to_html`:** carry the captures on +**`HtmlRenderConfig`** (a `captures: Vec` field, default empty) +rather than a new positional argument. Rationale: `render_qmd_to_html` is called +from two sites (the single-doc WASM branch and `pass2_renderer.rs:825`) and +across the wasm-bindgen boundary conventions; a config field is the smallest +churn and keeps the empty-captures path byte-identical. **(Confirm during 1A.)** + +## Phased plan (TDD) + +> Per CLAUDE.md: write/adjust the test first, watch it fail, then implement. +> This change touches `quarto-core` + the WASM crate, so the WASM leg is +> affected — full `cargo xtask verify` (with WASM rebuild) is required before +> the push request, and hub-client needs `npm run build:wasm` for the browser +> e2e. + +### Phase 1 — Rust: captures-aware HTML pipeline (`quarto-core`) + +- [x] **1A — captures-aware HTML stage builder.** ✅ done. Added a + `captures: Vec` field + `with_captures` to `HtmlRenderConfig` + (default empty); extracted a shared `insert_capture_splice_stage` helper + (the splice-insert + engine-stage-rebuild-with-spliced-names, formerly + inline in `build_q2_preview_pipeline_stages`, now used by both); + `build_html_pipeline_stages_with_captures` variant; `render_qmd_to_html` + uses it when `config.captures` is non-empty (empty → unchanged builder, + byte-identical). RED→GREEN native test + `render_qmd_to_html_splices_captures` (hand-built `.cell`-wrapped capture + with a fictitious non-spawning engine, mirroring `captureSplice.wasm.test.ts`) + → the marker appears in the HTML as a `.cell` Div; empty captures render + source-only. Full quarto-core suite (2406) green, clippy clean. + **Note:** the raw-cell `input_qmd` hash-matches the doc's post-sugar cell + (as the WASM test already relied on); a real engine name spawns a + subprocess, so tests must use a fictitious engine + a `.cell`-wrapped + `result.markdown` (a bare passthrough echo isn't a `Div.cell` and won't + splice). +- [x] **1B — `RenderToHtmlRenderer::with_captures`.** ✅ done. Added the + `captures` field + `with_captures` builder (mirrors + `RenderToPreviewAstRenderer`); `render` now builds the config with + `.with_captures(self.captures.clone())`. New integration test + `render_to_html_captures.rs`: a multi-file project (`_quarto.yml` + + index.qmd cell + about.qmd prose) rendered in `ActivePage` mode with a + capture → the active page's HTML has the spliced `.cell-output`; no + captures → source-only. Green (reuses the 1A `render_qmd_to_html` path). + +### Phase 2 — WASM: use captures in both HTML branches (`wasm-quarto-hub-client`) + +- [x] **2A — thread the already-parsed `captures` into the HTML branches.** + ✅ done. `render_single_doc_to_response` `_ =>` arm: config built with + `.with_captures(captures)`. `render_project_active_page_to_response` `_ =>` + arm: `renderer.with_captures(captures)`. No new wasm-bindgen signature — + the bytes already arrive via `render_page_in_project_with_attribution`. + RED→GREEN WASM vitest `captureSpliceHtml.wasm.test.ts` (a `format: html` + doc + a real gzipped capture → the marker appears in the rendered `html`; + no-capture ⇒ source-only). RED confirmed against the pre-rebuild WASM, + GREEN after `npm run build:wasm`. Full WASM suite (126) green. + +### Phase 3 — preview-runtime: forward `captureGzJson` through `renderToHtml` + +- [x] **3A — thread captures through the convenience wrappers.** ✅ done. Added + `captureGzJson?: Uint8Array` to `RenderToHtmlOptions`; `renderToHtmlInner` + forwards it into `renderPageInProject`; `renderPageInProject` gained a + captures param forwarding to the already-capable + `renderPageInProjectWithAttribution`. tsc clean; preview-runtime tests (74) + green. + - **Coverage note (deviation from the plan's mocked-unit-test idea):** + preview-runtime's render wrappers are exercised via *real* WASM in + `*.wasm.test.ts`, and `getWasm` has no injection seam; + `renderToHtml`→binding can't run in vitest because `initWasm()` calls + `wasm.default()` with no args (can't locate the `.wasm` in Node). So this + pure pass-through is covered by tsc + Phase 2 (the binding splices, real + WASM) + Phase 4 (Preview passes `captureGzJson` to `renderToHtml`) + + Phase 5 (browser e2e), rather than a standalone mock. + +### Phase 4 — hub-client: feed captures to the default `` + +- [x] **4A — route captures to ``.** ✅ done. `PreviewRouter` now + passes `captures={captures}` to ``. +- [x] **4B — `` consumes captures.** ✅ done. Factored the shared + capture-fetch into `hub-client/src/hooks/useActiveCaptureBytes.ts` (derive + `captureDocId` from `captures[path]`, fetch via `getBinaryDocById`, + keyed on the doc id); `ReactPreview` refactored to use it (removing its + inline copy); `Preview` uses it and threads `captureGzJson` into + `renderToHtml` (with `captureBytes` in the render-callback deps so a fresh + capture re-renders). Tests: `useActiveCaptureBytes.integration.test.tsx` + (3), `Preview.capture.integration.test.tsx` (2, mirroring the ReactPreview + test); the existing ReactPreview capture test still passes after the + refactor. All render+hooks integration tests (65) green; strict + `build:all` green. + +### Phase 5 — verify + end-to-end + +- [x] **5A — `cargo xtask verify`** ✅ full green (all 14 steps: Rust build + + workspace tests, WASM rebuild, ts-packages, hub-client build + tests). +- [x] **5B — browser e2e (recorded).** ✅ verified live in Chrome DevTools. A + project with `engine: knitr` + a `{r}` cell and **no `format:` key** (i.e. + the default `format: html`), on `wss://sync.automerge.org`, with a + connected `q2 provide-hub --allow-all`. Clicked **Run** → the provider + wrote a capture (`capture_doc_id=MDTAUnea…`) → the preview iframe now shows + `cat(1, 2, 3)` **followed by `1 2 3`** in a real `.cell-output` + (`previewHasOutput: true`, `hasCellOutput: true`) — with **no + `format: q2-preview`**. The exact case that previously rendered source-only. + +**bd-uy4uygha COMPLETE.** All phases done; the default `format: html` preview +displays executed engine output end-to-end. + +## Risks / open questions + +- **`HtmlRenderConfig` field vs. argument for `render_qmd_to_html`** — confirm in + 1A that a config field is clean across both call sites (the wasm single-doc + branch and `pass2_renderer.rs`). Fall back to an explicit argument if the + config is shared in a way that makes an empty-default awkward. +- **Shared builder helper** — 1A should factor the "insert splice + rebuild + engine stage with spliced names" logic so the q2-preview and HTML builders + can't drift (they must stay cell-aligned with `build_capture_pipeline_stages`). +- **Website multi-page** — captures are per-file and only the **active page** + (Pass-2) needs them; Pass-1 builds the index/profile with no captures. Verify + a capture on the active page doesn't leak into sibling-page renders (1B test). +- **`Preview`/`ReactPreview` duplication** — the capture-fetch effect is now + needed in both; factor a hook rather than copy-paste (bd-uy4uygha follow-up if + it grows). + +## Key source references + +- WASM HTML branches: `crates/wasm-quarto-hub-client/src/lib.rs:1438` (single-doc), + `:1616` (project active page); captures parsed at `:1148`. +- Preview splice model: `build_q2_preview_pipeline_stages` + (`crates/quarto-core/src/pipeline.rs:387-436`), splice insert `:431-433`. +- `CaptureSpliceStage`: `crates/quarto-core/src/stage/stages/capture_splice.rs`. +- HTML pipeline: `build_html_pipeline_stages_with_options` + (`crates/quarto-core/src/pipeline.rs:249-339`); `render_qmd_to_html` (~`:833`). +- Pass-2 renderer: `crates/quarto-core/src/project/pass2_renderer.rs:727,825,991`. +- Capture recording (cell alignment): `build_capture_pipeline_stages` + (`crates/quarto-core/src/engine/preview_record.rs:109-117`). +- TS wrappers: `ts-packages/preview-runtime/src/wasmRenderer.ts:459,491,959,1168`. +- hub-client: `hub-client/src/components/render/{PreviewRouter.tsx:148,157,163, + Preview.tsx:106, ReactPreview.tsx:582-605}`. diff --git a/claude-notes/plans/2026-07-01-merge-preview-status-line.md b/claude-notes/plans/2026-07-01-merge-preview-status-line.md new file mode 100644 index 000000000..b8ce1475a --- /dev/null +++ b/claude-notes/plans/2026-07-01-merge-preview-status-line.md @@ -0,0 +1,306 @@ +# Merge the preview executor + capture status bars into one status line + +**Strand:** bd-yai4w8ly (task, P2). Discovered from bd-sfet3264 +(remote code-execution provider). +**Date:** 2026-07-01. +**Status:** IN PROGRESS — open questions resolved 2026-07-01 (see +"Decisions locked"); implementing per the TDD checklist. + +## Decisions locked (user, 2026-07-01) + +1. **Palette** — don't bikeshed; a theme overhaul is coming anyway. + Merge onto one reasonable strip color (using the blue capture tone + `#eef6ff`, since the bar is mostly about output once you've run). +2. **Stale copy** — show **both** facts: `Showing executed output · + code changed` (don't swap the whole label to "Code changed…"). +3. **No-executor + capture** — **yes**, still show "Showing executed + output" + Clear when offline (Clear is executor-independent). +4. **Button order** — **Clear then Run**, right-aligned, so **Run stays + pinned to the far right** across the natural transitions: no bar → + executor online `… [Run]` → code runs `… [Clear] [Run]` → and Run + disappears only when the executor goes away. Action group order is + `[Clear results…] [Run/Re-run]`. +5. **Component boundary** — **one** `PreviewStatusBar` component. + +## Overview + +The hub-client preview pane currently shows **two independent status +bars** stacked above the preview iframe whenever code execution is in +play. They were built in separate phases of bd-sfet3264 and never +unified: + +- **Bar A — executor / Run** (green, `#eefaf0`). One of: + - `RunControl` (`src/components/render/RunControl.tsx`) — a green dot, + a status label (`Executor online` / `Code changed since the last + run.` / an error), and a **Run** / **Re-run** / **Executing…** + button. Shown when an executor is online **and** the active doc has + executable cells. + - a plain read-only `.executor-online-bar` div (inline in + `Editor.tsx`) — dot + `Executor online`, no button. Shown when an + executor is online but the doc has **no** executable cells. + - nothing, when no executor is online. +- **Bar B — capture / Clear** (blue, `#eef6ff`). `ClearCaptureControl` + (`src/components/render/ClearCaptureControl.tsx`) — `Showing executed + output` + a **Clear results…** button with a two-step inline + confirm. Shown whenever the active doc has a capture entry. + +The two are rendered back-to-back in `Editor.tsx:1102-1118`, so in the +common "an executor is online and I just ran the doc" state the user +sees **two stacked strips of different colors** saying closely related +things ("Executor online" + "Re-run" over "Showing executed output" + +"Clear results…"). + +**Goal:** collapse these into a **single status line** that selectively +shows the executor info when an executor exists, the "showing executed +output" message when a capture exists, and both the Run/Re-run and Clear +buttons as relevant — one strip, one color, one row. + +## Current wiring (verified 2026-07-01) + +All in `hub-client/`: + +| Concern | Location | +|---|---| +| Both bars rendered | `src/components/Editor.tsx:1102-1118` | +| Run affordance | `src/components/render/RunControl.tsx` | +| Read-only executor fallback | inline `.executor-online-bar` div, `Editor.tsx:1108-1113` | +| Clear affordance | `src/components/render/ClearCaptureControl.tsx` | +| CSS | `src/components/Editor.css:279-375` (`.capture-results-bar`, `.executor-online-bar`, `.executor-online-dot`, `.run-control*`) | +| `executorsOnline` source | `App.tsx:115-118,699` — `useExecutionChannel(...).executors.length > 0` | +| `onRequestExecution` source | `App.tsx:700` — `requestExecution` from `useExecutionChannel` → ephemeral `exec/request` broadcast | +| `captures` source | `App.tsx:108,458-459,698` — `onCapturesChange` sync callback | +| `clearCapture` | imported from `@quarto/preview-runtime` (`Editor.tsx:16`); removes the shared `CaptureRef` sidecar entry | +| Executable-cell gate | `src/services/executableCells.ts` — `hasExecutableCells(content)` | + +Inputs available at the render site (all already threaded to `Editor`): + +- `executorsOnline: boolean` +- `hasExecutableCells(content): boolean` +- `capture = captures?.[currentFile.path]` — a `CaptureRef` or + `undefined`, carrying `{ captureDocId, state ('idle'|'running'| + 'error'), staleness, lastError }` +- `onRequestExecution(path)` (ephemeral run request) +- `clearCapture(path)` (shared sidecar delete) + +Both control components are **presentational** (mutations injected as +props) with their own tests: +`RunControl.integration.test.tsx`, +`ClearCaptureControl.integration.test.tsx`. + +## The full state space + +The merged bar is a function of two roughly-independent axes: + +**Executor axis** (from `executorsOnline` + `hasExecutableCells`): +1. no executor online +2. executor online, doc has **no** executable cells +3. executor online, doc **has** executable cells + +**Capture axis** (from `capture`): +a. no capture +b. capture present, `state: idle` +c. capture present, `state: running` (or a local run pending) +d. capture present, `state: error` (+ `lastError`) +e. capture present, `staleness: true` ("code changed") + +Today Bar A owns the executor axis + the run-pending/error/stale text; +Bar B owns "showing executed output" + Clear. They overlap on the +running/stale/error signalling (those belong to the capture, but only +`RunControl` renders them, and only when there are executable cells). + +## Proposed design — one `PreviewStatusBar` + +Replace `RunControl`, the inline `.executor-online-bar`, and +`ClearCaptureControl` with a **single** `PreviewStatusBar` component +that renders **at most one strip**, laid out as: + +``` +[● dot?] [Clear results…] [Run/Re-run] + └ executor-online only └ left, flex:1 └ right-aligned action group +``` + +Button order is `[Clear] [Run]` so **Run stays pinned to the far +right** across the natural transitions (decision 4): `… [Run]` when an +executor comes online, `… [Clear] [Run]` after a run, and Run leaves +only when the executor goes away. + +### Visibility + +Render the bar iff **any** of: an executor is online, or a capture +exists. (i.e. hide only in state 1a — nothing to say and nothing to +do.) This matches today: Bar A shows when executor online, Bar B shows +when capture exists; the union is "either." + +### Left side — dot + status text (single precedence chain) + +Show the green dot iff `executorsOnline`. Then pick **one** status +message by precedence (busy/error first, so transient states win): + +1. **busy** (`state === 'running'` or local pending) → `Executing…` +2. **error** (`state === 'error'`) → `capture.lastError` (red text, + `role="alert"`) +3. **capture present** → `Showing executed output` + - if also `staleness` and executor online: append/replace with a + "code changed since the last run" hint (see open Q2) +4. **executor online, no capture** → `Executor online` + +(State 1a is already excluded by the visibility rule, so there is +always something to show.) + +### Right side — action group (both buttons, independently gated) + +Rendered in DOM order **Clear then Run** so Run is pinned far right +(decision 4): + +- **Clear results…** button — shown iff a capture exists. Keeps the + two-step inline confirm from `ClearCaptureControl`. When the user is + mid-confirm, the confirm prompt + Clear/Cancel replace the normal + status text (an `alertdialog`), as today. +- **Run / Re-run** button — shown iff `executorsOnline && + hasExecutableCells`. Label: `Executing…` (disabled) while busy, else + `Re-run` if a capture exists, else `Run`. Same pending-snapshot / + timeout logic as today's `RunControl` (lift it verbatim). + +Both can appear together (executor online + executable cells + a +capture) — that is exactly the doubled-bar case we're collapsing, now +one row: `● Showing executed output [Re-run] [Clear results…]`. + +### Behaviour preserved from the two components + +- Run pending-snapshot (`captureDocId` at request time), reset on + `path` change, cleared on new capture / error / `PENDING_TIMEOUT_MS` + (30s). — from `RunControl`. +- Clear two-step confirm, reset on `path` change; confirm names the + collaborator-wide effect. — from `ClearCaptureControl`. +- Run request = `onRequestExecution(path)` (ephemeral); Clear = + `clearCapture(path)` (shared sidecar delete). Same injected-prop + shape → same testability. + +### CSS + +One class (`.preview-status-bar`) replacing `.run-control` / +`.executor-online-bar` / `.capture-results-bar`. Pick a single palette +(open Q1: green vs blue vs neutral). Reuse `.executor-online-dot`. +Right-align the button group (`margin-left:auto` on the action group, +or `flex:1` on the label as today). Keep the red error and +destructive-confirm button styles. + +## State → rendering table (the contract to test) + +| Executor | Capture | Dot | Status text | Clear btn | Run btn | +|---|---|---|---|---|---| +| offline | none | – | *(bar hidden)* | – | – | +| offline | idle | – | Showing executed output | Clear results… | – | +| offline | error | – | *lastError* (red) | Clear results… | – | +| online, no exec cells | none | ● | Executor online | – | – | +| online, no exec cells | idle | ● | Showing executed output | Clear results… | – | +| online, exec cells | none | ● | Executor online | – | Run | +| online, exec cells | idle | ● | Showing executed output | Clear results… | Re-run | +| online, exec cells | running/pending | ● | Executing… | Clear results… | Executing… (disabled) | +| online, exec cells | error | ● | *lastError* (red) | Clear results… | Re-run | +| online, exec cells | idle + stale | ● | Showing executed output · code changed | Clear results… | Re-run | + +(Rows with a capture but `offline`/`no-exec-cells` are reachable: a +capture can outlive the executor that produced it, and non-`.qmd` or +prose-only docs can carry a capture from earlier. Clear must still work +with no executor — it is a pure CRDT mutation.) + +## Open questions — RESOLVED (see "Decisions locked" at top) + +All five settled with the user 2026-07-01: blue strip; show both facts +for stale; keep "Showing executed output" when offline; button order +`[Clear] [Run]` (Run pinned far right); one component. + +## Plan (TDD) — DRAFT, do not start yet + +Per CLAUDE.md: tests first, then implement, then verify green, then +end-to-end in a real browser session before declaring done. + +- [x] **0 — Lock the open questions** (palette, stale copy, + no-executor+capture, button order, component boundary) with the + user. ✅ 2026-07-01 (see "Decisions locked"). +- [x] **1 — Component test for `PreviewStatusBar`** (RED). ✅ + `PreviewStatusBar.integration.test.tsx` — 15 cases encoding the + state→rendering table (dot/label/buttons per row) + ported + behaviors: run pending clears on new capture / error / timeout / + path change; two-step confirm calls `onClear`; cancel and + path-change disarm it. Confirmed RED before the component existed. +- [x] **2 — Implement `PreviewStatusBar`** ✅ single component + (`PreviewStatusBar.tsx`); a small inner `StatusLabel` owns the + label precedence. `RunControl` + `ClearCaptureControl` deleted. + GREEN (15/15). +- [x] **3 — Rewire `Editor.tsx`** ✅ replaced the three-way Bar A block + + `ClearCaptureControl` with one `` fed + `executorsOnline`, `hasExecutableCells(content)`, the active + `capture`, `onRun`, `onClear`. Inline `.executor-online-bar` gone. +- [x] **4 — CSS** ✅ added `.preview-status-bar` (+ `.preview-status-*` + label/actions/buttons); removed `.run-control*`, + `.executor-online-bar`, `.capture-results-bar`. Kept + `.executor-online-dot`. Blue palette; Run pinned right via a + `margin-left:auto` action group. +- [x] **5 — Delete/retarget old tests** ✅ old test files removed; the + only remaining `RunControl`/`ClearCaptureControl` mentions are + documentation comments. +- [x] **6 — Build + suite** ✅ `npm run build:all` green (strict + `tsc -b` + vite); full suite green (unit 685 / integration 103 / + wasm 126). +- [x] **7 — End-to-end** in a real browser ✅ 2026-07-01 (see + "End-to-end evidence" below). Drove the full status-bar lifecycle + against the local Option-B harness; the merged single-row bar + renders and transitions correctly in every state. +- [x] **8 — Changelog** ✅ two-commit workflow: code `0b13dbcb`, then + `hub-client/changelog.md` entry `1eb0183e` under 2026-07-01. + +**Status: implementation complete on `feature/hub-execution-provider` +(commits `0b13dbcb` + `1eb0183e`).** All 8 checklist items done; TS +suite + strict build green; browser-verified. Not pushed (awaiting +approval). A full `cargo xtask verify` should run before the eventual +push, though this change is hub-client-only (TS/CSS); no Rust touched. + +## End-to-end evidence (2026-07-01) + +Per CLAUDE.md's end-to-end rule. Stood up the local Option-B harness +(`claude-notes/hub-execution-e2e/`): a no-auth `q2 hub --project` +(index id `HLDXSUAS7RBv9HPGYrvLTifK3sc`), the hub-client dev server +pointed at it (`VITE_DEFAULT_SYNC_SERVER=ws://127.0.0.1:3031 npm run +dev`), and a connected executor (`q2 provide-hub --server +ws://127.0.0.1:3031 --allow-all --token dev +HLDXSUAS7RBv9HPGYrvLTifK3sc` → "Execution ENABLED"). Opened the Share +URL in a real browser and exercised the **single merged status bar**: + +1. **online + executable cells + no capture** → one blue row: + `● Executor online … [Run]` (dot present, Clear absent). +2. **click Run → capture arrives** → one row: + `● Showing executed output … [Clear results…] [Re-run]`. This is + exactly the state that previously showed **two stacked bars**; it is + now a single strip, with **Clear before Run** and **Run pinned to + the far right** (decision 4). +3. **click "Clear results…"** → confirm state replaces the status text: + `● Clear executed output? This removes it for all collaborators + until the document is run again. [Clear] [Cancel]` (red Clear). +4. **click Clear** → capture removed → back to state 1 + (`● Executor online … [Run]`), Clear gone, "Re-run" → "Run". + +Every transition matched the state→rendering table. Observed directly +in the browser (screenshots captured in the session transcript). + +**Orthogonal bug found (not this change) → bd-gthycd33.** The +executor's computed value did not splice into the preview for +`engine: jupyter` (the `{python}` `2 + 3` cell rendered as source, no +`5`), even though the sidecar arrived and the bar correctly read +"Showing executed output". Re-testing with `engine: knitr` (added +`claude-notes/hub-execution-e2e/project/r-demo.qmd`) **splices +correctly**: `1 + 1` → `[1] 2`, `cat(…)` → the R version string, +`sum(1:10)` → `[1] 55`. So the provider + consumption/splice path work; +the defect is **jupyter-specific** (how the Jupyter engine's capture +output is produced / matched by `CaptureSpliceStage`). Filed as +bd-gthycd33; untouched by and out of scope for this status-bar merge. +This browser session also served as the parent plan's deferred +in-browser splice verification (1G / 4b "manual last mile") — confirmed +working for knitr. + +## References + +- Parent feature plan: `claude-notes/plans/2026-06-29-remote-execution-provider.md` + (Phases 1F, 2D, 4b built the two bars). +- Local e2e harness: `claude-notes/hub-execution-e2e/`. diff --git a/crates/quarto-core/src/pipeline.rs b/crates/quarto-core/src/pipeline.rs index 99864f1fb..6d153dff5 100644 --- a/crates/quarto-core/src/pipeline.rs +++ b/crates/quarto-core/src/pipeline.rs @@ -116,6 +116,18 @@ pub struct HtmlRenderConfig { /// [`crate::engine::EngineRegistry::with_replay`] from a /// [`quarto_trace::EngineCapture`] loaded from a trace file. pub engine_registry: Option, + + /// Server-recorded engine captures to splice into the HTML render + /// (bd-uy4uygha). When non-empty, a [`crate::stage::CaptureSpliceStage`] + /// is inserted before [`EngineExecutionStage`] so recorded engine output + /// appears in the rendered HTML without re-running the engine — this is how + /// hub-client's default `format: html` preview shows the output of a + /// document executed by a connected `q2 provide-hub`. + /// + /// Empty (the default) renders code cells as source, byte-identical to the + /// pre-bd-uy4uygha behavior for every existing caller (`q2 render` runs the + /// real engine natively instead). + pub captures: Vec, } impl HtmlRenderConfig { @@ -124,6 +136,7 @@ impl HtmlRenderConfig { Self { resolver: Some(resolver), engine_registry: None, + captures: Vec::new(), } } @@ -132,6 +145,13 @@ impl HtmlRenderConfig { self.engine_registry = Some(registry); self } + + /// Attach server-recorded engine captures to splice into the HTML render + /// (bd-uy4uygha). See the [`captures`](Self::captures) field. + pub fn with_captures(mut self, captures: Vec) -> Self { + self.captures = captures; + self + } } /// Output from the render pipeline. @@ -389,27 +409,38 @@ pub fn build_q2_preview_pipeline_stages( captures: Vec, ) -> Vec> { // Build the base list *without* threading the engine registry through; - // we reconstruct the engine-execution stage ourselves below so it can - // also carry the spliced-engine set (bd-sauc9iiq). Passing `None` here - // avoids both a registry clone and a discarded-registry bug. + // insert_capture_splice_stage reconstructs the engine-execution stage with + // it (so it can also carry the spliced-engine set, bd-sauc9iiq). Passing + // `None` here avoids both a registry clone and a discarded-registry bug. let mut stages = build_html_pipeline_stages_with_options(None, None); stages.retain(|s| !Q2_PREVIEW_STAGE_EXCLUDED.contains(&s.name())); + insert_capture_splice_stage(&mut stages, engine_registry, captures); + stages +} - // bd-sauc9iiq: the WASM preview registry has no knitr/jupyter, so - // EngineExecutionStage would fall back to markdown and warn - // "(no execution)" for every engine these captures replay — even though - // CaptureSpliceStage (inserted just below) has already provided their - // real output. Collect the captured engine names and hand them to the - // engine stage so it suppresses that misleading warning for exactly - // those engines (others still warn). Done before `captures` is moved - // into the splice stage. +/// Insert a [`crate::stage::CaptureSpliceStage`] immediately *before* +/// `EngineExecutionStage`, rebuilding that stage with `engine_registry` plus the +/// captured engine names. +/// +/// bd-lucp / bd-5yff4: the splice folds an ordered capture sequence (one per +/// engine) into the AST, replacing engine code cells with their recorded +/// output. bd-sauc9iiq: rebuilding the engine stage with +/// `.with_spliced_engines(...)` suppresses the misleading "(no execution)" +/// warning for exactly the engines the splice already served (the WASM preview +/// registry has no knitr/jupyter). +/// +/// Shared by the q2-preview pipeline and the HTML capture pipeline +/// ([`build_html_pipeline_stages_with_captures`], bd-uy4uygha) so the two stay +/// cell-aligned with `build_capture_pipeline_stages`. With empty `captures` the +/// inserted splice is a pass-through and the engine stage is still (re)built +/// with the registry, so callers may invoke this unconditionally. +fn insert_capture_splice_stage( + stages: &mut Vec>, + engine_registry: Option, + captures: Vec, +) { let spliced_engine_names: std::collections::HashSet = captures.iter().map(|c| c.engine_name.clone()).collect(); - - // Reconstruct the engine-execution stage with the caller's registry (if - // any — e.g. a ReplayEngine for regression testing) *and* the - // spliced-engine set. Mirrors the registry handling in - // `build_html_pipeline_stages_with_options`. let engine_stage = match engine_registry { Some(reg) => EngineExecutionStage::with_registry(reg), None => EngineExecutionStage::new(), @@ -418,20 +449,32 @@ pub fn build_q2_preview_pipeline_stages( let engine_idx = stages .iter() .position(|s| s.name() == "engine-execution") - .expect("engine-execution stage must exist in the q2-preview pipeline"); + .expect("engine-execution stage must exist in the pipeline"); stages[engine_idx] = Box::new(engine_stage); - - // Insert the splice stage immediately *before* EngineExecutionStage. - // bd-lucp: this is the q2-preview-specific consumer of recorded - // captures. bd-5yff4: the captures are an ordered sequence (one per - // engine); the splice folds them. The HTML pipeline doesn't include - // it — `q2 render` either runs the real engine natively or uses - // `--replay` (which goes through `EngineRegistry::with_replay_many`, - // an entirely different code path). let splice_stage: Box = Box::new(crate::stage::CaptureSpliceStage::new().with_captures(captures)); stages.insert(engine_idx, splice_stage); +} +/// Like [`build_html_pipeline_stages_with_options`] but splices server-recorded +/// engine captures into the HTML render (bd-uy4uygha): hub-client's default +/// `format: html` preview shows the output of a document executed by a connected +/// `q2 provide-hub`, without re-running the engine in the browser. +/// +/// With empty `captures` the result is behaviorally identical to +/// `build_html_pipeline_stages_with_options` (a pass-through splice + the same +/// engine stage). Cell alignment with the recorded capture is guaranteed because +/// captures are recorded from this same stage list truncated at engine-execution +/// (`build_capture_pipeline_stages`). +pub fn build_html_pipeline_stages_with_captures( + apply_config: Option, + engine_registry: Option, + captures: Vec, +) -> Vec> { + // Base with None registry — the helper rebuilds the engine stage with the + // registry (mirrors the q2-preview builder; avoids a second registry build). + let mut stages = build_html_pipeline_stages_with_options(apply_config, None); + insert_capture_splice_stage(&mut stages, engine_registry, captures); stages } @@ -845,7 +888,19 @@ pub async fn render_qmd_to_html( .clone() .map(|r| ApplyTemplateConfig::new().with_resolver(r)); let engine_registry = config.engine_registry.clone(); - let stages = build_html_pipeline_stages_with_options(apply_config, engine_registry); + // bd-uy4uygha: when the caller supplies server-recorded captures (hub-client + // executing via a connected `q2 provide-hub`), splice them into the HTML. + // Empty captures take the unchanged builder — byte-identical for every + // existing caller (`q2 render`, which runs the real engine natively). + let stages = if config.captures.is_empty() { + build_html_pipeline_stages_with_options(apply_config, engine_registry) + } else { + build_html_pipeline_stages_with_captures( + apply_config, + engine_registry, + config.captures.clone(), + ) + }; let (output, diagnostics) = run_pipeline(content, source_name, ctx, runtime, stages).await?; // Extract the rendered output @@ -1484,6 +1539,71 @@ mod tests { assert!(output.html.contains("Test")); } + /// bd-uy4uygha: `render_qmd_to_html` must splice server-recorded captures + /// into the HTML (hub-client's default `format: html` preview), not just the + /// q2-preview AST path. Mirrors `captureSplice.wasm.test.ts`: one engine cell + /// + a hand-built capture whose result markdown is a `.cell` wrapper carrying + /// a marker that appears ONLY in the capture, never in the source. + #[tokio::test] + async fn render_qmd_to_html_splices_captures() { + use quarto_trace::EngineCapture; + + // The doc renders as html (no `format:` key). Use a fictitious engine + // name no platform registers, so EngineExecutionStage takes the + // markdown-fallback branch (no subprocess) and the splice — which runs + // before it — is the only thing that can produce output. + let qmd = "---\ntitle: T\nengine: markerlang\n---\n\n```{markerlang}\n1 + 1\n```\n"; + let capture = EngineCapture { + engine_name: "markerlang".into(), + // Same `{markerlang}` cell as the doc, so its content-hash matches. + input_qmd: "```{markerlang}\n1 + 1\n```\n".into(), + // Post-engine markdown: a `.cell` wrapper whose stdout is the marker. + result: serde_json::json!({ + "markdown": "::: {.cell}\n```{.markerlang .cell-code}\n1 + 1\n```\n\n::: {.cell-output .cell-output-stdout}\n```\nSPLICEMARKER_ZX9\n```\n:::\n:::\n" + }), + }; + + let project = make_test_project(); + let doc = DocumentInfo::from_path("/project/test.qmd"); + let format = Format::html(); + let binaries = BinaryDependencies::new(); + let runtime = make_test_runtime(); + + // With the capture, the marker (which is only in the capture) appears. + let mut ctx = RenderContext::new(&project, &doc, &format, &binaries); + let config = HtmlRenderConfig::default().with_captures(vec![capture]); + let out = render_qmd_to_html( + qmd.as_bytes(), + "test.qmd", + &mut ctx, + &config, + runtime.clone(), + ) + .await + .unwrap(); + assert!( + out.html.contains("SPLICEMARKER_ZX9"), + "spliced engine output must appear in the HTML; got:\n{}", + out.html + ); + + // No capture => source-only render (byte-compatible default path). + let mut ctx2 = RenderContext::new(&project, &doc, &format, &binaries); + let out2 = render_qmd_to_html( + qmd.as_bytes(), + "test.qmd", + &mut ctx2, + &HtmlRenderConfig::default(), + runtime, + ) + .await + .unwrap(); + assert!( + !out2.html.contains("SPLICEMARKER_ZX9"), + "no capture => source-only render" + ); + } + #[test] fn test_render_with_callout() { let content = @@ -2274,6 +2394,7 @@ mod tests { let probe_config = HtmlRenderConfig { resolver: None, engine_registry: Some(probe_registry), + ..Default::default() }; let runtime = make_test_runtime(); let _ = pollster::block_on(render_qmd_to_html( @@ -2317,6 +2438,7 @@ mod tests { let config = HtmlRenderConfig { resolver: None, engine_registry: Some(replay_registry), + ..Default::default() }; let runtime = make_test_runtime(); let output = pollster::block_on(render_qmd_to_html( diff --git a/crates/quarto-core/src/project/pass2_renderer.rs b/crates/quarto-core/src/project/pass2_renderer.rs index 51b8a1de9..d90550a2e 100644 --- a/crates/quarto-core/src/project/pass2_renderer.rs +++ b/crates/quarto-core/src/project/pass2_renderer.rs @@ -722,6 +722,13 @@ pub struct RenderToHtmlRenderer { /// so `Rc>` is correct on both wasm32 and on the /// native single-task executor used by tests. (bd-izfv) user_grammars: Option>>, + + /// bd-uy4uygha: server-recorded engine captures for the active page, + /// spliced into the HTML so hub-client's default `format: html` preview + /// shows the output of a document executed by a connected `q2 provide-hub`. + /// Empty (the default) renders code cells as source. Mirrors + /// [`RenderToPreviewAstRenderer`]'s `captures` for the AST path. + captures: Vec, } impl RenderToHtmlRenderer { @@ -732,9 +739,18 @@ impl RenderToHtmlRenderer { vfs_root: vfs_root.into(), vfs_url_root: None, user_grammars: None, + captures: Vec::new(), } } + /// Attach server-recorded engine captures to splice into the active page's + /// HTML (bd-uy4uygha). Mirrors + /// [`RenderToPreviewAstRenderer::with_captures`]. + pub fn with_captures(mut self, captures: Vec) -> Self { + self.captures = captures; + self + } + /// Attach a user-grammar provider. The renderer installs it on /// every per-page [`crate::render::RenderContext`] before /// running the pipeline, so `CodeHighlightStage` consults it @@ -819,7 +835,10 @@ impl Pass2Renderer for RenderToHtmlRenderer { // shared across every page this renderer renders. ctx.user_grammar_provider = self.user_grammars.clone(); - let config = HtmlRenderConfig::with_resolver(resolver.clone()); + // bd-uy4uygha: thread the active page's captures into the HTML render + // so recorded engine output appears without re-running the engine. + let config = + HtmlRenderConfig::with_resolver(resolver.clone()).with_captures(self.captures.clone()); let source_name = doc_info.input.to_string_lossy().to_string(); let mut render_output = render_qmd_to_html( diff --git a/crates/quarto-core/tests/integration/main.rs b/crates/quarto-core/tests/integration/main.rs index e8ba0de3d..4d97e4fe7 100644 --- a/crates/quarto-core/tests/integration/main.rs +++ b/crates/quarto-core/tests/integration/main.rs @@ -35,6 +35,7 @@ pub mod project_pipeline; pub mod project_resources; pub mod render_page_in_project; pub mod render_preserves_source_files; +pub mod render_to_html_captures; pub mod render_to_html_user_grammars; pub mod replay_engine; pub mod revealjs_features; diff --git a/crates/quarto-core/tests/integration/render_to_html_captures.rs b/crates/quarto-core/tests/integration/render_to_html_captures.rs new file mode 100644 index 000000000..e29762fcb --- /dev/null +++ b/crates/quarto-core/tests/integration/render_to_html_captures.rs @@ -0,0 +1,132 @@ +/* + * tests/integration/render_to_html_captures.rs + * + * bd-uy4uygha: the project (website) active-page HTML render must splice + * server-recorded engine captures, so hub-client's default `format: html` + * preview shows the output of a document executed by a connected + * `q2 provide-hub` — not just the `format: q2-preview` AST path. + * + * Drives `ProjectPipeline` in `ActivePage` mode (exactly + * like the `render_page_in_project` WASM entry) with a capture attached via + * `RenderToHtmlRenderer::with_captures`, and asserts the recorded output + * appears in the active page's HTML while the source-only path (no captures) + * does not. + */ + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tempfile::TempDir; + +use quarto_core::format::Format; +use quarto_core::project::ProjectContext; +use quarto_core::project::orchestrator::{ProjectPipeline, RenderMode, project_type_for}; +use quarto_core::project::pass2_renderer::{RenderToHtmlRenderer, WasmPassTwoOutput}; +use quarto_system_runtime::{NativeRuntime, SystemRuntime}; +use quarto_trace::EngineCapture; + +fn write(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); +} + +fn canonical(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn render_active_page(active: &Path, captures: Vec) -> WasmPassTwoOutput { + let runtime: Arc = Arc::new(NativeRuntime::new()); + let mut project = ProjectContext::discover(active, runtime.as_ref()).unwrap(); + if !project.is_single_file { + project = ProjectContext::discover(&project.dir, runtime.as_ref()).unwrap(); + } + + let project_type = project_type_for(&project); + let vfs_root = project.dir.join(".quarto/project-artifacts"); + let renderer = RenderToHtmlRenderer::new(&vfs_root) + .with_url_root("/.quarto/project-artifacts") + .with_captures(captures); + + let mut pipeline = ProjectPipeline::with_renderer( + &mut project, + project_type, + Format::html(), + "html", + runtime.clone(), + renderer, + ) + .with_mode(RenderMode::ActivePage(active.to_path_buf())); + + let summary = pollster::block_on(pipeline.run()).expect("pipeline run"); + assert!( + summary.pass1_failures.is_empty(), + "unexpected pass-1 failures: {:?}", + summary.pass1_failures, + ); + assert!( + summary.pass2_failures.is_empty(), + "unexpected pass-2 failures: {:?}", + summary.pass2_failures, + ); + summary + .outputs + .into_iter() + .next() + .expect("ActivePage mode should produce one output") +} + +/// A capture for the `{markerlang}` cell whose stdout is a marker that only the +/// capture carries. A fictitious engine name keeps EngineExecutionStage on the +/// markdown-fallback branch (no subprocess). +fn marker_capture() -> EngineCapture { + EngineCapture { + engine_name: "markerlang".into(), + input_qmd: "```{markerlang}\n1 + 1\n```\n".into(), + result: serde_json::json!({ + "markdown": "::: {.cell}\n```{.markerlang .cell-code}\n1 + 1\n```\n\n::: {.cell-output .cell-output-stdout}\n```\nSPLICEMARKER_ZX9\n```\n:::\n:::\n" + }), + } +} + +#[test] +fn active_page_html_splices_captures() { + let temp = TempDir::new().unwrap(); + let project_dir = canonical(temp.path()); + + write( + &project_dir.join("_quarto.yml"), + "project:\n type: default\n", + ); + write( + &project_dir.join("index.qmd"), + "---\ntitle: Home\nengine: markerlang\n---\n\n```{markerlang}\n1 + 1\n```\n", + ); + // A sibling page with no cell, so the project has more than one file. + write( + &project_dir.join("about.qmd"), + "---\ntitle: About\n---\n\nJust prose.\n", + ); + + let active = canonical(&project_dir.join("index.qmd")); + + // With the capture, the marker appears in the active page's HTML. + let out = render_active_page(&active, vec![marker_capture()]); + assert!( + out.html().contains("SPLICEMARKER_ZX9"), + "spliced engine output must appear in the active page's HTML; got:\n{}", + &out.html()[..out.html().len().min(600)], + ); + assert!( + out.html().contains("cell-output"), + "the spliced cell should render as a .cell-output block", + ); + + // Without captures, the same page renders source-only. + let out2 = render_active_page(&active, vec![]); + assert!( + !out2.html().contains("SPLICEMARKER_ZX9"), + "no capture => source-only render", + ); +} diff --git a/crates/quarto-core/tests/integration/replay_engine.rs b/crates/quarto-core/tests/integration/replay_engine.rs index 6b6845d9b..68898ec5a 100644 --- a/crates/quarto-core/tests/integration/replay_engine.rs +++ b/crates/quarto-core/tests/integration/replay_engine.rs @@ -92,6 +92,7 @@ fn capture_engine_input(content: &[u8], source_name: &str, engine_name: &str) -> let config = HtmlRenderConfig { resolver: None, engine_registry: Some(registry), + ..Default::default() }; let runtime = runtime_arc(); let _ = pollster::block_on(render_qmd_to_html( diff --git a/crates/quarto-hub-provider/Cargo.toml b/crates/quarto-hub-provider/Cargo.toml new file mode 100644 index 000000000..a58fe82c4 --- /dev/null +++ b/crates/quarto-hub-provider/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "quarto-hub-provider" +version.workspace = true +authors.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +edition.workspace = true +description = "Connect q2 to a hub session as a code-execution provider (bd-sfet3264)" + +[lib] +name = "quarto_hub_provider" +path = "src/lib.rs" + +[dependencies] +# Automerge sync via the same samod fork the hub uses. +samod = { version = "0.9.0", git = "https://github.com/quarto-dev/samod.git", branch = "q2", features = ["tokio", "tungstenite"] } +# Reused index-document model (file list, capture sidecar) for the join+list path. +quarto-hub = { path = "../quarto-hub" } +# Read file documents out of the VFS during materialization (must match the +# version quarto-hub uses so DocHandle::with_document hands us the same type). +automerge = { version = "0.8.0", features = ["utf16-indexing"] } +# Node discovery + the embedded hub-mcp bundle (we spawn its auth-stream entry). +quarto-mcp-launcher = { path = "../quarto-mcp-launcher" } + +# Phase 4: run engines natively and write the capture back into automerge. +quarto-core = { workspace = true } +quarto-system-runtime = { workspace = true } +quarto-trace = { workspace = true } +# Gzip the capture JSON before storing it as a binary automerge doc (same wire +# format q2-preview uses; see CAPTURE_MIME_TYPE in execute.rs). +flate2 = { workspace = true } +# CBOR for the ephemeral exec channel. The browser's DocHandle.broadcast +# CBOR-encodes payloads (cbor-x, useRecords:false → standard CBOR maps), so we +# mirror it with ciborium for cross-language interop. +ciborium = "0.2" +# Per-run project materialization into a fresh temp dir from the VFS. +tempfile = "3" +# Drive the async capture write-back on a blocking worker (mirrors +# quarto-preview/re_execute.rs: engine execution must not block the reactor). +pollster = { workspace = true } + +# Our own websocket dial with an Authorization: Bearer header. We construct the +# transport ourselves (producing raw bytes for samod's Transport), so the +# tungstenite version is independent of samod's beyond the byte boundary — but +# we match 0.27 to avoid a duplicate in the tree. +tokio-tungstenite = { version = "0.27.0", features = ["native-tls"] } +tungstenite = { version = "0.27.0" } + +tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "time"] } +futures = "0.3" +url = "2.5" +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "time", "net", "test-util"] } +tokio-tungstenite = { version = "0.27.0", features = ["native-tls"] } + +[lints] +workspace = true diff --git a/crates/quarto-hub-provider/src/dialer.rs b/crates/quarto-hub-provider/src/dialer.rs new file mode 100644 index 000000000..3b399d13b --- /dev/null +++ b/crates/quarto-hub-provider/src/dialer.rs @@ -0,0 +1,169 @@ +//! A samod [`Dialer`] that injects an `Authorization: Bearer` header. +//! +//! samod's built-in `dial_websocket` cannot set request headers, so an +//! authenticated hub (`/ws` behind a Bearer JWT) is unreachable with it. This +//! dialer reimplements the (small) tungstenite connect path with the header +//! added, fetching a fresh token from a [`TokenSource`] on every (re)connect. +//! +//! The byte-mapping mirrors samod's private `ws_to_bytes`: binary frames carry +//! sync bytes, Ping/Pong are transport-level and dropped, Close ends the +//! stream, and a Text frame is a protocol error. We replicate it here (it is +//! not part of samod's public API) rather than fork samod. + +use std::sync::Arc; + +use futures::future::BoxFuture; +use futures::{SinkExt, StreamExt}; +use samod::{Dialer, Transport}; +use tungstenite::Message; +use tungstenite::client::IntoClientRequest; +use tungstenite::handshake::client::Request; +use tungstenite::http::header::AUTHORIZATION; +use url::Url; + +use crate::ProviderError; +use crate::token::TokenSource; + +/// A samod [`Dialer`] that dials `url` over an authenticated websocket. +pub struct BearerDialer { + url: Url, + token_source: Arc, +} + +impl BearerDialer { + pub fn new(url: Url, token_source: Arc) -> Self { + Self { url, token_source } + } +} + +/// Build a websocket client handshake request for `url` carrying an +/// `Authorization: Bearer ` header. +pub(crate) fn build_auth_request(url: &Url, bearer: &str) -> Result { + let mut request = url + .as_str() + .into_client_request() + .map_err(|e| ProviderError::Handshake(format!("invalid websocket url: {e}")))?; + let value = format!("Bearer {bearer}") + .parse() + .map_err(|_| ProviderError::Handshake("bearer token is not a valid header value".into()))?; + request.headers_mut().insert(AUTHORIZATION, value); + Ok(request) +} + +/// Map an inbound tungstenite message to the transport's byte protocol. +/// +/// `Some(Ok(bytes))` for a binary sync frame, `None` to drop the frame +/// (Close / Ping / Pong / raw Frame), `Some(Err(_))` for a protocol violation +/// (an unexpected text frame on the sync socket). +pub(crate) fn inbound_to_bytes(msg: Message) -> Option, ProviderError>> { + match msg { + Message::Binary(data) => Some(Ok(data.to_vec())), + Message::Close(_) | Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => None, + Message::Text(_) => Some(Err(ProviderError::Protocol( + "unexpected text message on sync websocket".into(), + ))), + } +} + +/// Wrap outbound sync bytes as a binary websocket frame. +pub(crate) fn outbound_to_ws(bytes: Vec) -> Message { + Message::Binary(bytes.into()) +} + +impl Dialer for BearerDialer { + fn url(&self) -> Url { + self.url.clone() + } + + fn connect( + &self, + ) -> BoxFuture<'static, Result>> + { + let url = self.url.clone(); + let token_source = self.token_source.clone(); + Box::pin(async move { + // Fresh token per (re)connect — the auth bridge may have refreshed + // it since the last attempt. + let bearer = token_source.fresh_bearer().await?; + let request = build_auth_request(&url, &bearer)?; + + let (ws, _response) = tokio_tungstenite::connect_async(request).await?; + let (write, read) = ws.split(); + + // Inbound: tungstenite frames -> sync bytes (dropping control frames). + let msg_stream = read + .filter_map(|res| async move { + match res { + Ok(msg) => inbound_to_bytes(msg), + Err(e) => Some(Err(ProviderError::Protocol(format!( + "websocket receive error: {e}" + )))), + } + }) + .boxed(); + + // Outbound: sync bytes -> binary frames. + let msg_sink = write + .sink_map_err(|e| ProviderError::Protocol(format!("websocket send error: {e}"))) + .with(|bytes: Vec| { + futures::future::ready(Ok::(outbound_to_ws(bytes))) + }); + + Ok(Transport::new(msg_stream, msg_sink)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn auth_request_carries_bearer_header_and_target() { + let url = Url::parse("wss://hub.example.com/ws").unwrap(); + let req = build_auth_request(&url, "tok-123").unwrap(); + + let auth = req + .headers() + .get(AUTHORIZATION) + .expect("Authorization header present"); + assert_eq!(auth, "Bearer tok-123"); + assert_eq!(req.uri().host(), Some("hub.example.com")); + assert_eq!(req.uri().path(), "/ws"); + } + + #[test] + fn auth_request_rejects_a_token_with_invalid_header_bytes() { + let url = Url::parse("wss://hub.example.com/ws").unwrap(); + // A newline is not a legal header value byte. + let err = build_auth_request(&url, "bad\ntoken").unwrap_err(); + assert!(matches!(err, ProviderError::Handshake(_))); + } + + #[test] + fn binary_frames_map_to_their_bytes() { + let out = inbound_to_bytes(Message::Binary(vec![1, 2, 3].into())); + assert_eq!(out.unwrap().unwrap(), vec![1, 2, 3]); + } + + #[test] + fn control_frames_are_dropped() { + assert!(inbound_to_bytes(Message::Close(None)).is_none()); + assert!(inbound_to_bytes(Message::Ping(vec![].into())).is_none()); + assert!(inbound_to_bytes(Message::Pong(vec![].into())).is_none()); + } + + #[test] + fn text_frames_are_a_protocol_error() { + let out = inbound_to_bytes(Message::Text("hello".into())); + assert!(matches!(out, Some(Err(ProviderError::Protocol(_))))); + } + + #[test] + fn outbound_bytes_become_a_binary_frame() { + match outbound_to_ws(vec![4, 5, 6]) { + Message::Binary(data) => assert_eq!(data.to_vec(), vec![4, 5, 6]), + other => panic!("expected Binary frame, got {other:?}"), + } + } +} diff --git a/crates/quarto-hub-provider/src/exec_channel.rs b/crates/quarto-hub-provider/src/exec_channel.rs new file mode 100644 index 000000000..79bc98a4c --- /dev/null +++ b/crates/quarto-hub-provider/src/exec_channel.rs @@ -0,0 +1,219 @@ +//! Rust mirror of the editor's execution channel wire format (bd-sfet3264, +//! Phase 4a). +//! +//! The hub-client editor and this provider exchange two ephemeral, +//! project-scoped signals on the **index** `DocHandle` (see the TS side in +//! `hub-client/src/services/executionChannel.ts`): +//! +//! - a **capability beacon** the provider re-broadcasts periodically +//! (`exec/beacon` — "I'm online and can run engines X, Y"), and +//! - an **execute request** the editor sends (`exec/request` — "run this +//! document now"). +//! +//! ## Cross-language encoding +//! +//! Automerge's ephemeral messages carry an opaque byte payload. The browser's +//! `DocHandle.broadcast(obj)` CBOR-encodes `obj` (cbor-x with +//! `{ useRecords: false }` → *standard* CBOR maps with string keys) and the +//! samod docs note the JS side "will only process payloads which are valid +//! CBOR". So we encode/decode the exact same shape with `ciborium`: an +//! internally-tagged enum on `kind`, camelCase field names — byte-compatible +//! with what the editor produces and consumes. +//! +//! In Phase 4a the provider only *sends* beacons and *receives* requests; +//! liveness bookkeeping (applyBeacon/pruneExecutors) is the editor's job and +//! stays on the TS side. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +/// How often the provider re-broadcasts its capability beacon. Mirrors +/// `BEACON_INTERVAL_MS = 3000` on the TS side. +pub const BEACON_INTERVAL: Duration = Duration::from_millis(3000); + +/// Liveness window the editor uses to mark a provider offline (1.5× the +/// interval — the locked `TIMEOUT = 1.5 × INTERVAL` contract from D2). The +/// provider doesn't consume this directly; it's mirrored here so both sides +/// name the same contract. +pub const BEACON_TIMEOUT: Duration = Duration::from_millis(4500); + +/// One message on the execution channel. Internally tagged on `kind` and +/// serialized with camelCase field names to match the TS `ExecMessage` union. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind")] +pub enum ExecMessage { + /// Provider → editors: "I'm online and can run these engines." + #[serde(rename = "exec/beacon")] + Beacon { + #[serde(rename = "actorId")] + actor_id: String, + engines: Vec, + /// Monotonic per-provider counter. Unused in Phase 4a (no claims); + /// reserved for the D5 `--force` takeover (Phase 6). + generation: u64, + }, + /// Editor → provider: "please run this document now." + #[serde(rename = "exec/request")] + Request { + path: String, + #[serde(rename = "requestId")] + request_id: String, + #[serde(rename = "requesterActorId")] + requester_actor_id: String, + }, +} + +impl ExecMessage { + /// Build a capability beacon. + pub fn beacon(actor_id: impl Into, engines: Vec, generation: u64) -> Self { + ExecMessage::Beacon { + actor_id: actor_id.into(), + engines, + generation, + } + } + + /// CBOR-encode this message for `DocHandle::broadcast`. + pub fn to_cbor(&self) -> Vec { + let mut buf = Vec::new(); + ciborium::into_writer(self, &mut buf) + .expect("serializing an ExecMessage to CBOR cannot fail"); + buf + } +} + +/// Decode an untrusted ephemeral payload into a typed [`ExecMessage`], or +/// `None` if it isn't a well-formed execution message. The index handle may +/// carry other ephemeral traffic, so anything that isn't an `exec/*` message we +/// recognize is ignored (mirrors the TS `parseExecMessage`). +pub fn parse_exec_message(bytes: &[u8]) -> Option { + ciborium::from_reader(bytes).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use ciborium::value::Value; + + #[test] + fn beacon_round_trips_through_cbor() { + let msg = ExecMessage::beacon("actor-1", vec!["knitr".into(), "jupyter".into()], 3); + let bytes = msg.to_cbor(); + assert_eq!(parse_exec_message(&bytes), Some(msg)); + } + + #[test] + fn request_round_trips_through_cbor() { + let msg = ExecMessage::Request { + path: "notebook.qmd".into(), + request_id: "req-abc".into(), + requester_actor_id: "actor-2".into(), + }; + let bytes = msg.to_cbor(); + assert_eq!(parse_exec_message(&bytes), Some(msg)); + } + + /// The strongest cross-language check we can make without the browser: the + /// CBOR bytes must decode to a *standard* CBOR map whose keys are exactly + /// the ones the TS `parseExecMessage` inspects. If serde ever emitted an + /// array-tagged enum or snake_case keys, the editor would silently drop + /// our beacon. + #[test] + fn beacon_cbor_shape_matches_the_ts_contract() { + let msg = ExecMessage::beacon("actor-1", vec!["knitr".into()], 7); + let value: Value = ciborium::from_reader(msg.to_cbor().as_slice()).unwrap(); + let Value::Map(entries) = value else { + panic!("beacon must encode as a CBOR map, got {value:?}"); + }; + let keys: Vec = entries + .iter() + .filter_map(|(k, _)| k.as_text().map(str::to_string)) + .collect(); + assert!(keys.contains(&"kind".to_string()), "keys: {keys:?}"); + assert!(keys.contains(&"actorId".to_string()), "keys: {keys:?}"); + assert!(keys.contains(&"engines".to_string()), "keys: {keys:?}"); + assert!(keys.contains(&"generation".to_string()), "keys: {keys:?}"); + + // `kind` must be the exact discriminator string the editor switches on. + let kind = entries + .iter() + .find(|(k, _)| k.as_text() == Some("kind")) + .and_then(|(_, v)| v.as_text()) + .unwrap(); + assert_eq!(kind, "exec/beacon"); + } + + #[test] + fn request_cbor_shape_uses_camel_case_keys() { + let msg = ExecMessage::Request { + path: "a.qmd".into(), + request_id: "r1".into(), + requester_actor_id: "actor".into(), + }; + let value: Value = ciborium::from_reader(msg.to_cbor().as_slice()).unwrap(); + let Value::Map(entries) = value else { + panic!("request must encode as a CBOR map"); + }; + let keys: Vec = entries + .iter() + .filter_map(|(k, _)| k.as_text().map(str::to_string)) + .collect(); + for expected in ["kind", "path", "requestId", "requesterActorId"] { + assert!( + keys.contains(&expected.to_string()), + "missing {expected}: {keys:?}" + ); + } + } + + #[test] + fn junk_and_unknown_kinds_parse_to_none() { + assert_eq!(parse_exec_message(&[0xff, 0x00, 0x13]), None); + assert_eq!(parse_exec_message(b"not cbor at all"), None); + + // Valid CBOR, but not an exec message. + let mut other = Vec::new(); + ciborium::into_writer( + &Value::Map(vec![( + Value::Text("kind".into()), + Value::Text("presence/hello".into()), + )]), + &mut other, + ) + .unwrap(); + assert_eq!(parse_exec_message(&other), None); + } + + #[test] + fn a_request_from_the_editor_is_decoded_from_a_cbor_map() { + // Simulate exactly what the browser sends: a plain CBOR map (as cbor-x + // with useRecords:false produces) with the editor's field names. + let mut bytes = Vec::new(); + ciborium::into_writer( + &Value::Map(vec![ + ( + Value::Text("kind".into()), + Value::Text("exec/request".into()), + ), + (Value::Text("path".into()), Value::Text("report.qmd".into())), + (Value::Text("requestId".into()), Value::Text("req-9".into())), + ( + Value::Text("requesterActorId".into()), + Value::Text("editor-actor".into()), + ), + ]), + &mut bytes, + ) + .unwrap(); + + assert_eq!( + parse_exec_message(&bytes), + Some(ExecMessage::Request { + path: "report.qmd".into(), + request_id: "req-9".into(), + requester_actor_id: "editor-actor".into(), + }) + ); + } +} diff --git a/crates/quarto-hub-provider/src/execute.rs b/crates/quarto-hub-provider/src/execute.rs new file mode 100644 index 000000000..c56802681 --- /dev/null +++ b/crates/quarto-hub-provider/src/execute.rs @@ -0,0 +1,385 @@ +//! Execute-on-request loop for the code-execution provider (bd-sfet3264, +//! Phase 4a). +//! +//! Once joined to a hub, a [`Provider`] does two things on the index +//! `DocHandle`'s ephemeral channel: +//! +//! 1. **Broadcasts a capability beacon** every [`BEACON_INTERVAL`] so editors +//! know an executor is online and which engines it can run. +//! 2. **Listens for `exec/request`** messages and, when the [`AuthzPolicy`] +//! allows the requester, materializes the project, runs the engines +//! natively (the same uncached `record_capture` path `q2 preview` uses), +//! and writes the result back as a capture binary doc + `CaptureRef` +//! sidecar entry — the transport the Phase 1 editor already consumes. +//! +//! The engine work runs on a blocking worker (`spawn_blocking` + +//! `pollster::block_on`), mirroring `quarto-preview`'s `re_execute.rs`, so a +//! long engine run never stalls the ephemeral-message reactor. +//! +//! ## Authorization (Phase 4a: mechanism-first, fail closed) +//! +//! [`AuthzPolicy`] is `AllowAll` (opened by `q2 provide-hub --allow-all`) or +//! `Deny` (the safe default: refuse every request). The real provider-only +//! gating — honoring a request only when `requesterActorId` equals the +//! provider's own per-project actor id — needs the hub to accept a Bearer on +//! `GET /auth/actor` and lands in Phase 5 as a third `ProviderOnly` variant +//! this enum is the seam for. + +use std::collections::HashSet; +use std::path::Component; +use std::sync::{Arc, Mutex}; + +use flate2::Compression; +use flate2::write::GzEncoder; +use futures::StreamExt; +use quarto_core::engine::EngineRegistry; +use quarto_core::engine::preview_record::record_capture; +use quarto_core::project::ProjectContext; +use quarto_hub::index::{CaptureRef, CaptureState, IndexDocument}; +use quarto_hub::resource::create_binary_document; +use quarto_system_runtime::{NativeRuntime, SystemRuntime}; +use quarto_trace::EngineCapture; +use samod::Repo; +use std::io::Write as _; + +use crate::ProviderError; +use crate::exec_channel::{BEACON_INTERVAL, ExecMessage, parse_exec_message}; + +/// MIME type stamped on capture binary docs. Must stay byte-identical to +/// `quarto_preview::capture_driver::CAPTURE_MIME_TYPE` and the literal the TS +/// consumers use (`ts-packages/quarto-sync-client`, +/// `hub-client/.../ReactPreview.capture.integration.test.tsx`). Duplicated here +/// rather than depending on the heavy `quarto-preview` crate (which pulls in an +/// axum server); the value is a self-describing label, not validated on read. +pub const CAPTURE_MIME_TYPE: &str = "application/x-engine-capture+gzip"; + +/// Who may trigger execution on this provider. +/// +/// Phase 4a ships only the two poles; Phase 5 adds +/// `ProviderOnly { self_actor_id }` (gate on `requesterActorId == self`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuthzPolicy { + /// Honor requests from anyone with access to the document + /// (`q2 provide-hub --allow-all`). + AllowAll, + /// Refuse every request (the safe default when `--allow-all` is absent). + Deny, +} + +impl AuthzPolicy { + /// Whether a request from `requester_actor_id` may run. + pub fn allows(&self, _requester_actor_id: &str) -> bool { + match self { + AuthzPolicy::AllowAll => true, + AuthzPolicy::Deny => false, + } + } +} + +/// A joined provider ready to serve execution requests. Held behind an `Arc` +/// so the beacon loop and per-request workers can share it. +pub struct Provider { + repo: Repo, + index: IndexDocument, + /// Beacon `actorId`. Phase 4a uses the samod peer id (stable for the + /// process); Phase 5 swaps in the per-project actor id from `/auth/actor`. + self_actor_id: String, + /// Engines advertised in the beacon (available, non-markdown). + engines: Vec, + authz: AuthzPolicy, + /// Engine registry override for the capture run. `None` in production (the + /// default registry); tests pass a passthrough engine. `Clone` is cheap + /// (engines are `Arc`ed) so each run gets its own copy. + registry: Option, + /// Paths currently executing, to collapse a duplicate request for a path + /// already in flight (mirrors `re_execute.rs`'s `IN_FLIGHT`). + in_flight: Mutex>, +} + +impl Provider { + /// Build a provider around a joined repo + index. + pub fn new( + repo: Repo, + index: IndexDocument, + self_actor_id: impl Into, + authz: AuthzPolicy, + registry: Option, + ) -> Arc { + let probe = registry.clone().unwrap_or_default(); + let engines = available_engines(&probe); + Arc::new(Self { + repo, + index, + self_actor_id: self_actor_id.into(), + engines, + authz, + registry, + in_flight: Mutex::new(HashSet::new()), + }) + } + + /// Run until `shutdown` resolves: broadcast the beacon on a timer and serve + /// requests from the ephemeral channel. + pub async fn run(self: Arc, shutdown: S) + where + S: std::future::Future + Send, + { + let beacon = tokio::spawn({ + let provider = Arc::clone(&self); + async move { provider.run_beacon_loop().await } + }); + + tokio::select! { + () = Arc::clone(&self).run_request_loop() => {} + () = shutdown => {} + } + + beacon.abort(); + } + + /// Broadcast the capability beacon every [`BEACON_INTERVAL`], bumping the + /// generation each tick. (Generation is reserved for the D5 `--force` + /// takeover; unused by editors in Phase 4a.) + async fn run_beacon_loop(self: Arc) { + let mut generation = 0u64; + let mut ticker = tokio::time::interval(BEACON_INTERVAL); + loop { + ticker.tick().await; + let beacon = + ExecMessage::beacon(self.self_actor_id.clone(), self.engines.clone(), generation); + self.index.handle().broadcast(beacon.to_cbor()); + generation = generation.wrapping_add(1); + } + } + + /// Consume the index handle's ephemeral messages, dispatching each allowed + /// `exec/request` to a blocking worker. + async fn run_request_loop(self: Arc) { + let mut stream = self.index.handle().ephemera(); + while let Some(bytes) = stream.next().await { + let Some(ExecMessage::Request { + path, + request_id, + requester_actor_id, + }) = parse_exec_message(&bytes) + else { + continue; // beacons from other providers, or non-exec traffic + }; + + if !self.authz.allows(&requester_actor_id) { + tracing::info!( + path = %path, + request_id = %request_id, + requester = %requester_actor_id, + "refusing exec request (provider not opened with --allow-all)" + ); + continue; + } + + if !self.claim(&path) { + tracing::debug!(path = %path, "exec request skipped: already in flight"); + continue; + } + + let provider = Arc::clone(&self); + let path_for_task = path.clone(); + tokio::task::spawn_blocking(move || { + let result = pollster::block_on(provider.execute_document(&path_for_task)); + provider.release(&path_for_task); + match result { + Ok(doc_id) => tracing::info!( + path = %path_for_task, + capture_doc_id = %doc_id, + "wrote capture for exec request" + ), + Err(e) => { + tracing::warn!(path = %path_for_task, error = %e, "exec request failed") + } + } + }); + } + } + + /// Reserve `path` in the in-flight set; returns false if already claimed. + fn claim(&self, path: &str) -> bool { + self.in_flight + .lock() + .expect("in-flight mutex poisoned") + .insert(path.to_string()) + } + + fn release(&self, path: &str) { + self.in_flight + .lock() + .expect("in-flight mutex poisoned") + .remove(path); + } + + /// Materialize the project, run the engines for `rel_path`, and write the + /// capture doc + sidecar. Returns the new capture document id. Public for + /// the scripted integration test (drives one request without networking). + pub async fn execute_document(&self, rel_path: &str) -> Result { + if !self.index.has_file(rel_path) { + return Err(format!("path '{rel_path}' is not in the project index")); + } + if !is_safe_relative(rel_path) { + return Err(format!("refusing to execute unsafe path '{rel_path}'")); + } + + // Flip an *existing* capture to `running` for durable status. A + // first-ever run has no CaptureRef to attach state to (the doc id is + // required), so it goes straight to producing the capture — same as + // re_execute.rs. + if let Some(existing) = self.index.get_capture(rel_path) { + let running = CaptureRef { + capture_doc_id: existing.capture_doc_id, + staleness: existing.staleness, + state: Some(CaptureState::Running), + last_error: None, + }; + let _ = self.index.set_capture(rel_path, &running); + } + + let result = self.run_and_store(rel_path).await; + + if let Err(msg) = &result { + // Surface the failure on an existing capture entry (best effort). + if let Some(existing) = self.index.get_capture(rel_path) { + let errored = CaptureRef { + capture_doc_id: existing.capture_doc_id, + staleness: existing.staleness, + state: Some(CaptureState::Error), + last_error: Some(msg.clone()), + }; + let _ = self.index.set_capture(rel_path, &errored); + } + } + + result + } + + /// The happy-path body of [`execute_document`](Self::execute_document): + /// materialize → discover → record (uncached) → write doc → set sidecar. + async fn run_and_store(&self, rel_path: &str) -> Result { + let tmp = + tempfile::tempdir().map_err(|e| format!("creating a materialization temp dir: {e}"))?; + crate::materialize::materialize_project(&self.repo, &self.index, tmp.path()) + .await + .map_err(|e| format!("materializing the project: {e}"))?; + + let abs_path = tmp.path().join(rel_path); + let runtime: Arc = Arc::new(NativeRuntime::new()); + let project = ProjectContext::discover(&abs_path, runtime.as_ref()) + .map_err(|e| format!("project discovery failed: {e}"))?; + + // Decision #4: always a fresh run (uncached record_capture) — code may + // have side effects and we don't prove it side-effect-free. + let captures = record_capture(&abs_path, &project, runtime.clone(), self.registry.clone()) + .await + .map_err(|e| format!("engine pipeline failed: {e}"))?; + if captures.is_empty() { + return Err("engine produced no capture (no code cells?)".to_string()); + } + + let new_doc_id = write_capture_doc(&self.repo, &captures) + .await + .map_err(|e| format!("failed to store capture binary doc: {e}"))?; + + let updated = CaptureRef { + capture_doc_id: new_doc_id.clone(), + staleness: Some(false), + state: Some(CaptureState::Idle), + last_error: None, + }; + self.index + .set_capture(rel_path, &updated) + .map_err(|e| format!("failed to update sidecar: {e}"))?; + + Ok(new_doc_id) + } +} + +/// Names of available execution engines, excluding the always-present markdown +/// engine (not a real "run" target). Advertised in the capability beacon. +fn available_engines(registry: &EngineRegistry) -> Vec { + let mut names: Vec = registry + .engine_names() + .into_iter() + .filter(|name| *name != "markdown") + .filter(|name| registry.get(name).is_some_and(|e| e.is_available())) + .map(str::to_string) + .collect(); + names.sort(); + names +} + +/// Whether `rel_path` is a safe project-relative path (no `..`, not absolute). +fn is_safe_relative(rel_path: &str) -> bool { + let path = std::path::Path::new(rel_path); + path.components() + .all(|c| matches!(c, Component::Normal(_) | Component::CurDir)) + && path.components().any(|c| matches!(c, Component::Normal(_))) +} + +/// Gzip the captures to JSON and store them as a capture binary automerge doc. +/// Mirror of `quarto-preview`'s `write_capture_doc` (kept in sync via the +/// shared [`CAPTURE_MIME_TYPE`] and JSON+gzip wire format). +async fn write_capture_doc( + repo: &Repo, + captures: &[EngineCapture], +) -> Result { + let json = serde_json::to_vec(captures) + .map_err(|e| ProviderError::Protocol(format!("serialize captures: {e}")))?; + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(&json) + .map_err(|e| ProviderError::Protocol(format!("gzip write: {e}")))?; + let gzipped = enc + .finish() + .map_err(|e| ProviderError::Protocol(format!("gzip finish: {e}")))?; + let doc = create_binary_document(&gzipped, CAPTURE_MIME_TYPE) + .map_err(|e| ProviderError::Protocol(format!("binary doc: {e}")))?; + let handle = repo + .create(doc) + .await + .map_err(|_| ProviderError::Repo("samod repo stopped".into()))?; + Ok(handle.document_id().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn allow_all_permits_any_requester() { + assert!(AuthzPolicy::AllowAll.allows("anyone")); + assert!(AuthzPolicy::AllowAll.allows("")); + } + + #[test] + fn deny_refuses_every_requester() { + assert!(!AuthzPolicy::Deny.allows("anyone")); + assert!(!AuthzPolicy::Deny.allows("")); + } + + #[test] + fn is_safe_relative_guards_traversal() { + assert!(is_safe_relative("report.qmd")); + assert!(is_safe_relative("chapters/intro.qmd")); + assert!(is_safe_relative("./a.qmd")); + assert!(!is_safe_relative("../escape.qmd")); + assert!(!is_safe_relative("a/../../x.qmd")); + assert!(!is_safe_relative("/etc/passwd")); + assert!(!is_safe_relative("")); + assert!(!is_safe_relative(".")); + } + + #[test] + fn available_engines_excludes_markdown() { + let registry = EngineRegistry::default(); + let engines = available_engines(®istry); + assert!( + !engines.iter().any(|e| e == "markdown"), + "markdown must not be advertised: {engines:?}" + ); + } +} diff --git a/crates/quarto-hub-provider/src/join.rs b/crates/quarto-hub-provider/src/join.rs new file mode 100644 index 000000000..d229d94e0 --- /dev/null +++ b/crates/quarto-hub-provider/src/join.rs @@ -0,0 +1,71 @@ +//! Join a hub as a client peer and list its files (Phase 3 deliverable). + +use std::sync::Arc; +use std::time::Duration; + +use quarto_hub::index::IndexDocument; +use samod::{BackoffConfig, Repo}; + +use crate::ProviderError; +use crate::dialer::BearerDialer; +use crate::token::TokenSource; + +/// What to connect to. +pub struct JoinConfig { + /// The hub's websocket endpoint, e.g. `wss://quarto-hub.com/ws`. + pub server_ws_url: url::Url, + /// The project's automerge index document id. + pub index_doc_id: String, + /// How long to wait for the peer connection to establish. + pub connect_timeout: Duration, +} + +/// Join the hub as an ephemeral (memory-storage) client peer, dialing with a +/// [`BearerDialer`], then `find()` the index document. Returns the live +/// [`Repo`] and [`IndexDocument`] so the caller can keep syncing — materialize +/// files, watch the ephemeral channel, and write captures back (Phase 4). +/// +/// A client peer keeps nothing on disk: the project lives on the hub, and each +/// execution materializes to a temp dir per run. +pub async fn join( + config: JoinConfig, + token_source: Arc, +) -> Result<(Repo, IndexDocument), ProviderError> { + let repo = Repo::build_tokio().load().await; + + let dialer = Arc::new(BearerDialer::new( + config.server_ws_url.clone(), + token_source, + )); + let handle = repo + .dial(BackoffConfig::default(), dialer) + .map_err(|_| ProviderError::Repo("repo is stopped".into()))?; + + // Wait for the first connection (or a failure) within the timeout. + tokio::time::timeout(config.connect_timeout, handle.established()) + .await + .map_err(|_| ProviderError::Repo("timed out connecting to hub".into()))? + .map_err(|_| ProviderError::Repo("hub connection failed (auth rejected?)".into()))?; + + let index = IndexDocument::load(&repo, &config.index_doc_id) + .await + .map_err(|e| ProviderError::Index(e.to_string()))? + .ok_or_else(|| ProviderError::Index("index document not found".into()))?; + + Ok((repo, index)) +} + +/// Join the hub and return the sorted list of project file paths. +/// +/// This is the narrow Phase 3 success criterion: it exercises the full +/// authenticated sync path (BearerDialer handshake → samod sync → index doc) +/// without any execution. +pub async fn join_and_list_files( + config: JoinConfig, + token_source: Arc, +) -> Result, ProviderError> { + let (_repo, index) = join(config, token_source).await?; + let mut files: Vec = index.get_all_files().into_keys().collect(); + files.sort(); + Ok(files) +} diff --git a/crates/quarto-hub-provider/src/lib.rs b/crates/quarto-hub-provider/src/lib.rs new file mode 100644 index 000000000..c9c06f58b --- /dev/null +++ b/crates/quarto-hub-provider/src/lib.rs @@ -0,0 +1,54 @@ +//! Connect a native `q2` process to a hub's automerge session as a +//! code-execution provider (bd-sfet3264, Phase 3). +//! +//! The hybrid architecture (D1=C): a Node child owns the OAuth/keyring auth +//! and streams Bearer tokens; this Rust side owns the automerge sync + (later, +//! Phase 4) the engine execution. It joins the hub as a samod **client peer** +//! by dialing the remote `/ws` with a [`BearerDialer`] that injects an +//! `Authorization: Bearer ` header — re-fetched on every (re)connect from +//! a [`TokenSource`]. +//! +//! Phase 3 deliverable is narrow: join an authenticated hub, `find()` the +//! index document, and list the project files. Execution, the capability +//! beacon, and temp-dir materialization land in Phase 4. + +mod dialer; +mod exec_channel; +mod execute; +mod join; +mod materialize; +mod token; +mod token_bridge; + +pub use dialer::BearerDialer; +pub use exec_channel::{BEACON_INTERVAL, BEACON_TIMEOUT, ExecMessage, parse_exec_message}; +pub use execute::{AuthzPolicy, CAPTURE_MIME_TYPE, Provider}; +pub use join::{JoinConfig, join, join_and_list_files}; +pub use materialize::materialize_project; +pub use token::{StaticTokenSource, TokenSource}; +pub use token_bridge::NodeBridge; + +/// Errors from joining a hub as an execution provider. +#[derive(Debug, thiserror::Error)] +pub enum ProviderError { + /// Failed to build or perform the websocket handshake (bad URL, header). + #[error("websocket handshake error: {0}")] + Handshake(String), + + /// A transport-level protocol violation (e.g. a text frame on the sync + /// socket, or a send/receive error). + #[error("sync transport error: {0}")] + Protocol(String), + + /// The auth bridge could not provide a token. + #[error("token error: {0}")] + Token(String), + + /// The samod repo was stopped (or could not be reached). + #[error("repo error: {0}")] + Repo(String), + + /// The index document could not be found or loaded. + #[error("index document error: {0}")] + Index(String), +} diff --git a/crates/quarto-hub-provider/src/materialize.rs b/crates/quarto-hub-provider/src/materialize.rs new file mode 100644 index 000000000..7806d6c6e --- /dev/null +++ b/crates/quarto-hub-provider/src/materialize.rs @@ -0,0 +1,140 @@ +//! Materialize a hub project's automerge VFS into a real on-disk directory +//! (bd-sfet3264, Phase 4a). +//! +//! Native engines (knitr/jupyter) read files from the filesystem, so before we +//! can run a document we copy the whole project out of automerge into a fresh +//! temp dir. Per the Phase 3 decision this is **read-only and per-run**: we +//! never write back, and each execution starts from a clean tree. (The hub's +//! own `sync_all_documents` is bidirectional and sync-state-heavy; this is a +//! lean one-way reader reusing the `resource::*` primitives.) +//! +//! Text file docs store a `text` Automerge `Text` object; binary file docs +//! store a `content` bytes field (`quarto_hub::resource`). We detect the type +//! and write the raw bytes under `/`. + +use std::path::{Component, Path, PathBuf}; +use std::str::FromStr; + +use automerge::{Automerge, ROOT, ReadDoc}; +use quarto_hub::index::IndexDocument; +use quarto_hub::resource::{DocumentType, detect_document_type, read_binary_content}; +use samod::{DocumentId, Repo}; +use tracing::warn; + +use crate::ProviderError; + +/// Copy every file tracked in `index` out of the automerge repo and into +/// `dest`, preserving relative paths. Returns the number of files written. +/// +/// Files whose document can't be found or whose content can't be read are +/// **skipped with a warning** rather than failing the whole run — a single +/// unreadable asset shouldn't block executing the document the user asked for. +/// Paths that would escape `dest` (absolute, or containing `..`) are rejected. +pub async fn materialize_project( + repo: &Repo, + index: &IndexDocument, + dest: &Path, +) -> Result { + let mut written = 0usize; + for (rel_path, doc_id_str) in index.get_all_files() { + let Some(target) = safe_join(dest, &rel_path) else { + warn!(path = %rel_path, "skipping file with an unsafe path during materialization"); + continue; + }; + + let doc_id = match DocumentId::from_str(&doc_id_str) { + Ok(id) => id, + Err(e) => { + warn!(path = %rel_path, error = %e, "skipping file with an invalid document id"); + continue; + } + }; + + let handle = repo + .find(doc_id) + .await + .map_err(|_| ProviderError::Repo("repo is stopped".into()))?; + let Some(handle) = handle else { + warn!(path = %rel_path, "skipping file whose document was not found on the hub"); + continue; + }; + + let bytes = handle.with_document(|doc| read_file_bytes(doc)); + let Some(bytes) = bytes else { + warn!(path = %rel_path, "skipping file with unreadable content"); + continue; + }; + + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + ProviderError::Protocol(format!("creating {}: {e}", parent.display())) + })?; + } + std::fs::write(&target, &bytes) + .map_err(|e| ProviderError::Protocol(format!("writing {}: {e}", target.display())))?; + written += 1; + } + Ok(written) +} + +/// Read a file document's bytes: the hydrated UTF-8 of a text doc, or the raw +/// `content` bytes of a binary doc. Returns `None` for an unrecognized shape. +fn read_file_bytes(doc: &Automerge) -> Option> { + match detect_document_type(doc) { + DocumentType::Text => { + let (_, text_obj) = doc.get(ROOT, "text").ok().flatten()?; + let text = doc.text(&text_obj).ok()?; + Some(text.into_bytes()) + } + DocumentType::Binary => read_binary_content(doc), + DocumentType::Invalid => None, + } +} + +/// Join `rel` under `base`, rejecting anything that would escape it (absolute +/// paths, `..` components, or Windows drive prefixes). Returns `None` when the +/// path is unsafe. Cross-platform: uses `Path::components` rather than string +/// separators. +fn safe_join(base: &Path, rel: &str) -> Option { + let rel_path = Path::new(rel); + let mut out = base.to_path_buf(); + for component in rel_path.components() { + match component { + Component::Normal(part) => out.push(part), + // Reject anything non-relative or upward-traversing. + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return None, + // A leading `./` is harmless. + Component::CurDir => {} + } + } + // Guard against a path that normalized to just `base` (e.g. "" or "."). + if out == base { None } else { Some(out) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn safe_join_accepts_nested_relative_paths() { + let base = Path::new("/tmp/x"); + assert_eq!( + safe_join(base, "chapters/intro.qmd"), + Some(PathBuf::from("/tmp/x/chapters/intro.qmd")) + ); + assert_eq!( + safe_join(base, "./a.qmd"), + Some(PathBuf::from("/tmp/x/a.qmd")) + ); + } + + #[test] + fn safe_join_rejects_traversal_and_absolute() { + let base = Path::new("/tmp/x"); + assert_eq!(safe_join(base, "../escape.qmd"), None); + assert_eq!(safe_join(base, "a/../../escape.qmd"), None); + assert_eq!(safe_join(base, "/etc/passwd"), None); + assert_eq!(safe_join(base, ""), None); + assert_eq!(safe_join(base, "."), None); + } +} diff --git a/crates/quarto-hub-provider/src/token.rs b/crates/quarto-hub-provider/src/token.rs new file mode 100644 index 000000000..c96bd2189 --- /dev/null +++ b/crates/quarto-hub-provider/src/token.rs @@ -0,0 +1,37 @@ +//! Bearer-token supply for the [`BearerDialer`](crate::BearerDialer). + +use std::future::Future; +use std::pin::Pin; + +use crate::ProviderError; + +/// A future yielding a fresh Bearer token (Send: the dialer runs on the +/// multi-threaded runtime and samod's `Dialer::connect` returns a Send future). +pub type BearerFuture = Pin> + Send>>; + +/// Supplies a fresh Bearer token for each (re)connect. +/// +/// Called by [`BearerDialer::connect`](crate::BearerDialer) on the initial +/// dial and every reconnection, so the implementation can hand back the latest +/// token the Node auth bridge has streamed (Phase 3C) — keeping reconnections +/// authenticated across token refreshes without restarting the process. +pub trait TokenSource: Send + Sync + 'static { + fn fresh_bearer(&self) -> BearerFuture; +} + +/// A fixed token. Used by tests and the dev `--token` path before the Node +/// auth bridge exists (Phase 3B). +pub struct StaticTokenSource(pub String); + +impl StaticTokenSource { + pub fn new(token: impl Into) -> Self { + Self(token.into()) + } +} + +impl TokenSource for StaticTokenSource { + fn fresh_bearer(&self) -> BearerFuture { + let token = self.0.clone(); + Box::pin(async move { Ok(token) }) + } +} diff --git a/crates/quarto-hub-provider/src/token_bridge.rs b/crates/quarto-hub-provider/src/token_bridge.rs new file mode 100644 index 000000000..fde2ea9a8 --- /dev/null +++ b/crates/quarto-hub-provider/src/token_bridge.rs @@ -0,0 +1,274 @@ +//! The Node auth bridge: spawn the bundled `auth-stream` helper and expose its +//! streamed Bearer tokens as a [`TokenSource`] (bd-sfet3264, Phase 3C). +//! +//! Topology (D1=C): this Rust process is the long-running parent. It spawns the +//! Node helper (`auth-stream.mjs`, extracted from the embedded hub-mcp bundle) +//! as a child, reads newline-delimited token frames from its **stdout**, and +//! can write `{"type":"refresh"}` to its **stdin** to pull a fresh token. The +//! helper's logs + interactive sign-in URL go to its **stderr**, which we +//! inherit so the user sees them. stdio pipes are identical across platforms. + +use std::process::Stdio; +use std::sync::Arc; +use std::sync::Mutex; + +use serde::Deserialize; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::process::{Child, ChildStdin, Command}; +use tokio::sync::{Mutex as AsyncMutex, Notify}; + +use quarto_mcp_launcher::{ + Discovery, ExtractedBundle, bundled_defaults, content_hash, default_cache_root, embedded_files, + extract_and_lock, find_node, injections, is_placeholder, +}; + +use crate::ProviderError; +use crate::token::{BearerFuture, TokenSource}; + +/// A frame on the helper's stdout. Matches the wire shape emitted by +/// `ts-packages/quarto-hub-mcp/src/auth-stream/protocol.ts`. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub(crate) enum AuthFrame { + Token { + bearer: String, + #[serde(rename = "expiresAt", default)] + #[allow(dead_code)] + expires_at: String, + }, + Error { + message: String, + }, +} + +/// Parse one stdout line into a frame, or `None` for blank/unrecognized lines +/// (the helper writes logs to stderr, but be defensive about stray stdout). +pub(crate) fn parse_auth_frame(line: &str) -> Option { + let trimmed = line.trim(); + if trimmed.is_empty() { + return None; + } + serde_json::from_str::(trimmed).ok() +} + +/// Latest-token cache shared between the stdout-reader task and waiters. +#[derive(Debug)] +enum TokenState { + Pending, + Ready(String), + Failed(String), +} + +#[derive(Debug)] +struct TokenCache { + state: Mutex, + notify: Notify, +} + +impl TokenCache { + fn new() -> Self { + Self { + state: Mutex::new(TokenState::Pending), + notify: Notify::new(), + } + } + + fn set_ready(&self, bearer: String) { + *self.state.lock().expect("token cache poisoned") = TokenState::Ready(bearer); + self.notify.notify_waiters(); + } + + /// An error frame is fatal only while we have no token yet; an error after + /// a good token is non-fatal (the existing token may still be valid). + fn set_failed_if_pending(&self, message: String) { + let mut state = self.state.lock().expect("token cache poisoned"); + if matches!(*state, TokenState::Pending) { + *state = TokenState::Failed(message); + self.notify.notify_waiters(); + } + } + + /// Resolve to the current Bearer, waiting for the first one if the helper + /// is still authenticating. Errors if the helper reported a fatal failure. + async fn get(&self) -> Result { + loop { + // Register for notification *before* inspecting the state so a + // concurrent `notify_waiters` between the check and the await is + // not lost. + let notified = self.notify.notified(); + { + let state = self.state.lock().expect("token cache poisoned"); + match &*state { + TokenState::Ready(token) => return Ok(token.clone()), + TokenState::Failed(message) => { + return Err(ProviderError::Token(message.clone())); + } + TokenState::Pending => {} + } + } + notified.await; + } + } +} + +/// A running Node auth bridge. Holds the child process and the extracted-bundle +/// lock for its lifetime; dropping it kills the child (`kill_on_drop`). +pub struct NodeBridge { + _child: Child, + // Async mutex: the write is held across `.await` points (a std Mutex + // can't be, and would risk a deadlock). + stdin: AsyncMutex, + cache: Arc, + // Holds the lifetime shared lock on the extracted bundle dir. + _extracted: ExtractedBundle, +} + +impl NodeBridge { + /// Spawn the bundled auth helper. Must be called within a Tokio runtime + /// (it spawns a background stdout-reader task). + pub fn spawn() -> Result { + if is_placeholder() { + return Err(ProviderError::Token( + "the hub-mcp bundle is not built; run `cargo xtask build-hub-mcp-bundle`".into(), + )); + } + + let files = embedded_files(); + let hash = content_hash(&files); + let cache_root = + default_cache_root().map_err(|e| ProviderError::Token(format!("cache dir: {e}")))?; + let extracted = extract_and_lock(&cache_root, &files, &hash) + .map_err(|e| ProviderError::Token(format!("extract bundle: {e}")))?; + let entry = extracted.dir.join("auth-stream.mjs"); + + let node = find_node(&Discovery::from_env()) + .map_err(|e| ProviderError::Token(format!("node discovery: {e}")))?; + + let mut command = Command::new(&node.path); + command + .arg(&entry) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + // Inherit stderr so the user sees the sign-in URL + helper logs. + .stderr(Stdio::inherit()) + .kill_on_drop(true); + + // Inject the bundled OAuth client id/secret/server when the user's env + // doesn't already set them (mirrors the `q2 mcp` launcher). + for (var, value) in injections(&bundled_defaults(), |k| std::env::var(k).ok()) { + command.env(var, value); + } + + let mut child = command + .spawn() + .map_err(|e| ProviderError::Token(format!("spawn node auth helper: {e}")))?; + + let stdout = child + .stdout + .take() + .ok_or_else(|| ProviderError::Token("auth helper has no stdout".into()))?; + let stdin = child + .stdin + .take() + .ok_or_else(|| ProviderError::Token("auth helper has no stdin".into()))?; + + let cache = Arc::new(TokenCache::new()); + let reader_cache = cache.clone(); + tokio::spawn(async move { + let mut lines = BufReader::new(stdout).lines(); + while let Ok(Some(line)) = lines.next_line().await { + match parse_auth_frame(&line) { + Some(AuthFrame::Token { bearer, .. }) => reader_cache.set_ready(bearer), + Some(AuthFrame::Error { message }) => { + reader_cache.set_failed_if_pending(message) + } + None => {} + } + } + // stdout closed: if we never got a token, surface it as a failure + // so waiters don't hang forever. + reader_cache.set_failed_if_pending("auth helper exited without a token".into()); + }); + + Ok(Self { + _child: child, + stdin: AsyncMutex::new(stdin), + cache, + _extracted: extracted, + }) + } + + /// Ask the helper to mint a fresh token (used before a reconnect once the + /// cached token is near expiry). Best-effort; the new token arrives on the + /// stdout stream and updates the cache. + pub async fn request_refresh(&self) -> Result<(), ProviderError> { + // Async mutex: the guard is safely held across the write/flush awaits. + let mut stdin = self.stdin.lock().await; + stdin + .write_all(b"{\"type\":\"refresh\"}\n") + .await + .map_err(|e| ProviderError::Token(format!("write refresh: {e}")))?; + stdin + .flush() + .await + .map_err(|e| ProviderError::Token(format!("flush refresh: {e}")))?; + Ok(()) + } +} + +impl TokenSource for NodeBridge { + fn fresh_bearer(&self) -> BearerFuture { + let cache = self.cache.clone(); + Box::pin(async move { cache.get().await }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_token_frame() { + let frame = parse_auth_frame( + r#"{"type":"token","bearer":"abc.def.ghi","expiresAt":"2026-07-01T00:00:00Z"}"#, + ); + assert_eq!( + frame, + Some(AuthFrame::Token { + bearer: "abc.def.ghi".into(), + expires_at: "2026-07-01T00:00:00Z".into(), + }) + ); + } + + #[test] + fn parses_a_token_frame_without_expiry() { + let frame = parse_auth_frame(r#"{"type":"token","bearer":"t"}"#); + assert_eq!( + frame, + Some(AuthFrame::Token { + bearer: "t".into(), + expires_at: String::new(), + }) + ); + } + + #[test] + fn parses_an_error_frame() { + let frame = parse_auth_frame(r#"{"type":"error","message":"reauth required"}"#); + assert_eq!( + frame, + Some(AuthFrame::Error { + message: "reauth required".into(), + }) + ); + } + + #[test] + fn ignores_blank_and_non_frame_lines() { + assert_eq!(parse_auth_frame(""), None); + assert_eq!(parse_auth_frame(" "), None); + assert_eq!(parse_auth_frame("[hub-mcp] some log line"), None); + assert_eq!(parse_auth_frame(r#"{"type":"surprise"}"#), None); + } +} diff --git a/crates/quarto-hub-provider/tests/integration/auth_bridge.rs b/crates/quarto-hub-provider/tests/integration/auth_bridge.rs new file mode 100644 index 000000000..fe1e71582 --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/auth_bridge.rs @@ -0,0 +1,57 @@ +//! NodeBridge ↔ `auth-stream` helper plumbing (bd-sfet3264, Phase 3C). +//! +//! Spawns the **real** bundled helper with no OAuth credentials and asserts the +//! bridge reads its stdout, parses the error frame, and surfaces it through +//! `fresh_bearer()`. This exercises the Rust↔Node plumbing (find node, extract +//! bundle, spawn, read stream) end-to-end without real interactive OAuth. +//! +//! Gated: skips cleanly when the embedded bundle isn't built (a plain +//! `cargo nextest` without the bundle step) or Node isn't installed — the +//! plumbing isn't exercisable then. A timeout guards against a build that has +//! compiled-in credentials (which would launch a browser instead of erroring). + +use std::sync::Arc; +use std::time::Duration; + +use quarto_hub_provider::{NodeBridge, TokenSource}; + +#[tokio::test] +async fn node_bridge_surfaces_helper_error_when_unauthenticated() { + // Make sure this process has no OAuth creds so the child helper errors fast + // (a dev build has no compiled-in defaults to inject either). nextest runs + // each test in its own process, so this env edit is isolated. + unsafe { + std::env::remove_var("QUARTO_HUB_MCP_CLIENT_ID"); + std::env::remove_var("QUARTO_HUB_MCP_CLIENT_SECRET"); + } + + let bridge = match NodeBridge::spawn() { + Ok(bridge) => bridge, + Err(e) => { + eprintln!("skipping: auth bridge not spawnable here ({e})"); + return; + } + }; + + let source = Arc::new(bridge); + let result = tokio::time::timeout(Duration::from_secs(20), source.fresh_bearer()).await; + + match result { + Err(_) => { + // Timed out waiting for a token: this build likely has compiled-in + // credentials and the helper is attempting an interactive sign-in. + // Not exercisable as an offline test — skip. + eprintln!("skipping: helper did not error within the deadline (compiled-in creds?)"); + } + Ok(Ok(token)) => panic!("expected an auth error, but got a token: {token}"), + Ok(Err(err)) => { + let msg = err.to_string(); + assert!( + msg.contains("CLIENT_ID") + || msg.contains("not set") + || msg.contains("without a token"), + "expected a missing-credentials error from the helper, got: {msg}" + ); + } + } +} diff --git a/crates/quarto-hub-provider/tests/integration/execute.rs b/crates/quarto-hub-provider/tests/integration/execute.rs new file mode 100644 index 000000000..f427e0c2d --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/execute.rs @@ -0,0 +1,243 @@ +//! Execute-on-request end-to-end test (bd-sfet3264, Phase 4a). +//! +//! A bare samod acceptor stands in for the hub; a seeded index points at a +//! passthrough-engine `.qmd`. A provider joins as a client peer over the real +//! `BearerDialer` transport, then an editor-side broadcast of an `exec/request` +//! on the index handle drives the provider to materialize the project, run the +//! (passthrough) engine, and write a capture binary doc + `CaptureRef` sidecar +//! — which syncs back to the server, exactly as an editor peer would observe. +//! +//! A second test asserts the `Deny` policy writes nothing. + +use std::io::Read as _; +use std::str::FromStr as _; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use automerge::{Automerge, ObjType, ROOT, transaction::Transactable}; +use flate2::read::GzDecoder; +use quarto_core::engine::{ + EngineRegistry, ExecuteResult, ExecutionContext, ExecutionEngine, ExecutionError, +}; +use quarto_hub::index::{CaptureState, IndexDocument}; +use quarto_hub::resource::read_binary_content; +use quarto_hub_provider::{ + AuthzPolicy, ExecMessage, JoinConfig, Provider, StaticTokenSource, join, +}; +use quarto_trace::EngineCapture; +use samod::{DocumentId, Repo}; +use tokio::net::TcpListener; + +/// A stand-in engine so the test doesn't need a real knitr/jupyter runtime. It +/// passes the input markdown through with a marker appended, which is enough +/// for `EngineExecutionStage` to emit an `EngineCapture`. +struct PassthroughEngine; + +impl ExecutionEngine for PassthroughEngine { + fn name(&self) -> &str { + "test-passthrough" + } + fn execute( + &self, + input: &str, + _ctx: &ExecutionContext, + ) -> Result { + let mut out = String::from(input); + out.push_str("\n\n"); + Ok(ExecuteResult::passthrough(&out)) + } +} + +fn passthrough_registry() -> EngineRegistry { + let mut registry = EngineRegistry::new(); + registry.register(Arc::new(PassthroughEngine)); + registry +} + +const PASSTHROUGH_QMD: &str = + "---\nengine: test-passthrough\n---\n\n```{test-passthrough}\n1 + 1\n```\n"; + +async fn create_text_doc(repo: &Repo, text: &str) -> String { + let mut doc = Automerge::new(); + doc.transact::<_, _, automerge::AutomergeError>(|tx| { + let obj = tx.put_object(ROOT, "text", ObjType::Text)?; + tx.update_text(&obj, text)?; + Ok(()) + }) + .unwrap(); + repo.create(doc).await.unwrap().document_id().to_string() +} + +/// Spin up a bare samod acceptor behind a tungstenite ws server and return the +/// server repo + its ws URL. +async fn spawn_server() -> (Repo, url::Url) { + let server_repo = Repo::build_tokio().load().await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let ws_url: url::Url = format!("ws://{addr}").parse().unwrap(); + let acceptor = server_repo.make_acceptor(ws_url.clone()).expect("acceptor"); + + tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + let acceptor = acceptor.clone(); + tokio::spawn(async move { + if let Ok(ws) = tokio_tungstenite::accept_async(stream).await { + let _ = acceptor.accept_tungstenite(ws); + } + }); + } + }); + + (server_repo, ws_url) +} + +/// Read + gunzip + deserialize a capture binary doc by id from `repo`. +async fn fetch_captures(repo: &Repo, doc_id: &str) -> Vec { + let id = DocumentId::from_str(doc_id).unwrap(); + let handle = repo.find(id).await.unwrap().expect("capture doc synced"); + let gz = handle + .with_document(|doc| read_binary_content(doc)) + .expect("capture doc has content"); + let mut json = Vec::new(); + GzDecoder::new(gz.as_slice()) + .read_to_end(&mut json) + .unwrap(); + serde_json::from_slice(&json).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn provider_executes_an_allowed_request_and_writes_a_capture() { + let (server_repo, ws_url) = spawn_server().await; + + let file_id = create_text_doc(&server_repo, PASSTHROUGH_QMD).await; + let (server_index, index_id) = IndexDocument::create(&server_repo).await.unwrap(); + server_index.add_file("doc.qmd", &file_id).unwrap(); + assert!(server_index.get_capture("doc.qmd").is_none()); + + let (provider_repo, provider_index) = join( + JoinConfig { + server_ws_url: ws_url, + index_doc_id: index_id.clone(), + connect_timeout: Duration::from_secs(10), + }, + Arc::new(StaticTokenSource::new("test-token")), + ) + .await + .expect("provider joins"); + + let provider = Provider::new( + provider_repo, + provider_index, + "provider-actor", + AuthzPolicy::AllowAll, + Some(passthrough_registry()), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let run_handle = tokio::spawn({ + let provider = Arc::clone(&provider); + async move { + provider + .run(async move { + let _ = shutdown_rx.await; + }) + .await; + } + }); + + // Ephemeral messages are best-effort: re-broadcast the request until the + // capture lands (or the deadline trips), the way a real "Run" button retry + // would. The provider's in-flight guard collapses duplicate broadcasts. + let request = ExecMessage::Request { + path: "doc.qmd".into(), + request_id: "req-1".into(), + requester_actor_id: "editor-actor".into(), + }; + let deadline = Instant::now() + Duration::from_secs(20); + let capture = loop { + if let Some(cap) = server_index.get_capture("doc.qmd") + && cap.state == Some(CaptureState::Idle) + { + break cap; + } + assert!(Instant::now() < deadline, "capture never landed"); + server_index.handle().broadcast(request.to_cbor()); + tokio::time::sleep(Duration::from_millis(250)).await; + }; + + // The sidecar the editor observes: idle, fresh, pointing at a real doc. + assert_eq!(capture.state, Some(CaptureState::Idle)); + assert_eq!(capture.staleness, Some(false)); + assert_eq!(capture.last_error, None); + assert!(!capture.capture_doc_id.is_empty()); + + // The capture binary doc synced back and holds the passthrough engine's + // recorded output. + let captures = fetch_captures(&server_repo, &capture.capture_doc_id).await; + assert_eq!(captures.len(), 1); + assert_eq!(captures[0].engine_name, "test-passthrough"); + + let _ = shutdown_tx.send(()); + let _ = run_handle.await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn deny_policy_writes_no_capture() { + let (server_repo, ws_url) = spawn_server().await; + + let file_id = create_text_doc(&server_repo, PASSTHROUGH_QMD).await; + let (server_index, index_id) = IndexDocument::create(&server_repo).await.unwrap(); + server_index.add_file("doc.qmd", &file_id).unwrap(); + + let (provider_repo, provider_index) = join( + JoinConfig { + server_ws_url: ws_url, + index_doc_id: index_id.clone(), + connect_timeout: Duration::from_secs(10), + }, + Arc::new(StaticTokenSource::new("test-token")), + ) + .await + .expect("provider joins"); + + let provider = Provider::new( + provider_repo, + provider_index, + "provider-actor", + AuthzPolicy::Deny, + Some(passthrough_registry()), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let run_handle = tokio::spawn({ + let provider = Arc::clone(&provider); + async move { + provider + .run(async move { + let _ = shutdown_rx.await; + }) + .await; + } + }); + + // Broadcast the request repeatedly for a bounded window; a Deny provider + // must never write a capture. + let request = ExecMessage::Request { + path: "doc.qmd".into(), + request_id: "req-deny".into(), + requester_actor_id: "editor-actor".into(), + }; + let until = Instant::now() + Duration::from_secs(3); + while Instant::now() < until { + server_index.handle().broadcast(request.to_cbor()); + tokio::time::sleep(Duration::from_millis(250)).await; + } + assert!( + server_index.get_capture("doc.qmd").is_none(), + "Deny policy must not write a capture" + ); + + let _ = shutdown_tx.send(()); + let _ = run_handle.await; +} diff --git a/crates/quarto-hub-provider/tests/integration/join.rs b/crates/quarto-hub-provider/tests/integration/join.rs new file mode 100644 index 000000000..e5dac4652 --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/join.rs @@ -0,0 +1,75 @@ +//! End-to-end join test (bd-sfet3264, Phase 3B). +//! +//! Spins up a *bare* samod acceptor behind a tungstenite websocket server (no +//! auth — the BearerDialer's `Authorization` header is sent and harmlessly +//! ignored), seeds an index document with files, and verifies the provider's +//! [`join_and_list_files`] connects over the real `BearerDialer` transport, +//! syncs the index, and lists the files. +//! +//! This exercises the actual transport (BearerDialer -> ws -> samod acceptor -> +//! sync -> IndexDocument::load), which the unit tests can't. The +//! *authenticated* acceptance path (hub validates the Bearer) is covered by +//! `quarto-hub`'s `auth_bearer` tests on the hub side, and end-to-end by the +//! Phase 3C real-binary run. + +use std::sync::Arc; +use std::time::Duration; + +use quarto_hub::index::IndexDocument; +use quarto_hub_provider::{JoinConfig, StaticTokenSource, join_and_list_files}; +use samod::Repo; +use tokio::net::TcpListener; + +#[tokio::test] +async fn join_connects_and_lists_files_over_the_bearer_dialer() { + // ── Server side: a samod repo with an index doc behind a ws acceptor ── + let server_repo = Repo::build_tokio().load().await; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let ws_url: url::Url = format!("ws://{addr}").parse().unwrap(); + + let acceptor = server_repo + .make_acceptor(ws_url.clone()) + .expect("make acceptor"); + + // Seed the index document with two files. The file values are placeholder + // doc ids — `join_and_list_files` only reads the path keys. + let (index, index_doc_id) = IndexDocument::create(&server_repo) + .await + .expect("create index"); + index.add_file("a.qmd", "doc-a").unwrap(); + index.add_file("docs/b.qmd", "doc-b").unwrap(); + + // Accept incoming websocket connections and hand them to the acceptor. + tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + let acceptor = acceptor.clone(); + tokio::spawn(async move { + match tokio_tungstenite::accept_async(stream).await { + Ok(ws) => { + let _ = acceptor.accept_tungstenite(ws); + } + Err(e) => eprintln!("ws accept failed: {e}"), + } + }); + } + }); + + // ── Provider side: dial with the BearerDialer and list files ── + let files = join_and_list_files( + JoinConfig { + server_ws_url: ws_url, + index_doc_id, + connect_timeout: Duration::from_secs(10), + }, + Arc::new(StaticTokenSource::new("test-bearer-token")), + ) + .await + .expect("join + list files"); + + assert_eq!(files, vec!["a.qmd".to_string(), "docs/b.qmd".to_string()]); +} diff --git a/crates/quarto-hub-provider/tests/integration/main.rs b/crates/quarto-hub-provider/tests/integration/main.rs new file mode 100644 index 000000000..6b00d92ee --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/main.rs @@ -0,0 +1,7 @@ +// One integration binary per crate (see .claude/rules/integration-tests.md). +pub mod auth_bridge; +pub mod execute; +pub mod join; +pub mod materialize; +pub mod relay_sync; +pub mod sync_probe; diff --git a/crates/quarto-hub-provider/tests/integration/materialize.rs b/crates/quarto-hub-provider/tests/integration/materialize.rs new file mode 100644 index 000000000..2e721f51c --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/materialize.rs @@ -0,0 +1,104 @@ +//! VFS → temp-dir materialization test (bd-sfet3264, Phase 4a). +//! +//! Seeds a samod repo with a text file document and a binary file document, +//! wires them into an index, then materializes the project into a temp dir and +//! asserts the on-disk bytes match — including a nested path. + +use automerge::{Automerge, ObjType, ROOT, transaction::Transactable}; +use quarto_hub::index::IndexDocument; +use quarto_hub::resource::create_binary_document; +use quarto_hub_provider::materialize_project; +use samod::Repo; + +/// Create a text file document with the given contents and return its id. +async fn create_text_doc(repo: &Repo, text: &str) -> String { + let mut doc = Automerge::new(); + doc.transact::<_, _, automerge::AutomergeError>(|tx| { + let obj = tx.put_object(ROOT, "text", ObjType::Text)?; + tx.update_text(&obj, text)?; + Ok(()) + }) + .unwrap(); + let handle = repo.create(doc).await.unwrap(); + handle.document_id().to_string() +} + +#[tokio::test] +async fn materializes_text_and_binary_files_to_disk() { + let repo = Repo::build_tokio().load().await; + + // A top-level text file and a nested binary asset. + let qmd = "---\ntitle: Demo\n---\n\n```{r}\n1 + 1\n```\n"; + let text_id = create_text_doc(&repo, qmd).await; + + let png_bytes: &[u8] = &[0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x01, 0x02]; + let binary_doc = create_binary_document(png_bytes, "image/png").unwrap(); + let binary_id = repo + .create(binary_doc) + .await + .unwrap() + .document_id() + .to_string(); + + let (index, _index_id) = IndexDocument::create(&repo).await.unwrap(); + index.add_file("report.qmd", &text_id).unwrap(); + index.add_file("assets/logo.png", &binary_id).unwrap(); + + let dest = tempfile::tempdir().unwrap(); + let written = materialize_project(&repo, &index, dest.path()) + .await + .expect("materialize"); + assert_eq!(written, 2, "both files should be written"); + + let on_disk_qmd = std::fs::read_to_string(dest.path().join("report.qmd")).unwrap(); + assert_eq!(on_disk_qmd, qmd); + + let on_disk_png = std::fs::read(dest.path().join("assets/logo.png")).unwrap(); + assert_eq!(on_disk_png, png_bytes); +} + +/// A project authored in hub-client stores `files[path] = docId` as an automerge +/// `Text` object (not a scalar string). The materializer must still resolve and +/// write those files — otherwise a hub-client project materializes to nothing +/// and execution fails with "project discovery failed" (bd-bm0vaetl). +#[tokio::test] +async fn materializes_a_js_authored_index_with_text_valued_ids() { + let repo = Repo::build_tokio().load().await; + + let qmd = "---\ntitle: JS project\nengine: knitr\n---\n\n```{r}\ncat(1)\n```\n"; + let text_id = create_text_doc(&repo, qmd).await; + + // Build the index the way hub-client does: the file id is a Text object. + let mut index_doc = Automerge::new(); + index_doc + .transact::<_, _, automerge::AutomergeError>(|tx| { + let files = tx.put_object(ROOT, "files", ObjType::Map)?; + let id_text = tx.put_object(&files, "index.qmd", ObjType::Text)?; + tx.update_text(&id_text, &text_id)?; + Ok(()) + }) + .unwrap(); + let index_id = repo + .create(index_doc) + .await + .unwrap() + .document_id() + .to_string(); + let index = IndexDocument::load(&repo, &index_id) + .await + .unwrap() + .unwrap(); + + let dest = tempfile::tempdir().unwrap(); + let written = materialize_project(&repo, &index, dest.path()) + .await + .expect("materialize"); + assert_eq!( + written, 1, + "the Text-valued file id must resolve and materialize" + ); + assert_eq!( + std::fs::read_to_string(dest.path().join("index.qmd")).unwrap(), + qmd + ); +} diff --git a/crates/quarto-hub-provider/tests/integration/relay_sync.rs b/crates/quarto-hub-provider/tests/integration/relay_sync.rs new file mode 100644 index 000000000..75c47be87 --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/relay_sync.rs @@ -0,0 +1,137 @@ +//! Reproduce the real topology (bd-sfet3264): a relay server, peer A creates a +//! doc with files, peer B (like the provider) connects later and `find`s it. +//! This is different from `join.rs`, where the *server itself* creates the doc. +//! +//! Not ignored — fully local (bare samod acceptor + two client repos), no +//! network. If B ends up with 0 files, we've reproduced the provider's +//! "0 file(s)" bug in isolation. + +use std::str::FromStr; +use std::time::Duration; + +use quarto_hub::index::IndexDocument; +use samod::{BackoffConfig, DocumentId, NeverAnnounce, Repo}; +use tokio::net::TcpListener; + +async fn spawn_relay() -> (Repo, url::Url) { + spawn_relay_with(Repo::build_tokio().load().await).await +} + +async fn spawn_relay_never_announce() -> (Repo, url::Url) { + spawn_relay_with( + Repo::build_tokio() + .with_announce_policy(NeverAnnounce) + .load() + .await, + ) + .await +} + +async fn spawn_relay_with(repo: Repo) -> (Repo, url::Url) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let ws_url: url::Url = format!("ws://{addr}").parse().unwrap(); + let acceptor = repo.make_acceptor(ws_url.clone()).expect("acceptor"); + tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + let acceptor = acceptor.clone(); + tokio::spawn(async move { + if let Ok(ws) = tokio_tungstenite::accept_async(stream).await { + let _ = acceptor.accept_tungstenite(ws); + } + }); + } + }); + (repo, ws_url) +} + +async fn dial(url: &url::Url) -> Repo { + let repo = Repo::build_tokio().load().await; + let handle = repo + .dial_websocket(url.clone(), BackoffConfig::default()) + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), handle.established()) + .await + .expect("established timeout") + .expect("established failed"); + repo +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn peer_b_syncs_a_doc_created_by_peer_a_through_the_relay() { + let (_relay, url) = spawn_relay().await; + + // Peer A: create the index + files, then keep its repo alive. + let repo_a = dial(&url).await; + let (index_a, doc_id) = IndexDocument::create(&repo_a).await.unwrap(); + index_a.add_file("index.qmd", "file-doc-1").unwrap(); + index_a.add_file("about.qmd", "file-doc-2").unwrap(); + + // Give sync a moment to push A's doc to the relay. + tokio::time::sleep(Duration::from_millis(500)).await; + + // Peer B (like the provider): connect fresh and find A's doc. + let repo_b = dial(&url).await; + let _handle = repo_b + .find(DocumentId::from_str(&doc_id).unwrap()) + .await + .unwrap() + .expect("B finds the doc"); + let index_b = IndexDocument::load(&repo_b, &doc_id) + .await + .unwrap() + .unwrap(); + + // Poll for the files to arrive. + let mut last = 0; + for _ in 0..40 { + last = index_b.get_all_files().len(); + if last >= 2 { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + assert_eq!( + last, 2, + "peer B never synced peer A's files through the relay (reproduces the provider bug)" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn peer_b_syncs_through_a_never_announce_relay() { + // The real servers (quarto-hub, sync.automerge.org) do NOT proactively + // announce docs to a connecting peer. If peer B still can't sync A's doc + // here, we've pinned the provider's "0 file(s)" bug to a NeverAnnounce + // relay — B must actively request/pull, not wait to be announced to. + let (_relay, url) = spawn_relay_never_announce().await; + + let repo_a = dial(&url).await; + let (index_a, doc_id) = IndexDocument::create(&repo_a).await.unwrap(); + index_a.add_file("index.qmd", "file-doc-1").unwrap(); + index_a.add_file("about.qmd", "file-doc-2").unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + + let repo_b = dial(&url).await; + let _handle = repo_b + .find(DocumentId::from_str(&doc_id).unwrap()) + .await + .unwrap() + .expect("B finds the doc"); + let index_b = IndexDocument::load(&repo_b, &doc_id) + .await + .unwrap() + .unwrap(); + + let mut last = 0; + for _ in 0..40 { + last = index_b.get_all_files().len(); + if last >= 2 { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + assert_eq!(last, 2, "peer B never synced through a NeverAnnounce relay"); +} diff --git a/crates/quarto-hub-provider/tests/integration/sync_probe.rs b/crates/quarto-hub-provider/tests/integration/sync_probe.rs new file mode 100644 index 000000000..a96836392 --- /dev/null +++ b/crates/quarto-hub-provider/tests/integration/sync_probe.rs @@ -0,0 +1,213 @@ +//! Diagnostic probes (bd-sfet3264). Ignored by default — need network. +//! +//! probe_live_doc_sync: join an existing doc and watch its `files` map sync. +//! PROBE_SERVER=wss://sync.automerge.org PROBE_DOC_ID= [PROBE_NATIVE=1] \ +//! cargo nextest run -p quarto-hub-provider --test integration \ +//! probe_live_doc_sync --run-ignored all --no-capture +//! +//! probe_create_then_find: peer A (samod) creates + pushes a doc, peer B +//! (samod) finds it — both against PROBE_SERVER. Isolates whether samod can +//! round-trip a doc through the real server at all (vs. only failing on +//! hub-client-created docs). +//! PROBE_SERVER=wss://sync.automerge.org \ +//! cargo nextest run -p quarto-hub-provider --test integration \ +//! probe_create_then_find --run-ignored all --no-capture + +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; + +use quarto_hub::index::IndexDocument; +use quarto_hub_provider::{BearerDialer, StaticTokenSource}; +use samod::{BackoffConfig, DocumentId, Repo}; + +async fn dial_bearer(url: &url::Url) -> Repo { + let repo = Repo::build_tokio().load().await; + let dialer = Arc::new(BearerDialer::new( + url.clone(), + Arc::new(StaticTokenSource::new("dev")), + )); + let handle = repo.dial(BackoffConfig::default(), dialer).unwrap(); + tokio::time::timeout(Duration::from_secs(30), handle.established()) + .await + .expect("established timeout") + .expect("established failed"); + repo +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs network + a live PROBE_DOC_ID"] +async fn probe_live_doc_sync() { + let server = std::env::var("PROBE_SERVER").expect("set PROBE_SERVER"); + let doc_id = std::env::var("PROBE_DOC_ID").expect("set PROBE_DOC_ID"); + let native = std::env::var("PROBE_NATIVE").is_ok(); + let url: url::Url = server.parse().unwrap(); + + let repo = if native { + eprintln!("dialing with samod native dial_websocket"); + let repo = Repo::build_tokio().load().await; + let handle = repo.dial_websocket(url, BackoffConfig::default()).unwrap(); + tokio::time::timeout(Duration::from_secs(30), handle.established()) + .await + .expect("established timeout") + .expect("established failed"); + repo + } else { + eprintln!("dialing with BearerDialer"); + dial_bearer(&url).await + }; + eprintln!("connection established"); + + let id = DocumentId::from_str(&doc_id).unwrap(); + match repo.find(id).await.unwrap() { + Some(_) => eprintln!("find returned Some(handle)"), + None => { + eprintln!("find returned None (doc not found)"); + return; + } + } + let index = IndexDocument::load(&repo, &doc_id).await.unwrap().unwrap(); + for i in 0..40 { + let files = index.get_all_files(); + eprintln!( + "[t={:>5}ms] {} file(s): {:?}", + i * 500, + files.len(), + files.keys().collect::>() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +/// Schema-agnostic reader: dump the doc's ROOT keys over time. Works for any +/// document (e.g. a bare `{ value: 42 }` from the interop repro), so "0 keys" +/// unambiguously means "content never synced" rather than "wrong field". +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs network + a live PROBE_DOC_ID"] +async fn probe_root_keys() { + use automerge::ReadDoc; + + let server = std::env::var("PROBE_SERVER").expect("set PROBE_SERVER"); + let doc_id = std::env::var("PROBE_DOC_ID").expect("set PROBE_DOC_ID"); + let url: url::Url = server.parse().unwrap(); + let repo = dial_bearer(&url).await; + + let id = DocumentId::from_str(&doc_id).unwrap(); + let handle = repo.find(id).await.unwrap().expect("find Some"); + for i in 0..30 { + let dump = handle.with_document(|doc| { + let mut out = String::new(); + for key in doc.keys(automerge::ROOT).collect::>() { + match doc.get(automerge::ROOT, &key) { + Ok(Some((automerge::Value::Object(automerge::ObjType::Map), obj))) => { + let entries: Vec = doc + .keys(&obj) + .collect::>() + .into_iter() + .map(|k| { + let v = doc + .get(&obj, &k) + .ok() + .flatten() + .map(|(v, _)| format!("{v:?}")); + format!("{k}={v:?}") + }) + .collect(); + out.push_str(&format!(" {key}=map{{{}}}", entries.join(", "))); + } + Ok(Some((v, _))) => out.push_str(&format!(" {key}={v:?}")), + _ => out.push_str(&format!(" {key}=?")), + } + } + out + }); + eprintln!("[t={:>5}ms]{dump}", i * 500); + if !dump.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +/// Validate the fix: read the `files` map treating each value as EITHER a +/// scalar `Str` OR a `Text` object (which is how automerge 3.x / hub-client +/// stores string map-values). If this recovers the doc ids, the fix for +/// `IndexDocument::get_all_files` is "also read Text values". +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs network + a live PROBE_DOC_ID"] +async fn probe_files_str_or_text() { + use automerge::{ReadDoc, Value}; + + let server = std::env::var("PROBE_SERVER").expect("set PROBE_SERVER"); + let doc_id = std::env::var("PROBE_DOC_ID").expect("set PROBE_DOC_ID"); + let url: url::Url = server.parse().unwrap(); + let repo = dial_bearer(&url).await; + + let id = DocumentId::from_str(&doc_id).unwrap(); + let handle = repo.find(id).await.unwrap().expect("find Some"); + tokio::time::sleep(Duration::from_millis(500)).await; + + let files: Vec<(String, String)> = handle.with_document(|doc| { + let mut out = Vec::new(); + if let Some((_, files_obj)) = doc.get(automerge::ROOT, "files").ok().flatten() { + for k in doc.keys(&files_obj).collect::>() { + if let Some((v, vid)) = doc.get(&files_obj, &k).ok().flatten() { + let s = match v { + Value::Scalar(s) => s.to_str().map(str::to_string), + Value::Object(automerge::ObjType::Text) => doc.text(&vid).ok(), + _ => None, + }; + if let Some(s) = s { + out.push((k, s)); + } + } + } + } + out + }); + eprintln!("RECOVERED {} file(s): {:?}", files.len(), files); + assert!( + !files.is_empty(), + "fix should recover the JS-authored file ids" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "needs network (PROBE_SERVER)"] +async fn probe_create_then_find() { + let server = std::env::var("PROBE_SERVER").expect("set PROBE_SERVER"); + let url: url::Url = server.parse().unwrap(); + + // Peer A creates + pushes a doc, then stays connected. + let repo_a = dial_bearer(&url).await; + let (index_a, doc_id) = IndexDocument::create(&repo_a).await.unwrap(); + index_a.add_file("index.qmd", "file-doc-1").unwrap(); + index_a.add_file("about.qmd", "file-doc-2").unwrap(); + eprintln!("peer A created doc {doc_id} with 2 files; waiting to push…"); + tokio::time::sleep(Duration::from_secs(3)).await; + + // Peer B finds it. + let repo_b = dial_bearer(&url).await; + match repo_b + .find(DocumentId::from_str(&doc_id).unwrap()) + .await + .unwrap() + { + Some(_) => eprintln!("B: find returned Some(handle)"), + None => eprintln!("B: find returned None"), + } + let index_b = IndexDocument::load(&repo_b, &doc_id) + .await + .unwrap() + .unwrap(); + let mut last = 0; + for i in 0..30 { + last = index_b.get_all_files().len(); + eprintln!("[B t={:>5}ms] {} file(s)", i * 500, last); + if last >= 2 { + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + eprintln!("RESULT: peer B saw {last} of A's 2 files via {server}"); +} diff --git a/crates/quarto-hub/src/index.rs b/crates/quarto-hub/src/index.rs index 8b165da20..1adbb1506 100644 --- a/crates/quarto-hub/src/index.rs +++ b/crates/quarto-hub/src/index.rs @@ -153,10 +153,8 @@ impl IndexDocument { // Iterate over all keys in the map for key in keys { - if let Some((value, _)) = doc.get(&files_obj, &key).ok().flatten() - && let Some(doc_id) = value.to_str() - { - files.insert(key, doc_id.to_string()); + if let Some(doc_id) = read_str_or_text(doc, &files_obj, &key) { + files.insert(key, doc_id); } } } @@ -216,8 +214,7 @@ impl IndexDocument { pub fn get_file(&self, path: &str) -> Option { self.handle.with_document(|doc| { let (_, files_obj) = doc.get(ROOT, FILES_KEY).ok().flatten()?; - let (value, _) = doc.get(files_obj, path).ok().flatten()?; - value.to_str().map(|s| s.to_string()) + read_str_or_text(doc, &files_obj, path) }) } @@ -320,6 +317,28 @@ impl IndexDocument { } } +/// Read a string-valued map entry that may be stored as EITHER a scalar `Str` +/// OR an automerge `Text` object. +/// +/// hub-client (`@automerge/automerge` 3.x, via `automerge-repo`) stores plain +/// string map-values — including the file-document ids in the `files` map — as +/// `Text` objects, not scalar strings (a `doc.files[path] = id` assignment +/// becomes a collaborative `Text`). A Rust reader using `Value::to_str()` alone +/// silently drops the `Text` form, so a project *created in hub-client* reads as +/// having **zero files** — which broke `q2 provide-hub` materialization +/// (bd-bm0vaetl). Rust-authored docs (project-mode hub) keep using the scalar +/// form; this accepts both. Same class of bug as the `metadata-as-str` lint. +fn read_str_or_text(doc: &D, obj: &automerge::ObjId, key: &str) -> Option { + let (value, id) = doc.get(obj, key).ok().flatten()?; + if let Some(s) = value.to_str() { + return Some(s.to_string()); + } + if matches!(value, automerge::Value::Object(ObjType::Text)) { + return doc.text(&id).ok(); + } + None +} + /// Read a single CaptureRef out of an automerge entry map. /// Returns None when the required `captureDocId` field is absent /// (treated as a corrupt entry — caller may ignore or log). @@ -435,6 +454,44 @@ mod tests { ); } + #[tokio::test] + async fn get_all_files_reads_text_valued_ids() { + // hub-client (@automerge/automerge 3.x) stores files[path] = docId as a + // Text object, not a scalar string. get_all_files/get_file must read + // both forms, or a project created in hub-client reads as zero files + // and `q2 provide-hub` materializes nothing (bd-bm0vaetl). + let repo = create_test_repo().await; + + let mut doc = Automerge::new(); + doc.transact::<_, _, automerge::AutomergeError>(|tx| { + let files = tx.put_object(ROOT, FILES_KEY, ObjType::Map)?; + // JS style: the id is a Text object. + let t = tx.put_object(&files, "index.qmd", ObjType::Text)?; + tx.update_text(&t, "doc-id-from-js")?; + // Rust style: the id is a scalar string. + tx.put(&files, "about.qmd", "doc-id-from-rust")?; + Ok(()) + }) + .unwrap(); + let handle = repo.create(doc).await.unwrap(); + let doc_id = handle.document_id().to_string(); + let index = IndexDocument::load(&repo, &doc_id).await.unwrap().unwrap(); + + let files = index.get_all_files(); + assert_eq!(files.len(), 2, "both the Text and scalar ids must be read"); + assert_eq!(files.get("index.qmd"), Some(&"doc-id-from-js".to_string())); + assert_eq!( + files.get("about.qmd"), + Some(&"doc-id-from-rust".to_string()) + ); + + assert_eq!( + index.get_file("index.qmd"), + Some("doc-id-from-js".to_string()) + ); + assert!(index.has_file("index.qmd")); + } + #[tokio::test] async fn test_remove_file() { let repo = create_test_repo().await; diff --git a/crates/quarto-mcp-launcher/src/lib.rs b/crates/quarto-mcp-launcher/src/lib.rs index 5a0be25f2..1e629e91d 100644 --- a/crates/quarto-mcp-launcher/src/lib.rs +++ b/crates/quarto-mcp-launcher/src/lib.rs @@ -22,8 +22,10 @@ mod defaults; mod delegate; mod node; +pub use bundle::{BundleFile, content_hash, embedded_files, is_placeholder}; pub use cache::{ - DEFAULT_MAX_AGE, ExtractedBundle, LAST_USED_FILE, LOCK_FILE, extract_and_lock, gc, + DEFAULT_MAX_AGE, ExtractedBundle, LAST_USED_FILE, LOCK_FILE, default_cache_root, + extract_and_lock, gc, }; pub use defaults::{BundledDefault, Source, bundled_defaults, classify, injections, sources}; pub use node::{Discovery, MIN_NODE_MAJOR, NodeError, NodeInfo, find_node, parse_version}; diff --git a/crates/quarto/Cargo.toml b/crates/quarto/Cargo.toml index fed706f29..da3664408 100644 --- a/crates/quarto/Cargo.toml +++ b/crates/quarto/Cargo.toml @@ -20,7 +20,8 @@ tracing-subscriber.workspace = true serde.workspace = true serde_json.workspace = true pollster.workspace = true -tokio.workspace = true +# `signal` for the `provide-hub` serve loop's Ctrl-C shutdown. +tokio = { workspace = true, features = ["signal"] } quarto-core = { workspace = true, features = ["clap"] } quarto-error-catalog.workspace = true @@ -33,8 +34,10 @@ quarto-system-runtime.workspace = true quarto-doctemplate.workspace = true quarto-lsp = { workspace = true } quarto-hub.workspace = true +quarto-hub-provider = { path = "../quarto-hub-provider" } quarto-mcp-launcher = { path = "../quarto-mcp-launcher" } quarto-preview = { path = "../quarto-preview" } +url = "2.5" quarto-publish.workspace = true quarto-sass.workspace = true quarto-test.workspace = true diff --git a/crates/quarto/src/commands/mod.rs b/crates/quarto/src/commands/mod.rs index f1455588f..a2fb05793 100644 --- a/crates/quarto/src/commands/mod.rs +++ b/crates/quarto/src/commands/mod.rs @@ -16,6 +16,7 @@ pub mod lsp; pub mod mcp; pub mod pandoc; pub mod preview; +pub mod provide_hub; pub mod publish; pub mod remove; pub mod render; diff --git a/crates/quarto/src/commands/provide_hub.rs b/crates/quarto/src/commands/provide_hub.rs new file mode 100644 index 000000000..2dd8d9e0f --- /dev/null +++ b/crates/quarto/src/commands/provide_hub.rs @@ -0,0 +1,161 @@ +//! `provide-hub` — connect to a hub session as a code-execution provider. +//! +//! Joins an existing hub project's automerge session (authenticating via the +//! Node auth bridge), lists the files, and — with `--allow-all` (Phase 4a) — +//! serves execution requests: it materializes the project, runs the engines +//! natively, and writes the results back as capture docs every collaborator's +//! editor consumes. +//! +//! Execution is **fail-closed** by default: without `--allow-all` the command +//! connects, lists files, and exits. The provider-only default (gate on the +//! provider's own actor id) lands in Phase 5. +//! +//! See `claude-notes/plans/2026-06-29-remote-execution-provider.md` (bd-sfet3264). + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use quarto_hub_provider::{ + AuthzPolicy, JoinConfig, NodeBridge, Provider, StaticTokenSource, TokenSource, join, +}; + +/// Arguments for `q2 provide-hub`. +pub struct ProvideHubArgs { + /// A quarto-hub share URL (`https://quarto-hub.com/#/share/?…`) or a + /// bare index-document id of the project to join. + pub project: String, + /// Hub websocket URL. Defaults to `$QUARTO_HUB_SERVER`, else the canonical + /// hub. + pub server: Option, + /// Serve execution requests from any collaborator. Without this the command + /// is fail-closed (connect + list + exit). + pub allow_all: bool, + /// Dev/testing escape hatch: use this bearer token verbatim instead of + /// running the interactive OAuth bridge. Intended for a **local, no-auth + /// hub** (`q2 hub`), which ignores the bearer entirely. Never needed + /// against quarto-hub.com. + pub token: Option, +} + +const DEFAULT_SERVER_WS: &str = "wss://quarto-hub.com/ws"; + +pub fn execute(args: ProvideHubArgs) -> Result<()> { + // A full multi-threaded runtime: the auth bridge's stdout reader and the + // samod sync both run as background tasks. + let runtime = tokio::runtime::Runtime::new()?; + runtime.block_on(run(args)) +} + +/// Extract the index-document id from a quarto-hub share URL, or return the +/// input unchanged when it is already a bare id. +fn parse_index_doc_id(project: &str) -> String { + if let Some(rest) = project.split("#/share/").nth(1) { + // The id is up to the first query/separator character. + let id = rest.split(['?', '&', '/']).next().unwrap_or(rest); + return id.to_string(); + } + project.to_string() +} + +/// Resolve the hub websocket URL: explicit `--server`, else `$QUARTO_HUB_SERVER`, +/// else the canonical hub. +fn resolve_server_ws(arg: Option) -> String { + arg.or_else(|| std::env::var("QUARTO_HUB_SERVER").ok()) + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_SERVER_WS.to_string()) +} + +async fn run(args: ProvideHubArgs) -> Result<()> { + let index_doc_id = parse_index_doc_id(&args.project); + let server_ws = resolve_server_ws(args.server); + let server_ws_url = url::Url::parse(&server_ws) + .with_context(|| format!("invalid hub server URL: {server_ws}"))?; + + // A dev `--token` bypasses the OAuth bridge with a static bearer — for a + // local no-auth hub, which ignores it. Otherwise spawn the Node auth + // bridge and sign in interactively. + let token_source: Arc = if let Some(token) = args.token { + eprintln!("Using a static bearer token (dev mode; the hub must not require auth)."); + Arc::new(StaticTokenSource::new(token)) + } else { + eprintln!("Authenticating with the hub…"); + Arc::new(NodeBridge::spawn().context("starting the auth bridge")?) + }; + + eprintln!("Connecting to project {index_doc_id} at {server_ws_url}…"); + let (repo, index) = join( + JoinConfig { + server_ws_url, + index_doc_id, + connect_timeout: Duration::from_secs(30), + }, + token_source, + ) + .await + .context("joining the hub session")?; + + let mut files: Vec = index.get_all_files().into_keys().collect(); + files.sort(); + println!("Connected. {} file(s) in the project:", files.len()); + for file in &files { + println!(" {file}"); + } + + if !args.allow_all { + eprintln!(); + eprintln!("Execution is DISABLED (fail-closed default)."); + eprintln!("Serving requests would run this project's code on THIS machine."); + eprintln!("Re-run with --allow-all to let collaborators execute this project's"); + eprintln!("code here. (A safer provider-only default is coming in a later release.)"); + return Ok(()); + } + + // The beacon's actorId in Phase 4a is the samod peer id (stable for this + // process); Phase 5 swaps in the per-project actor id from /auth/actor. + let self_actor_id = repo.peer_id().to_string(); + let provider = Provider::new(repo, index, self_actor_id, AuthzPolicy::AllowAll, None); + + eprintln!(); + eprintln!("Execution ENABLED for all collaborators (--allow-all)."); + eprintln!("This project's code will run on THIS machine on request. Press Ctrl-C to stop."); + provider + .run(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await; + eprintln!("Provider stopped."); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_share_url() { + assert_eq!( + parse_index_doc_id("https://quarto-hub.com/#/share/abc123?file=x.qmd&name=y"), + "abc123" + ); + assert_eq!( + parse_index_doc_id("https://quarto-hub.com/#/share/abc123"), + "abc123" + ); + } + + #[test] + fn passes_a_bare_id_through() { + assert_eq!(parse_index_doc_id("abc123"), "abc123"); + } + + #[test] + fn server_resolution_prefers_the_explicit_arg() { + assert_eq!( + resolve_server_ws(Some("wss://example.test/ws".into())), + "wss://example.test/ws" + ); + // Blank explicit arg falls through to the default. + assert_eq!(resolve_server_ws(Some(" ".into())), DEFAULT_SERVER_WS); + } +} diff --git a/crates/quarto/src/main.rs b/crates/quarto/src/main.rs index a08f9f912..49f47b089 100644 --- a/crates/quarto/src/main.rs +++ b/crates/quarto/src/main.rs @@ -459,6 +459,37 @@ enum Commands { args: Vec, }, + /// Connect to a hub session as a code-execution provider. + /// + /// Authenticates with the hub (opening a browser the first time) and + /// joins the project's collaborative session, offering this machine to + /// run the project's code on request. + /// + /// Execution is DISABLED by default (fail-closed): the command connects, + /// lists the files, and exits. Pass --allow-all to serve execution + /// requests — this project's code then runs on THIS machine on request + /// from any collaborator. + ProvideHub { + /// A quarto-hub share URL or a bare project index-document id. + project: String, + + /// Hub websocket URL (defaults to $QUARTO_HUB_SERVER, else the + /// canonical hub). + #[arg(long, env = "QUARTO_HUB_SERVER")] + server: Option, + + /// Serve execution requests from any collaborator with access to the + /// document. Their code runs on THIS machine. Without this flag the + /// command is fail-closed (connect + list + exit). + #[arg(long = "allow-all")] + allow_all: bool, + + /// Dev/testing: use this bearer token instead of the interactive OAuth + /// bridge. Only for a local, no-auth hub (`q2 hub`), which ignores it. + #[arg(long, env = "QUARTO_HUB_TOKEN")] + token: Option, + }, + /// Start collaborative hub server for real-time editing. /// By default, watches the current directory (or --project path). /// Use --no-project to run as a standalone sync server. @@ -728,6 +759,18 @@ fn main() -> Result<()> { }), Commands::Mcp { args } => commands::mcp::run(&args), + Commands::ProvideHub { + project, + server, + allow_all, + token, + } => commands::provide_hub::execute(commands::provide_hub::ProvideHubArgs { + project, + server, + allow_all, + token, + }), + Commands::Hub { project, no_project, diff --git a/crates/wasm-quarto-hub-client/src/lib.rs b/crates/wasm-quarto-hub-client/src/lib.rs index 4a76ca22c..5ba2a487e 100644 --- a/crates/wasm-quarto-hub-client/src/lib.rs +++ b/crates/wasm-quarto-hub-client/src/lib.rs @@ -1050,12 +1050,12 @@ pub async fn render_qmd_content( /// /// Phase 9 entry point used by the hub-client live preview. /// Equivalent to -/// [`render_page_in_project_with_attribution(path, user_grammars, None)`](render_page_in_project_with_attribution). +/// [`render_page_in_project_with_attribution(path, user_grammars, None, None)`](render_page_in_project_with_attribution). /// Kept as a separate entry point for callers that have no -/// attribution payload to ship and want the simpler signature. +/// attribution payload or capture to ship and want the simpler signature. #[wasm_bindgen] pub async fn render_page_in_project(path: &str, user_grammars: Option) -> String { - render_page_in_project_with_attribution(path, user_grammars, None).await + render_page_in_project_with_attribution(path, user_grammars, None, None).await } /// Render a single page **in the context of its surrounding project**, @@ -1093,8 +1093,9 @@ pub async fn render_page_in_project(path: &str, user_grammars: Option, attribution_json: Option, + // bd-sfet3264 (Phase 1A): optional recorded engine capture sequence, + // gzipped JSON of `EngineCapture[]` — the same wire format the capture + // binary doc holds and that `render_page_for_preview` consumes. hub-client + // threads it from the IndexDocument's capture sidecar so executed engine + // output is spliced in *without* losing attribution. `None` is + // byte-identical to the pre-feature behaviour for every existing caller. + capture_gz_json: Option>, ) -> String { let runtime = get_runtime(); let path_buf = std::path::PathBuf::from(path); @@ -1128,6 +1141,17 @@ pub async fn render_page_in_project_with_attribution( } }; + // Deserialize the gzipped JSON capture sequence (empty when absent). + // Both render branches thread it into the q2-preview pipeline's + // `CaptureSpliceStage` alongside attribution; the non-preview branch + // ignores it. Same parsing as `render_page_for_preview`. + let captures = match parse_capture_from(capture_gz_json) { + Ok(caps) => caps, + Err(e) => { + return error_response(format!("Failed to parse capture: {}", e)); + } + }; + // Discover from the active path first to find any // `_quarto.yml` ancestor and learn whether this is a // single-file or multi-file project. @@ -1148,7 +1172,7 @@ pub async fn render_page_in_project_with_attribution( &project, user_grammars, false, - Vec::new(), + captures, attribution_json, ) .await; @@ -1172,7 +1196,7 @@ pub async fn render_page_in_project_with_attribution( project, user_grammars, false, - Vec::new(), + captures, attribution_json, ) .await @@ -1412,7 +1436,10 @@ async fn render_single_doc_to_response( } } _ => { - let config = HtmlRenderConfig::with_resolver(resolver.clone()); + // bd-uy4uygha: splice server-recorded captures into the HTML render + // (hub-client's default `format: html` preview), the same way the + // `preview` branch above does for the AST path. + let config = HtmlRenderConfig::with_resolver(resolver.clone()).with_captures(captures); match render_qmd_to_html(content, &source_name, &mut ctx, &config, runtime_arc).await { Ok(out) => ( Some(out.html), @@ -1594,6 +1621,9 @@ async fn render_project_active_page_to_response( if let Some(ref provider) = user_grammars_rc { renderer = renderer.with_user_grammars(provider.clone()); } + // bd-uy4uygha: splice the active page's captures into its HTML the + // same way the `preview` branch does for the AST path. + renderer = renderer.with_captures(captures); let mut pipeline = ProjectPipeline::with_renderer( &mut project, project_type, diff --git a/hub-client/changelog.md b/hub-client/changelog.md index 7e56c43af..1d2182f19 100644 --- a/hub-client/changelog.md +++ b/hub-client/changelog.md @@ -15,6 +15,17 @@ be in reverse chronological order (latest first). --> +### 2026-07-01 + +- [`0b13dbcb`](https://github.com/quarto-dev/q2/commits/0b13dbcb): The preview's code-execution controls are now a single status line — executor status, "showing executed output", and the Run/Re-run and Clear-results buttons share one bar instead of stacking as two. +- [`465f41d2`](https://github.com/quarto-dev/q2/commits/465f41d2): The preview now shows executed code output for **regular documents** (the default `format: html`, including website pages), not only documents with `format: q2-preview` — so after running a document via a connected `q2` executor you see the results in the normal preview without changing the format. +- [`76a01167`](https://github.com/quarto-dev/q2/commits/76a01167): When a `q2` client is online to execute the project's code, documents with executable cells now show a **Run** button in the preview — click it to run the code on the connected machine and see the executed output; the button reflects progress ("Executing…"), errors, and when the code has changed since the last run. + +### 2026-06-30 + +- [`7b823876`](https://github.com/quarto-dev/q2/commits/7b823876): The editor now detects when a connected `q2` client is available to execute the project's code and shows an "Executor online" indicator (groundwork for running code from the shared editor; running it is not wired up yet). +- [`42fa84de`](https://github.com/quarto-dev/q2/commits/42fa84de): The live preview now shows recorded code-execution output (when a project has it) instead of raw `{r}`/`{python}` source, and a "Clear results" control lets you remove that output for the document (and all collaborators) when you want a clean source view. + ### 2026-06-25 - [`d6066dc9`](https://github.com/quarto-dev/q2/commits/d6066dc9): The editing toolbar (and breadcrumb navigator) no longer gets cut off when you edit the very first block of a document with no title — it now flips below the block when there isn't room above. diff --git a/hub-client/src/App.tsx b/hub-client/src/App.tsx index 554ea94e4..98cd3f7f7 100644 --- a/hub-client/src/App.tsx +++ b/hub-client/src/App.tsx @@ -26,6 +26,7 @@ import { applyEditorOperations, createNewProject, type ActorIdentity, + type CaptureRef, type EditorContentChange, } from '@quarto/preview-runtime'; import type { ProjectFile } from '@quarto/preview-runtime'; @@ -36,6 +37,7 @@ import { useRouting } from './hooks/useRouting'; import { useProjectSet } from './hooks/useProjectSet'; import { useAuth } from './hooks/useAuth'; import { useAuthProbe } from './hooks/useAuthProbe'; +import { useExecutionChannel } from './hooks/useExecutionChannel'; import { resolveActorId as resolveActorIdRequest } from './services/authService'; import type { Route, ShareRoute, LinkProjectSetRoute } from './utils/routing'; import './App.css'; @@ -100,8 +102,21 @@ function App() { const [screenName, setScreenName] = useState(); const [cursorColor, setCursorColor] = useState(); const [identities, setIdentities] = useState>({}); + // bd-sfet3264 (Phase 1C): IndexDocument V2 capture sidecar (path → CaptureRef). + // Populated by the sync client's onCapturesChange; threaded down to the + // preview so recorded engine output can be spliced into the rendered AST. + const [captures, setCaptures] = useState>({}); const [isOnline, setIsOnline] = useState(false); + // bd-sfet3264 (Phase 2D + Phase 4b): track which q2 executors are online for + // the connected project (via the index-handle capability beacon) and expose + // a way to ask one to run a document. Beacons come from a connected + // `q2 provide-hub` (Phase 4). + const { executors: liveExecutors, requestExecution } = useExecutionChannel( + isOnline, + project?.indexDocId ?? null, + ); + // While a project's sync is disconnected, check whether the disconnect is // actually an auth rejection (browsers hide the WS upgrade status). Only // definitive 401/403 evidence ever clears auth — never network errors. @@ -440,6 +455,9 @@ function App() { onIdentitiesChange: (newIdentities) => { setIdentities(newIdentities); }, + onCapturesChange: (newCaptures) => { + setCaptures(newCaptures); + }, onFileContent: (path, content, _patches) => { // Note: patches are ignored - we use diff-based sync in Editor.tsx setFileContents((prev) => { @@ -677,6 +695,9 @@ function App() { navigateToFile(project.id, filePath, options); }} identities={identities} + captures={captures} + executorsOnline={liveExecutors.length > 0} + onRequestExecution={requestExecution} isOnline={isOnline} /> diff --git a/hub-client/src/components/Editor.css b/hub-client/src/components/Editor.css index b09c8e286..01d4acb79 100644 --- a/hub-client/src/components/Editor.css +++ b/hub-client/src/components/Editor.css @@ -275,6 +275,73 @@ position: relative; } +/* bd-yai4w8ly: the single merged preview execution status line. Combines the + former .executor-online-bar / .run-control / .capture-results-bar. One blue + strip (a theme overhaul will revisit the palette); the green dot still + signals executor liveness. */ +.preview-status-bar { + display: flex; + align-items: center; + gap: 8px; + padding: 3px 10px; + font-size: 12px; + background: #eef6ff; + border-bottom: 1px solid #cfe2f5; + color: #234; +} + +.preview-status-label { + flex: 1; + min-width: 0; +} + +.preview-status-error { + color: #a12d2d; +} + +/* Right-aligned action group; Run is last so it stays pinned to the far + right across state transitions. */ +.preview-status-actions { + display: flex; + align-items: center; + gap: 8px; + margin-left: auto; + flex: 0 0 auto; +} + +.preview-status-bar button { + font-size: 12px; + padding: 2px 8px; + border: 1px solid #b9c7d6; + border-radius: 4px; + background: #fff; + cursor: pointer; + white-space: nowrap; +} + +.preview-status-bar button:hover:not(:disabled) { + background: #f0f4f8; +} + +.preview-status-run-btn:disabled { + opacity: 0.6; + cursor: default; +} + +.preview-status-clear-confirm-btn { + border-color: #d08a8a !important; + color: #a12d2d; +} + +/* Green liveness dot, shown only while an executor is online. */ +.executor-online-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: #2ea043; + flex: 0 0 auto; +} + .preview-pane.fullscreen { flex: 1; width: 100%; diff --git a/hub-client/src/components/Editor.tsx b/hub-client/src/components/Editor.tsx index 9fc6353dc..83aacbb70 100644 --- a/hub-client/src/components/Editor.tsx +++ b/hub-client/src/components/Editor.tsx @@ -13,7 +13,9 @@ import { exportProjectAsZip, type EditorContentChange, } from '@quarto/preview-runtime'; -import { vfsAddFile, isWasmReady } from '@quarto/preview-runtime'; +import { vfsAddFile, isWasmReady, clearCapture } from '@quarto/preview-runtime'; +import { PreviewStatusBar } from './render/PreviewStatusBar'; +import { hasExecutableCells } from '../services/executableCells'; import type { Diagnostic } from '@quarto/preview-renderer/types/diagnostic'; import { useIntelligenceProviders } from '../hooks/useIntelligenceProviders'; import { registerQmdLanguage } from './quartoTheme'; @@ -58,6 +60,18 @@ interface Props { onNavigateToFile: (filePath: string, options?: { anchor?: string; replace?: boolean }) => void; /** Actor ID -> identity mapping from the IndexDocument */ identities?: Record; + /** Path -> recorded engine capture sidecar entry (bd-sfet3264). */ + captures?: Record; + /** + * Whether at least one q2 executor is currently online for this project + * (bd-sfet3264 Phase 2). Gates the Run affordance (Phase 4b). + */ + executorsOnline?: boolean; + /** + * Broadcast an "execute this document now" request to a connected executor + * (bd-sfet3264 Phase 4b). Returns the request id, or null when not connected. + */ + onRequestExecution?: (path: string) => string | null; /** Whether the project is connected to the sync server */ isOnline: boolean; } @@ -136,7 +150,7 @@ function selectDefaultFile(files: FileEntry[]): FileEntry | null { return files[0]; } -export default function Editor({ project, files, fileContents, onDisconnect, onContentOperations, route, onNavigateToFile, identities, isOnline }: Props) { +export default function Editor({ project, files, fileContents, onDisconnect, onContentOperations, route, onNavigateToFile, identities, captures, executorsOnline, onRequestExecution, isOnline }: Props) { // View mode for pane sizing const { viewMode } = useViewMode(); const { effectiveTheme } = useTheme(); @@ -1084,6 +1098,14 @@ export default function Editor({ project, files, fileContents, onDisconnect, onC ✕ )} + { onRequestExecution?.(p); }} + onClear={(p) => clearCapture(p)} + /> diff --git a/hub-client/src/components/render/Preview.capture.integration.test.tsx b/hub-client/src/components/render/Preview.capture.integration.test.tsx new file mode 100644 index 000000000..3af8cc293 --- /dev/null +++ b/hub-client/src/components/render/Preview.capture.integration.test.tsx @@ -0,0 +1,114 @@ +/** + * Capture-consumption test for the default `format: html` Preview (bd-uy4uygha). + * + * The sibling `ReactPreview.capture.integration.test.tsx` covers the q2-preview + * (AST) renderer. hub-client's *default* renderer for a plain document (and + * every website page) is `Preview`, which renders `format: html` via + * `renderToHtml`. This test pins that `Preview` fetches the active document's + * capture (`getBinaryDocById`) and threads its bytes into the render call as + * `captureGzJson`, so a document executed by a connected `q2 provide-hub` shows + * its output. The actual splicing is covered by `captureSpliceHtml.wasm.test.ts`. + * + * @vitest-environment jsdom + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, waitFor } from '@testing-library/react'; +import React from 'react'; + +const CAPTURE_BYTES = new Uint8Array([9, 8, 7, 6]); + +const { renderToHtml, getBinaryDocById } = vi.hoisted(() => ({ + renderToHtml: vi.fn(async () => ({ + success: true, + html: '
', + diagnostics: [], + warnings: [], + pass1_failures: [], + })), + getBinaryDocById: vi.fn(async () => ({ + content: new Uint8Array([9, 8, 7, 6]), + mimeType: 'application/x-engine-capture+gzip', + })), +})); + +vi.mock('@quarto/preview-runtime', () => ({ + renderToHtml, + getBinaryDocById, + isWasmReady: () => true, + setScrollSyncEnabled: vi.fn(), + getFileContent: vi.fn(() => null), + getBinaryFileContent: vi.fn(() => null), +})); + +vi.mock('../../hooks/usePreference', () => ({ usePreference: () => [false, vi.fn()] })); +vi.mock('../../hooks/useScrollSync', () => ({ + useScrollSync: () => ({ handlePreviewScroll: vi.fn(), handlePreviewClick: vi.fn() }), +})); +vi.mock('../../hooks/useSelectionSync', () => ({ + useSelectionSync: () => ({ handlePreviewSelection: vi.fn() }), +})); +vi.mock('@quarto/preview-renderer/iframe/MorphIframe', () => ({ + default: React.forwardRef(() =>
), +})); +vi.mock('@quarto/preview-renderer/overlays/PreviewErrorOverlay', () => ({ + PreviewErrorOverlay: () => null, +})); +vi.mock('@quarto/preview-renderer/overlays/PreviewStaticInfoViews', () => ({ + ErrorView: () => null, +})); + +import Preview from './Preview'; +import type { CaptureRef } from '@quarto/preview-runtime'; + +function baseProps(captures?: Record) { + return { + content: '---\nformat: html\nengine: knitr\n---\n\n```{r}\n1 + 1\n```\n', + currentFile: { path: 'doc.qmd', name: 'doc.qmd' } as any, + files: [], + fileContents: new Map([['doc.qmd', 'x']]), + scrollSyncEnabled: false, + editorRef: { current: null } as any, + editorReady: true, + editorHasFocusRef: { current: false } as any, + onFileChange: () => {}, + onOpenNewFileDialog: () => {}, + onDiagnosticsChange: () => {}, + captures, + }; +} + +describe('Preview capture consumption (bd-uy4uygha)', () => { + beforeEach(() => { + renderToHtml.mockClear(); + getBinaryDocById.mockClear(); + }); + + it('fetches the active doc capture and threads its bytes into the html render call', async () => { + const captures: Record = { + 'doc.qmd': { captureDocId: 'cap-1', state: 'idle' }, + }; + + render(); + + await waitFor(() => expect(getBinaryDocById).toHaveBeenCalledWith('cap-1')); + + await waitFor(() => { + const withCapture = renderToHtml.mock.calls.find( + (c) => c[0]?.captureGzJson !== undefined, + ); + expect(withCapture, 'a render call must carry the capture bytes').toBeTruthy(); + expect(withCapture![0].captureGzJson).toEqual(CAPTURE_BYTES); + }); + }); + + it('passes no capture bytes when the active doc has no capture entry', async () => { + render(); + + await waitFor(() => expect(renderToHtml).toHaveBeenCalled()); + expect(getBinaryDocById).not.toHaveBeenCalled(); + for (const call of renderToHtml.mock.calls) { + expect(call[0].captureGzJson).toBeUndefined(); + } + }); +}); diff --git a/hub-client/src/components/render/Preview.tsx b/hub-client/src/components/render/Preview.tsx index 9d675d827..d202c2563 100644 --- a/hub-client/src/components/render/Preview.tsx +++ b/hub-client/src/components/render/Preview.tsx @@ -2,8 +2,9 @@ import { useState, useCallback, useRef, useEffect, useMemo } from 'react'; import type * as Monaco from 'monaco-editor'; import type { FileEntry } from '@quarto/preview-renderer/types/project'; import type { Diagnostic } from '@quarto/preview-renderer/types/diagnostic'; -import { renderToHtml, isWasmReady, setScrollSyncEnabled, type Pass1Failure } from '@quarto/preview-runtime'; +import { renderToHtml, isWasmReady, setScrollSyncEnabled, type Pass1Failure, type CaptureRef } from '@quarto/preview-runtime'; import { getFileContent, getBinaryFileContent } from '@quarto/preview-runtime'; +import { useActiveCaptureBytes } from '../../hooks/useActiveCaptureBytes'; import { useScrollSync } from '../../hooks/useScrollSync'; import { useSelectionSync } from '../../hooks/useSelectionSync'; import { PreviewErrorOverlay } from '@quarto/preview-renderer/overlays/PreviewErrorOverlay'; @@ -53,6 +54,13 @@ interface PreviewProps { onFileChange: (file: FileEntry, anchor?: string) => void; onOpenNewFileDialog: (initialFilename: string) => void; onDiagnosticsChange: (diagnostics: Diagnostic[]) => void; + /** + * Path → recorded engine capture sidecar entry (bd-uy4uygha). The active + * file's capture (if any) is fetched and spliced into the html render so a + * document executed by a connected `q2 provide-hub` shows its output in the + * default `format: html` preview. + */ + captures?: Record; /** Callback to register scrollToLine function for external use */ onRegisterScrollToLine?: (fn: (line: number) => void) => void; /** Callback to register setScrollRatio function for external use */ @@ -91,6 +99,7 @@ function pass1FailuresBannerMessage(failures: Pass1Failure[]): string { async function doRender( documentPath: string, projectFilePaths: readonly string[], + captureGzJson: Uint8Array | undefined, ): Promise { // Caller should check isWasmReady() before calling this if (!isWasmReady()) { @@ -111,6 +120,9 @@ async function doRender( getBinaryFileContent(path)?.content ?? null, getTextContent: async (path) => getFileContent(path), }, + // bd-uy4uygha: splice the active document's recorded engine output into + // the html (undefined ⇒ code cells render as source). + captureGzJson, }); // Collect all diagnostics from both success and error paths @@ -165,9 +177,15 @@ export default function Preview({ onFileChange, onOpenNewFileDialog, onDiagnosticsChange, + captures, onRegisterScrollToLine, onRegisterSetScrollRatio, }: PreviewProps) { + // bd-uy4uygha: the active document's recorded capture bytes (if any), fetched + // from the capture sidecar. Threaded into `doRender` so the html render + // splices the executed output. A freshly-arrived capture updates this and, via + // the render-callback deps below, re-renders the preview. + const captureBytes = useActiveCaptureBytes(captures, currentFile?.path); // Preview state machine for error handling const [previewState, setPreviewState] = useState('START'); const [currentError, setCurrentError] = useState(null); @@ -275,7 +293,7 @@ export default function Preview({ // any user-defined tree-sitter grammars under `_quarto/grammars/*`. // The discovery step is pure + cache-backed so this is ~free when // grammars haven't changed since the last render. - const result = await doRender(documentPath, projectFilePaths); + const result = await doRender(documentPath, projectFilePaths, captureBytes); if (qmdContent !== lastContentRef.current) return; // Update diagnostics @@ -317,7 +335,7 @@ export default function Preview({ setPreviewState('ERROR_FROM_GOOD'); } } - }, [onDiagnosticsChange, projectFilePaths]); + }, [onDiagnosticsChange, projectFilePaths, captureBytes]); // Debounced render update const updatePreview = useCallback((newContent: string, documentPath: string) => { diff --git a/hub-client/src/components/render/PreviewRouter.tsx b/hub-client/src/components/render/PreviewRouter.tsx index c05e1351f..042475936 100644 --- a/hub-client/src/components/render/PreviewRouter.tsx +++ b/hub-client/src/components/render/PreviewRouter.tsx @@ -3,7 +3,7 @@ import type * as Monaco from 'monaco-editor'; import type { FileEntry } from '@quarto/preview-renderer/types/project'; import { isQmdFile } from '@quarto/preview-renderer/types/project'; import type { Diagnostic } from '@quarto/preview-renderer/types/diagnostic'; -import type { ActorIdentity } from '@quarto/preview-runtime'; +import type { ActorIdentity, CaptureRef } from '@quarto/preview-runtime'; import { parseQmdToAst, isWasmReady, initWasm } from '@quarto/preview-runtime'; import Preview from './Preview'; import ReactPreview from './ReactPreview'; @@ -39,6 +39,12 @@ interface PreviewRouterProps { * of the `actor.slice(0, 8)` fallback hash. */ identities?: Record; + /** + * Path → recorded engine capture sidecar entry (bd-sfet3264). + * Threaded into ReactPreview so the active document's capture can be + * fetched and spliced into the rendered AST. + */ + captures?: Record; /** * Attribution overlay on/off. Session-only — owned by `Editor.tsx` * as `useState`, threaded down here and into `ReactPreview` to @@ -139,7 +145,7 @@ export default function PreviewRouter(props: PreviewRouterProps) { // Render the appropriate preview component with shared WASM error banner. // `identities` and `attributionOn` are for ReactPreview only — Preview // doesn't know about either. - const { onRegisterScrollToLine, onRegisterSetScrollRatio, onRegisterReplayScroll, onFormatChange, onContentRewrite, fileContents, identities, attributionOn, onAttributionGeneratingChange, ...commonProps } = props; + const { onRegisterScrollToLine, onRegisterSetScrollRatio, onRegisterReplayScroll, onFormatChange, onContentRewrite, fileContents, identities, captures, attributionOn, onAttributionGeneratingChange, ...commonProps } = props; return (
@@ -149,12 +155,15 @@ export default function PreviewRouter(props: PreviewRouterProps) { )}
{reactFormat ? ( - + ) : ( // Phase 9 Decision 6: pass `fileContents` so any sibling // edit (including `_quarto.yml`) triggers a re-render via // the Map identity changing on every Automerge update. - + // bd-uy4uygha: `captures` threads the capture sidecar so the default + // `format: html` preview can splice executed output (previously only + // ReactPreview / q2-preview consumed captures). + )}
diff --git a/hub-client/src/components/render/PreviewStatusBar.integration.test.tsx b/hub-client/src/components/render/PreviewStatusBar.integration.test.tsx new file mode 100644 index 000000000..167181500 --- /dev/null +++ b/hub-client/src/components/render/PreviewStatusBar.integration.test.tsx @@ -0,0 +1,350 @@ +/** + * Tests for PreviewStatusBar (bd-yai4w8ly). + * + * The single, merged preview status line that replaces the former three + * pieces (RunControl, the inline `.executor-online-bar`, and + * ClearCaptureControl). It selectively shows executor liveness, the + * "showing executed output" message, and both the Clear and Run/Re-run + * buttons — as a function of `executorsOnline`, `hasExecutableCells`, and + * the active document's `CaptureRef`. + * + * The state -> rendering contract mirrors the table in + * claude-notes/plans/2026-07-01-merge-preview-status-line.md. + * + * @vitest-environment jsdom + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, fireEvent, act } from '@testing-library/react'; +import type { CaptureRef } from '@quarto/preview-runtime'; +import { PreviewStatusBar, PENDING_TIMEOUT_MS } from './PreviewStatusBar'; + +const idle: CaptureRef = { captureDocId: 'cap-1', state: 'idle' }; + +/** The Run/Re-run button, matched by its stable aria-label. */ +const runButton = () => screen.queryByRole('button', { name: /run code cells/i }); +/** The initial "Clear results…" button (not the confirm-state "Clear"). */ +const clearButton = () => screen.queryByRole('button', { name: /clear results/i }); + +describe('PreviewStatusBar (bd-yai4w8ly merged status line)', () => { + // ---- visibility ------------------------------------------------------- + + it('renders nothing when no executor is online and there is no capture', () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); + + // ---- executor axis, no capture --------------------------------------- + + it('shows "Executor online" with a dot and no buttons when online but doc has no executable cells', () => { + const { container } = render( + , + ); + expect(screen.getByText(/executor online/i)).toBeInTheDocument(); + expect(container.querySelector('.executor-online-dot')).toBeInTheDocument(); + expect(runButton()).toBeNull(); + expect(clearButton()).toBeNull(); + }); + + it('shows "Run" and calls onRun(path) when online with executable cells and no capture', () => { + const onRun = vi.fn(); + render( + , + ); + const btn = runButton()!; + expect(btn).toHaveTextContent('Run'); + expect(clearButton()).toBeNull(); + fireEvent.click(btn); + expect(onRun).toHaveBeenCalledWith('doc.qmd'); + }); + + // ---- capture present, executor online -------------------------------- + + it('shows "Showing executed output", Clear, and "Re-run" for an idle capture', () => { + render( + , + ); + expect(screen.getByText(/showing executed output/i)).toBeInTheDocument(); + expect(clearButton()).toBeInTheDocument(); + expect(runButton()).toHaveTextContent('Re-run'); + }); + + it('reflects a running capture: status "Executing…" and a disabled Run button; Clear stays available', () => { + const capture: CaptureRef = { captureDocId: 'cap-1', state: 'running' }; + const { container } = render( + , + ); + // Both the status label and the Run button read "Executing…" while busy. + expect(container.querySelector('.preview-status-label')).toHaveTextContent('Executing…'); + expect(runButton()).toBeDisabled(); + expect(runButton()).toHaveTextContent('Executing…'); + expect(clearButton()).toBeInTheDocument(); + }); + + it('surfaces the last error (as an alert) and re-enables the Run button', () => { + const capture: CaptureRef = { + captureDocId: 'cap-1', + state: 'error', + lastError: 'engine boom', + }; + render( + , + ); + expect(screen.getByRole('alert')).toHaveTextContent('engine boom'); + expect(runButton()).not.toBeDisabled(); + }); + + it('shows BOTH facts for a stale capture: "Showing executed output" and "code changed"', () => { + const capture: CaptureRef = { captureDocId: 'cap-1', state: 'idle', staleness: true }; + render( + , + ); + expect(screen.getByText(/showing executed output/i)).toBeInTheDocument(); + expect(screen.getByText(/code changed/i)).toBeInTheDocument(); + expect(runButton()).toHaveTextContent('Re-run'); + }); + + // ---- capture present, executor OFFLINE (Clear is executor-independent) - + + it('still shows the capture status + Clear (but no Run) when no executor is online', () => { + render( + , + ); + expect(screen.getByText(/showing executed output/i)).toBeInTheDocument(); + expect(clearButton()).toBeInTheDocument(); + expect(runButton()).toBeNull(); + }); + + // ---- button order: Clear before Run so Run stays pinned far right ----- + + it('renders Clear before Run in DOM order so Run is pinned to the far right', () => { + render( + , + ); + const buttons = screen.getAllByRole('button'); + expect(buttons[0]).toHaveAccessibleName(/clear results/i); + expect(buttons[buttons.length - 1]).toHaveAccessibleName(/run code cells/i); + }); + + // ---- run pending state machine (ported from RunControl) --------------- + + it('goes busy on Run click, then re-enables when a new capture arrives', () => { + const onRun = vi.fn(); + const { rerender } = render( + , + ); + + fireEvent.click(runButton()!); + expect(runButton()).toHaveTextContent('Executing…'); + expect(runButton()).toBeDisabled(); + + const next: CaptureRef = { captureDocId: 'cap-2', state: 'idle' }; + rerender( + , + ); + expect(runButton()).toHaveTextContent('Re-run'); + expect(runButton()).not.toBeDisabled(); + }); + + it('clears a stuck pending flag after PENDING_TIMEOUT_MS (request found no executor)', () => { + vi.useFakeTimers(); + try { + render( + , + ); + fireEvent.click(runButton()!); + expect(runButton()).toBeDisabled(); + act(() => { + vi.advanceTimersByTime(PENDING_TIMEOUT_MS + 1); + }); + expect(runButton()).not.toBeDisabled(); + expect(runButton()).toHaveTextContent('Re-run'); + } finally { + vi.useRealTimers(); + } + }); + + it('disarms a pending Run when the active document changes', () => { + const { rerender } = render( + , + ); + fireEvent.click(runButton()!); + expect(runButton()).toBeDisabled(); + + rerender( + , + ); + // New doc, no capture -> back to "Run", not disabled. + expect(runButton()).toHaveTextContent('Run'); + expect(runButton()).not.toBeDisabled(); + }); + + // ---- clear confirm state machine (ported from ClearCaptureControl) ---- + + it('requires confirmation before clearing, then calls onClear with the path', () => { + const onClear = vi.fn(); + render( + , + ); + + fireEvent.click(clearButton()!); + expect(onClear).not.toHaveBeenCalled(); + // The confirmation must name the collaborator-wide effect. + expect(screen.getByText(/collaborator/i)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: /^clear$/i })); + expect(onClear).toHaveBeenCalledTimes(1); + expect(onClear).toHaveBeenCalledWith('doc.qmd'); + }); + + it('cancelling the confirmation does not clear and restores the affordance', () => { + const onClear = vi.fn(); + render( + , + ); + + fireEvent.click(clearButton()!); + fireEvent.click(screen.getByRole('button', { name: /cancel/i })); + expect(onClear).not.toHaveBeenCalled(); + expect(clearButton()).toBeInTheDocument(); + }); + + it('disarms a pending clear confirmation when the active document changes', () => { + const { rerender } = render( + , + ); + fireEvent.click(clearButton()!); + expect(screen.getByText(/collaborator/i)).toBeInTheDocument(); + + rerender( + , + ); + // Confirmation is gone; back to the plain "Clear results…" affordance. + expect(screen.queryByText(/collaborator/i)).toBeNull(); + expect(clearButton()).toBeInTheDocument(); + }); +}); diff --git a/hub-client/src/components/render/PreviewStatusBar.tsx b/hub-client/src/components/render/PreviewStatusBar.tsx new file mode 100644 index 000000000..4ae70bdca --- /dev/null +++ b/hub-client/src/components/render/PreviewStatusBar.tsx @@ -0,0 +1,211 @@ +import { useEffect, useState } from 'react'; +import type { CaptureRef } from '@quarto/preview-runtime'; + +/** + * The preview pane's single execution status line (bd-yai4w8ly). + * + * This merges what used to be three separate strips stacked above the preview + * iframe: + * - `RunControl` — the "Executor online" + Run/Re-run affordance, + * - the inline `.executor-online-bar` read-only indicator, and + * - `ClearCaptureControl` — "Showing executed output" + Clear results. + * + * It renders **at most one row**: a green liveness dot (when an executor is + * online), a single status label chosen by precedence, and a right-aligned + * action group. The buttons are gated independently — **Clear** whenever a + * capture exists, **Run/Re-run** whenever an executor is online and the doc has + * executable cells — and rendered in DOM order `[Clear] [Run]` so Run stays + * pinned to the far right as state transitions (see the plan, + * claude-notes/plans/2026-07-01-merge-preview-status-line.md). + * + * Both mutations are injected (`onRun` = ephemeral `exec/request`, `onClear` = + * shared `CaptureRef` sidecar delete) so this component stays presentational + * and unit-testable. + * + * Two small local state machines are preserved from the former components: + * - a **run pending** snapshot (the `captureDocId` at request time) that + * shows "Executing…" optimistically and clears when a new capture arrives, + * the provider reports an error, the doc changes, or after + * {@link PENDING_TIMEOUT_MS} (an ephemeral request may reach no executor); + * - a **clear confirmation** (two-step, because clearing affects every + * collaborator), disarmed when the active document changes. + */ + +/** How long to show "Executing…" before assuming the request was lost. */ +export const PENDING_TIMEOUT_MS = 30_000; + +export interface PreviewStatusBarProps { + /** Active document path, or null when no document is open. */ + path: string | null; + /** Whether at least one `q2` executor is online (a live capability beacon). */ + executorsOnline: boolean; + /** Whether the active document has executable code cells. */ + hasExecutableCells: boolean; + /** The active document's capture sidecar entry, if any. */ + capture?: CaptureRef; + /** Broadcast an execute request for `path`. */ + onRun: (path: string) => void; + /** Clear the capture for `path` (removes the shared sidecar entry). */ + onClear: (path: string) => void; +} + +export function PreviewStatusBar({ + path, + executorsOnline, + hasExecutableCells, + capture, + onRun, + onClear, +}: PreviewStatusBarProps) { + // The `captureDocId` snapshot taken when we sent a run request, or null when + // no request is in flight. `''` means "no capture existed at request time". + const [pendingSnapshot, setPendingSnapshot] = useState(null); + // Two-step clear confirmation. + const [confirming, setConfirming] = useState(false); + + const state = capture?.state; + const captureDocId = capture?.captureDocId; + + // Disarm both local state machines when the active document changes, so a + // pending run or a mid-confirm clear never carries over to a different file. + useEffect(() => { + setPendingSnapshot(null); + setConfirming(false); + }, [path]); + + // Clear the pending flag once a run resolves: a new capture arrived (doc id + // changed) or the provider reported an error. + useEffect(() => { + if (pendingSnapshot === null) return; + if (state === 'error' || (captureDocId ?? '') !== pendingSnapshot) { + setPendingSnapshot(null); + } + }, [captureDocId, state, pendingSnapshot]); + + // Safety net: an ephemeral request may find no executor, so never stay + // "Executing…" forever. + useEffect(() => { + if (pendingSnapshot === null) return; + const timer = setTimeout(() => setPendingSnapshot(null), PENDING_TIMEOUT_MS); + return () => clearTimeout(timer); + }, [pendingSnapshot]); + + const hasCapture = !!capture; + + // Nothing to say and nothing to do — hide the bar entirely. + if (!executorsOnline && !hasCapture) { + return null; + } + + const busy = pendingSnapshot !== null || state === 'running'; + const canRun = executorsOnline && hasExecutableCells && !!path; + const canClear = hasCapture && !!path; + const runLabel = busy ? 'Executing…' : hasCapture ? 'Re-run' : 'Run'; + + const handleRun = () => { + if (!path) return; + setPendingSnapshot(captureDocId ?? ''); + onRun(path); + }; + + const handleConfirmClear = () => { + if (path) onClear(path); + setConfirming(false); + }; + + return ( +
+ {executorsOnline &&
+ ); +} + +/** + * The single status label, chosen by precedence so transient run states win + * over the steady-state "showing output" / "executor online" messages. + */ +function StatusLabel({ + busy, + capture, + executorsOnline, +}: { + busy: boolean; + capture?: CaptureRef; + executorsOnline: boolean; +}) { + if (busy) { + return Executing…; + } + if (capture?.state === 'error' && capture.lastError) { + return ( + + {capture.lastError} + + ); + } + if (capture) { + // Show BOTH facts when the capture is stale (decision 2): the output is + // still displayed, and the code has changed since it was produced. + return ( + + Showing executed output{capture.staleness ? ' · code changed' : ''} + + ); + } + // Guaranteed reachable only when an executor is online (the bar is hidden + // when there is neither a capture nor an executor). + return {executorsOnline ? 'Executor online' : null}; +} diff --git a/hub-client/src/components/render/ReactPreview.capture.integration.test.tsx b/hub-client/src/components/render/ReactPreview.capture.integration.test.tsx new file mode 100644 index 000000000..af20b9136 --- /dev/null +++ b/hub-client/src/components/render/ReactPreview.capture.integration.test.tsx @@ -0,0 +1,123 @@ +/** + * Capture-consumption test for `ReactPreview` (bd-sfet3264, Phase 1D). + * + * hub-client must consume a recorded engine capture for the active document: + * when the `captures` sidecar (threaded down from App.tsx) has an entry for + * the active file, ReactPreview fetches that capture binary doc's bytes via + * `getBinaryDocById` and threads them into the q2-preview render call + * (`renderPageInProjectWithAttribution`, 4th arg) so the recorded engine + * output is spliced into the AST. + * + * This test pins the wiring at the hub-client boundary (props → fetch → + * render-call argument). The actual splicing is covered by the WASM-level + * test `captureSplice.wasm.test.ts`. + * + * @vitest-environment jsdom + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, waitFor } from '@testing-library/react'; +import React from 'react'; + +// Distinct capture bytes the getBinaryDocById mock will return, so we can +// assert the EXACT bytes reach the render call (not just "something truthy"). +const CAPTURE_BYTES = new Uint8Array([1, 2, 3, 4, 5]); + +const { renderPageInProjectWithAttribution, getBinaryDocById } = vi.hoisted(() => ({ + renderPageInProjectWithAttribution: vi.fn(async () => ({ + success: true, + ast_json: '{"pandoc-api-version":[1,23,1],"meta":{},"blocks":[]}', + untransformed_ast_json: '{"pandoc-api-version":[1,23,1],"meta":{},"blocks":[]}', + theme_fingerprint: 'fp-1', + diagnostics: [], + warnings: [], + })), + getBinaryDocById: vi.fn(async () => ({ + content: new Uint8Array([1, 2, 3, 4, 5]), + mimeType: 'application/x-engine-capture+gzip', + })), +})); + +vi.mock('@quarto/preview-runtime', () => ({ + renderPageInProjectWithAttribution, + getBinaryDocById, + renderPageForPreview: vi.fn(), + parseQmdToAstWithAttribution: vi.fn(async () => ({ success: true, ast: '{}', diagnostics: [] })), + isWasmReady: () => true, + incrementalWriteQmd: vi.fn(), + applyNodeEdit: vi.fn(), + parseQmdContentSync: vi.fn(() => ({ success: true, ast: '{}' })), + getActorId: () => 'actor-1', + regenerateNestedBuffers: vi.fn(() => ({})), + pipelineKindForFormat: (f: string) => (f === 'q2-preview' ? 'preview' : undefined), +})); + +vi.mock('../../hooks/useAttribution', () => ({ + useAttribution: () => ({ payload: null, generating: false }), +})); +vi.mock('../../hooks/usePreference', () => ({ + usePreference: () => [false, vi.fn()], +})); +vi.mock('./ReactRenderer', () => ({ + default: () =>
, +})); + +import ReactPreview from './ReactPreview'; +import type { CaptureRef } from '@quarto/preview-runtime'; + +function baseProps(captures?: Record) { + return { + content: '---\nformat: q2-preview\nengine: knitr\n---\n\n```{r}\n1 + 1\n```\n', + currentFile: { path: 'doc.qmd', name: 'doc.qmd' } as any, + files: [], + fileContents: new Map([['doc.qmd', 'x']]), + scrollSyncEnabled: false, + editorRef: { current: null } as any, + editorReady: true, + editorHasFocusRef: { current: false } as any, + onFileChange: () => {}, + onOpenNewFileDialog: () => {}, + onDiagnosticsChange: () => {}, + onContentRewrite: () => {}, + format: 'q2-preview', + attributionOn: false, + captures, + }; +} + +describe('ReactPreview capture consumption (bd-sfet3264, Phase 1D)', () => { + beforeEach(() => { + renderPageInProjectWithAttribution.mockClear(); + getBinaryDocById.mockClear(); + }); + + it('fetches the active doc capture and threads its bytes into the render call', async () => { + const captures: Record = { + 'doc.qmd': { captureDocId: 'cap-1', state: 'idle' }, + }; + + render(); + + // The capture binary doc for the active file must be fetched by id. + await waitFor(() => expect(getBinaryDocById).toHaveBeenCalledWith('cap-1')); + + // The fetched bytes must reach the render call as the 4th argument. + await waitFor(() => { + const calls = renderPageInProjectWithAttribution.mock.calls; + const withCapture = calls.find((c) => c[3] !== undefined); + expect(withCapture, 'a render call must carry the capture bytes').toBeTruthy(); + expect(withCapture![3]).toEqual(CAPTURE_BYTES); + }); + }); + + it('passes no capture bytes when the active doc has no capture entry', async () => { + render(); + + await waitFor(() => expect(renderPageInProjectWithAttribution).toHaveBeenCalled()); + // No capture sidecar ⇒ no fetch, and every render call's 4th arg is undefined. + expect(getBinaryDocById).not.toHaveBeenCalled(); + for (const call of renderPageInProjectWithAttribution.mock.calls) { + expect(call[3]).toBeUndefined(); + } + }); +}); diff --git a/hub-client/src/components/render/ReactPreview.tsx b/hub-client/src/components/render/ReactPreview.tsx index a5db021d1..349f33750 100644 --- a/hub-client/src/components/render/ReactPreview.tsx +++ b/hub-client/src/components/render/ReactPreview.tsx @@ -3,7 +3,7 @@ import type { CSSProperties } from 'react'; import type * as Monaco from 'monaco-editor'; import type { FileEntry } from '@quarto/preview-renderer/types/project'; import type { Diagnostic, PreviewNodeEditPayload } from '@quarto/preview-renderer/types/diagnostic'; -import type { ActorIdentity } from '@quarto/preview-runtime'; +import type { ActorIdentity, CaptureRef } from '@quarto/preview-runtime'; import { parseQmdToAstWithAttribution, renderPageInProjectWithAttribution, @@ -17,6 +17,7 @@ import { } from '@quarto/preview-runtime'; import { pipelineKindForFormat } from '@quarto/preview-runtime'; import { useAttribution } from '../../hooks/useAttribution'; +import { useActiveCaptureBytes } from '../../hooks/useActiveCaptureBytes'; import { stripAnsi } from '@quarto/preview-renderer/utils/stripAnsi'; import { PreviewErrorOverlay } from '@quarto/preview-renderer/overlays/PreviewErrorOverlay'; import { usePreference } from '../../hooks/usePreference'; @@ -110,6 +111,14 @@ interface PreviewProps { * entry, so the Phase 6 producer invariant always holds. */ identities?: Record; + /** + * Path → recorded engine capture sidecar entry (bd-sfet3264). The + * active document's entry (if any) points at a capture binary doc; + * ReactPreview fetches its gzipped `EngineCapture[]` bytes and threads + * them into the render so executed engine output is spliced into the + * AST. Absent/empty ⇒ source-only rendering (today's behaviour). + */ + captures?: Record; /** * Attribution overlay on/off. Session-only, owned by `Editor.tsx` * and driven by the toggle in the replay bar. When false, @@ -186,6 +195,11 @@ async function doRender( documentPath?: string; format: string; attributionJson: string | null; + // bd-sfet3264: gzipped-JSON `EngineCapture[]` for the active document + // (fetched from the capture binary doc). When present, the q2-preview + // pipeline splices the recorded engine output into the AST. `undefined` + // renders code cells as source. + captureGzJson?: Uint8Array; } ): Promise { if (!isWasmReady()) { @@ -223,11 +237,12 @@ async function doRender( // performs the preview format substitution but does not yet thread // attribution — slides have no attribution overlay today (follow-up). const result = isSlidesPreview - ? await renderPageForPreview(options.documentPath, undefined, undefined) + ? await renderPageForPreview(options.documentPath, undefined, options.captureGzJson) : await renderPageInProjectWithAttribution( options.documentPath, undefined, options.attributionJson, + options.captureGzJson, ); const allDiagnostics: Diagnostic[] = [ ...(result.diagnostics ?? []), @@ -429,6 +444,7 @@ export default function ReactPreview({ onRegisterReplayScroll, format, identities, + captures, attributionOn, onAttributionGeneratingChange, }: PreviewProps) { @@ -553,6 +569,16 @@ export default function ReactPreview({ return () => onAttributionGeneratingChange?.(false); }, [attributionGenerating, onAttributionGeneratingChange]); + // bd-sfet3264 (Phase 1D): recorded engine capture for the active document. + // + // The `captures` sidecar (threaded down from App.tsx) maps each path to a + // CaptureRef pointing at a capture binary doc. `useActiveCaptureBytes` fetches + // the active file's capture bytes so `doRender` can splice the recorded engine + // output into the AST; a freshly-arrived capture updates `captureBytes`, a + // render input below, so the preview re-renders to show executed output. The + // same hook feeds the default `format: html` renderer (`Preview`). + const captureBytes = useActiveCaptureBytes(captures, currentFile?.path); + // Debounce rendering const renderTimeoutRef = useRef(null); const lastContentRef = useRef(''); @@ -635,6 +661,7 @@ export default function ReactPreview({ documentPath, format, attributionJson: attributionPayload, + captureGzJson: captureBytes, }); if (qmdContent !== lastContentRef.current) return; @@ -678,7 +705,7 @@ export default function ReactPreview({ setPreviewState('ERROR_FROM_GOOD'); } } - }, [scrollSyncEnabled, onDiagnosticsChange, onAstChange, format, attributionPayload]); + }, [scrollSyncEnabled, onDiagnosticsChange, onAstChange, format, attributionPayload, captureBytes]); // Immediate render update (no debounce) const updatePreview = useCallback((newContent: string, documentPath?: string) => { @@ -713,6 +740,7 @@ export default function ReactPreview({ currentFile?.path, onDiagnosticsChange, attributionPayload, + captureBytes, ]); // Reset preview state when file changes diff --git a/hub-client/src/hooks/useActiveCaptureBytes.integration.test.tsx b/hub-client/src/hooks/useActiveCaptureBytes.integration.test.tsx new file mode 100644 index 000000000..729a10645 --- /dev/null +++ b/hub-client/src/hooks/useActiveCaptureBytes.integration.test.tsx @@ -0,0 +1,60 @@ +/** + * Tests for useActiveCaptureBytes (bd-uy4uygha). + * + * The shared capture-fetch hook used by both the q2-preview renderer + * (`ReactPreview`) and the default `format: html` renderer (`Preview`): given + * the project's capture sidecar + the active path, it resolves the active + * document's `captureDocId` and fetches the capture binary doc's bytes. + * + * @vitest-environment jsdom + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { renderHook, waitFor } from '@testing-library/react'; + +const CAPTURE_BYTES = new Uint8Array([9, 8, 7, 6]); + +const { getBinaryDocById } = vi.hoisted(() => ({ + getBinaryDocById: vi.fn(async () => ({ + content: new Uint8Array([9, 8, 7, 6]), + mimeType: 'application/x-engine-capture+gzip', + })), +})); + +vi.mock('@quarto/preview-runtime', () => ({ getBinaryDocById })); + +import { useActiveCaptureBytes } from './useActiveCaptureBytes'; +import type { CaptureRef } from '@quarto/preview-runtime'; + +describe('useActiveCaptureBytes (bd-uy4uygha)', () => { + beforeEach(() => { + getBinaryDocById.mockClear(); + }); + + it('fetches the active document capture bytes by captureDocId', async () => { + const captures: Record = { + 'doc.qmd': { captureDocId: 'cap-1', state: 'idle' }, + 'other.qmd': { captureDocId: 'cap-other', state: 'idle' }, + }; + const { result } = renderHook(() => useActiveCaptureBytes(captures, 'doc.qmd')); + + await waitFor(() => expect(getBinaryDocById).toHaveBeenCalledWith('cap-1')); + await waitFor(() => expect(result.current).toEqual(CAPTURE_BYTES)); + }); + + it('returns undefined and does not fetch when the active path has no capture', async () => { + const captures: Record = { 'other.qmd': { captureDocId: 'cap-x' } }; + const { result } = renderHook(() => useActiveCaptureBytes(captures, 'doc.qmd')); + + expect(result.current).toBeUndefined(); + expect(getBinaryDocById).not.toHaveBeenCalled(); + }); + + it('returns undefined when the path is undefined', () => { + const captures: Record = { 'doc.qmd': { captureDocId: 'cap-1' } }; + const { result } = renderHook(() => useActiveCaptureBytes(captures, undefined)); + + expect(result.current).toBeUndefined(); + expect(getBinaryDocById).not.toHaveBeenCalled(); + }); +}); diff --git a/hub-client/src/hooks/useActiveCaptureBytes.ts b/hub-client/src/hooks/useActiveCaptureBytes.ts new file mode 100644 index 000000000..809f5f72e --- /dev/null +++ b/hub-client/src/hooks/useActiveCaptureBytes.ts @@ -0,0 +1,46 @@ +import { useEffect, useState } from 'react'; +import { getBinaryDocById, type CaptureRef } from '@quarto/preview-runtime'; + +/** + * Fetch the gzipped `EngineCapture[]` bytes for the active document's recorded + * capture (bd-sfet3264 / bd-uy4uygha). + * + * Given the project's `captures` sidecar and the active file path, resolves the + * active document's `captureDocId` and fetches the capture binary doc's bytes + * via `getBinaryDocById`. Returns `undefined` when there's no capture (so the + * caller renders code cells as source). Keyed on the `captureDocId` (not + * content), so it only re-fetches when a fresh capture arrives — not on every + * keystroke — and a dangling/unreachable capture falls back to `undefined`. + * + * Shared by the `q2-preview` renderer (`ReactPreview`) and the default + * `format: html` renderer (`Preview`), which both splice captures into their + * respective render entries. + */ +export function useActiveCaptureBytes( + captures: Record | undefined, + path: string | undefined, +): Uint8Array | undefined { + const activeCaptureDocId = path ? captures?.[path]?.captureDocId : undefined; + const [captureBytes, setCaptureBytes] = useState(undefined); + + useEffect(() => { + let cancelled = false; + if (!activeCaptureDocId) { + setCaptureBytes(undefined); + return; + } + (async () => { + try { + const doc = await getBinaryDocById(activeCaptureDocId); + if (!cancelled) setCaptureBytes(doc?.content); + } catch { + if (!cancelled) setCaptureBytes(undefined); + } + })(); + return () => { + cancelled = true; + }; + }, [activeCaptureDocId]); + + return captureBytes; +} diff --git a/hub-client/src/hooks/useExecutionChannel.integration.test.tsx b/hub-client/src/hooks/useExecutionChannel.integration.test.tsx new file mode 100644 index 000000000..a0efeed40 --- /dev/null +++ b/hub-client/src/hooks/useExecutionChannel.integration.test.tsx @@ -0,0 +1,86 @@ +/** + * Tests for useExecutionChannel (bd-sfet3264, Phase 2D). + * + * Verifies the lifecycle glue: while connected, an injected capability beacon + * surfaces as a live executor; disconnecting tears the channel down. + * + * @vitest-environment jsdom + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { renderHook, act } from '@testing-library/react'; + +// A fake index DocHandle the mocked getIndexHandle returns. +const fake = (() => { + const handlers = new Set<(p: { message: unknown }) => void>(); + return { + handle: { + broadcast: vi.fn(), + on: (_e: string, h: (p: { message: unknown }) => void) => handlers.add(h), + off: (_e: string, h: (p: { message: unknown }) => void) => handlers.delete(h), + }, + inject: (message: unknown) => handlers.forEach((h) => h({ message })), + handlerCount: () => handlers.size, + }; +})(); + +vi.mock('@quarto/preview-runtime', () => ({ + getIndexHandle: () => fake.handle, +})); + +import { useExecutionChannel } from './useExecutionChannel'; + +describe('useExecutionChannel (Phase 2D)', () => { + beforeEach(() => { + vi.useFakeTimers(); + fake.handle.broadcast.mockClear(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('returns no executors when offline and does not subscribe', () => { + const { result } = renderHook(() => useExecutionChannel(false, 'idx-1')); + expect(result.current.executors).toEqual([]); + expect(fake.handlerCount()).toBe(0); + }); + + it('surfaces a live executor from an injected beacon while connected', () => { + const { result } = renderHook(() => useExecutionChannel(true, 'idx-1')); + expect(fake.handlerCount()).toBe(1); + + act(() => { + fake.inject({ kind: 'exec/beacon', actorId: 'exec-1', engines: ['knitr'], generation: 0 }); + }); + + expect(result.current.executors).toHaveLength(1); + expect(result.current.executors[0]).toMatchObject({ actorId: 'exec-1', engines: ['knitr'] }); + }); + + it('requestExecution broadcasts an exec/request while connected', () => { + const { result } = renderHook(() => useExecutionChannel(true, 'idx-1')); + + let requestId: string | null = null; + act(() => { + requestId = result.current.requestExecution('doc.qmd'); + }); + + expect(requestId).toBeTruthy(); + expect(fake.handle.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'exec/request', path: 'doc.qmd' }), + ); + }); + + it('requestExecution returns null when offline', () => { + const { result } = renderHook(() => useExecutionChannel(false, 'idx-1')); + expect(result.current.requestExecution('doc.qmd')).toBeNull(); + expect(fake.handle.broadcast).not.toHaveBeenCalled(); + }); + + it('tears the channel down on unmount', () => { + const { unmount } = renderHook(() => useExecutionChannel(true, 'idx-1')); + expect(fake.handlerCount()).toBe(1); + unmount(); + expect(fake.handlerCount()).toBe(0); + }); +}); diff --git a/hub-client/src/hooks/useExecutionChannel.ts b/hub-client/src/hooks/useExecutionChannel.ts new file mode 100644 index 000000000..a07d96dfe --- /dev/null +++ b/hub-client/src/hooks/useExecutionChannel.ts @@ -0,0 +1,61 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { getIndexHandle } from '@quarto/preview-runtime'; +import { + createExecutionChannel, + type ExecutionChannel, + type LiveExecutor, +} from '../services/executionChannel'; + +/** What {@link useExecutionChannel} exposes to the editor. */ +export interface UseExecutionChannel { + /** Executors currently believed online (refreshed by beacons, pruned stale). */ + executors: LiveExecutor[]; + /** + * Broadcast an "execute this document now" request on the index channel. + * Returns the request id, or `null` if not connected. Stable across renders. + */ + requestExecution: (path: string) => string | null; +} + +/** + * Track which `q2` executors are online for the connected project and expose a + * way to ask one to run a document (bd-sfet3264, Phase 2D + Phase 4b). + * + * Starts an execution channel on the index DocHandle while connected: it + * surfaces the live-executor set (Phase 2) and, for Phase 4b, hands back a + * stable `requestExecution` the Run affordance calls. The channel is torn down + * and rebuilt when the connection drops/restores or the active project changes, + * so it always listens/broadcasts on the current project's index handle. + */ +export function useExecutionChannel( + isOnline: boolean, + indexDocId: string | null, +): UseExecutionChannel { + const [executors, setExecutors] = useState([]); + const channelRef = useRef(null); + + useEffect(() => { + if (!isOnline || !indexDocId) { + setExecutors([]); + return; + } + const channel = createExecutionChannel({ + getIndexHandle: () => getIndexHandle(), + onExecutorsChange: setExecutors, + }); + channelRef.current = channel; + channel.start(); + return () => { + channel.stop(); + channelRef.current = null; + setExecutors([]); + }; + }, [isOnline, indexDocId]); + + const requestExecution = useCallback( + (path: string) => channelRef.current?.requestExecution(path) ?? null, + [], + ); + + return { executors, requestExecution }; +} diff --git a/hub-client/src/services/captureSplice.wasm.test.ts b/hub-client/src/services/captureSplice.wasm.test.ts new file mode 100644 index 000000000..216538f35 --- /dev/null +++ b/hub-client/src/services/captureSplice.wasm.test.ts @@ -0,0 +1,195 @@ +/** + * WASM test for capture-aware hub-client rendering (bd-sfet3264, Phase 1A). + * + * hub-client's main preview path calls `render_page_in_project_with_attribution`. + * For the remote-execution-provider feature, that entry must be able to + * consume a recorded engine capture (the same gzipped-JSON `EngineCapture[]` + * wire format the capture binary doc holds) and splice the recorded engine + * output into the rendered AST — exactly as `render_page_for_preview` already + * does for the q2-preview SPA, but *without* losing attribution. + * + * The inner WASM helpers already accept both captures and attribution; this + * test pins the outer entry's new 4th argument (`capture_gz_json`). + * + * Strategy: render a single-file q2-preview doc with one `{r}` cell, passing + * a hand-built capture whose post-engine markdown contains a marker string + * (`SPLICEDOUTPUT42`) that appears ONLY in the capture, never in the source. + * If the marker shows up in the rendered AST, the splice fired through this + * entry. A no-capture baseline confirms the marker is genuinely capture-only. + */ + +import { describe, it, expect, beforeAll, beforeEach } from 'vitest'; +import { readFile } from 'fs/promises'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { gzipSync } from 'zlib'; +import { setVfsCallbacks } from '/src/wasm-js-bridge/sass.js'; + +interface WasmModule { + default: (input?: BufferSource) => Promise; + vfs_add_file: (path: string, content: string) => string; + vfs_clear: () => string; + vfs_read_file: (path: string) => string; + render_page_in_project_with_attribution: ( + path: string, + user_grammars?: unknown, + attribution_json?: string, + capture_gz_json?: Uint8Array, + ) => Promise; +} + +interface RenderResponse { + success: boolean; + error?: string; + ast_json?: string; +} + +const MARKER = 'SPLICEDOUTPUT42'; + +// A single-file q2-preview doc with one knitr cell. `format: q2-preview` +// forces the preview pipeline branch (the only branch that runs the +// CaptureSpliceStage); `engine: knitr` marks the cell as executable. +const DOC = [ + '---', + 'format: q2-preview', + 'engine: knitr', + '---', + '', + '```{r}', + '1 + 1', + '```', + '', +].join('\n'); + +// One capture, knitr engine. `input_qmd` carries the same `{r}` cell so its +// content-hash matches the doc's cell; `result.markdown` is the post-engine +// markdown — a `.cell` wrapper whose stdout output is the marker. +function captureBytes(): Uint8Array { + const captures = [ + { + engine_name: 'knitr', + input_qmd: '```{r}\n1 + 1\n```\n', + result: { + markdown: [ + '::: {.cell}', + '```{.r .cell-code}', + '1 + 1', + '```', + '', + '::: {.cell-output .cell-output-stdout}', + '```', + MARKER, + '```', + '', + ':::', + '', + ':::', + '', + ].join('\n'), + }, + }, + ]; + return new Uint8Array(gzipSync(Buffer.from(JSON.stringify(captures)))); +} + +let wasm: WasmModule; + +beforeAll(async () => { + const __dirname = dirname(fileURLToPath(import.meta.url)); + const wasmDir = join(__dirname, '../../wasm-quarto-hub-client'); + const wasmPath = join(wasmDir, 'wasm_quarto_hub_client_bg.wasm'); + const wasmBytes = await readFile(wasmPath); + + wasm = (await import('wasm-quarto-hub-client')) as unknown as WasmModule; + await wasm.default(wasmBytes); + + setVfsCallbacks( + (path: string): string | null => { + try { + const result = JSON.parse(wasm.vfs_read_file(path)) as { + success: boolean; + content?: string; + }; + return result.success && result.content !== undefined ? result.content : null; + } catch { + return null; + } + }, + (path: string): boolean => { + try { + const result = JSON.parse(wasm.vfs_read_file(path)) as { + success: boolean; + content?: string; + }; + return result.success && result.content !== undefined; + } catch { + return false; + } + }, + ); +}); + +beforeEach(() => { + wasm.vfs_clear(); +}); + +describe('render_page_in_project_with_attribution capture splicing (Phase 1A)', () => { + it('baseline: without a capture, the source renders without the engine-output marker', async () => { + wasm.vfs_add_file('/project/doc.qmd', DOC); + + const json = await wasm.render_page_in_project_with_attribution( + '/project/doc.qmd', + undefined, + undefined, + undefined, + ); + const result = JSON.parse(json) as RenderResponse; + + expect(result.success, `Render failed: ${result.error}`).toBe(true); + expect(result.ast_json).toBeTruthy(); + expect( + result.ast_json!.includes(MARKER), + 'the marker must be capture-only — it must NOT appear in a no-capture render', + ).toBe(false); + }); + + it('with a capture, the recorded engine output is spliced into the rendered AST', async () => { + wasm.vfs_add_file('/project/doc.qmd', DOC); + + const json = await wasm.render_page_in_project_with_attribution( + '/project/doc.qmd', + undefined, + undefined, + captureBytes(), + ); + const result = JSON.parse(json) as RenderResponse; + + expect(result.success, `Render failed: ${result.error}`).toBe(true); + expect(result.ast_json).toBeTruthy(); + expect( + result.ast_json!.includes(MARKER), + 'the capture output marker must appear in the AST when a capture is threaded through this entry', + ).toBe(true); + }); + + it('captures and attribution coexist: a capture splices even when an attribution payload is also supplied', async () => { + wasm.vfs_add_file('/project/doc.qmd', DOC); + + // `{}` is a valid no-op attribution payload (runs/identities both default + // to empty). The point is that supplying attribution must NOT cause the + // capture argument to be dropped — the inner renderer attaches both. + const json = await wasm.render_page_in_project_with_attribution( + '/project/doc.qmd', + undefined, + '{}', + captureBytes(), + ); + const result = JSON.parse(json) as RenderResponse; + + expect(result.success, `Render failed: ${result.error}`).toBe(true); + expect( + result.ast_json!.includes(MARKER), + 'capture must still splice when an attribution payload is also passed (both coexist)', + ).toBe(true); + }); +}); diff --git a/hub-client/src/services/captureSpliceHtml.wasm.test.ts b/hub-client/src/services/captureSpliceHtml.wasm.test.ts new file mode 100644 index 000000000..6f8d7409b --- /dev/null +++ b/hub-client/src/services/captureSpliceHtml.wasm.test.ts @@ -0,0 +1,158 @@ +/** + * WASM test: capture splicing in the DEFAULT `format: html` render (bd-uy4uygha). + * + * The sibling `captureSplice.wasm.test.ts` covers the `format: q2-preview` AST + * path. hub-client's *default* preview for a plain document (and every website + * page) renders `format: html` via a different WASM branch, which historically + * ignored captures — so a document executed by a connected `q2 provide-hub` + * showed source instead of output. This test pins that the HTML branch now + * splices the recorded engine output into the rendered `html`. + * + * Strategy mirrors the q2-preview test but with `format: html` and asserts on + * the `html` field (not `ast_json`). WASM has no native engines, so the `{r}` + * cell takes the markdown fallback; the splice (which runs before engine + * execution) is the only thing that can produce the marker. + */ + +import { describe, it, expect, beforeAll, beforeEach } from 'vitest'; +import { readFile } from 'fs/promises'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { gzipSync } from 'zlib'; +import { setVfsCallbacks } from '/src/wasm-js-bridge/sass.js'; + +interface WasmModule { + default: (input?: BufferSource) => Promise; + vfs_add_file: (path: string, content: string) => string; + vfs_clear: () => string; + vfs_read_file: (path: string) => string; + render_page_in_project_with_attribution: ( + path: string, + user_grammars?: unknown, + attribution_json?: string, + capture_gz_json?: Uint8Array, + ) => Promise; +} + +interface RenderResponse { + success: boolean; + error?: string; + html?: string; + ast_json?: string; +} + +const MARKER = 'SPLICEDHTML42'; + +// A single-file `format: html` doc with one knitr cell. +const DOC = ['---', 'format: html', 'engine: knitr', '---', '', '```{r}', '1 + 1', '```', ''].join( + '\n', +); + +function captureBytes(): Uint8Array { + const captures = [ + { + engine_name: 'knitr', + input_qmd: '```{r}\n1 + 1\n```\n', + result: { + markdown: [ + '::: {.cell}', + '```{.r .cell-code}', + '1 + 1', + '```', + '', + '::: {.cell-output .cell-output-stdout}', + '```', + MARKER, + '```', + '', + ':::', + '', + ':::', + '', + ].join('\n'), + }, + }, + ]; + return new Uint8Array(gzipSync(Buffer.from(JSON.stringify(captures)))); +} + +let wasm: WasmModule; + +beforeAll(async () => { + const __dirname = dirname(fileURLToPath(import.meta.url)); + const wasmDir = join(__dirname, '../../wasm-quarto-hub-client'); + const wasmPath = join(wasmDir, 'wasm_quarto_hub_client_bg.wasm'); + const wasmBytes = await readFile(wasmPath); + + wasm = (await import('wasm-quarto-hub-client')) as unknown as WasmModule; + await wasm.default(wasmBytes); + + setVfsCallbacks( + (path: string): string | null => { + try { + const result = JSON.parse(wasm.vfs_read_file(path)) as { + success: boolean; + content?: string; + }; + return result.success && result.content !== undefined ? result.content : null; + } catch { + return null; + } + }, + (path: string): boolean => { + try { + const result = JSON.parse(wasm.vfs_read_file(path)) as { + success: boolean; + content?: string; + }; + return result.success && result.content !== undefined; + } catch { + return false; + } + }, + ); +}); + +beforeEach(() => { + wasm.vfs_clear(); +}); + +describe('format: html capture splicing (bd-uy4uygha)', () => { + it('baseline: without a capture, the html has no engine-output marker', async () => { + wasm.vfs_add_file('/project/doc.qmd', DOC); + + const json = await wasm.render_page_in_project_with_attribution( + '/project/doc.qmd', + undefined, + undefined, + undefined, + ); + const result = JSON.parse(json) as RenderResponse; + + expect(result.success, `Render failed: ${result.error}`).toBe(true); + expect(result.html, 'html branch must return html').toBeTruthy(); + expect( + result.html!.includes(MARKER), + 'the marker must be capture-only — absent in a no-capture render', + ).toBe(false); + }); + + it('with a capture, the recorded engine output is spliced into the rendered html', async () => { + wasm.vfs_add_file('/project/doc.qmd', DOC); + + const json = await wasm.render_page_in_project_with_attribution( + '/project/doc.qmd', + undefined, + undefined, + captureBytes(), + ); + const result = JSON.parse(json) as RenderResponse; + + expect(result.success, `Render failed: ${result.error}`).toBe(true); + expect(result.html).toBeTruthy(); + expect( + result.html!.includes(MARKER), + 'the capture output marker must appear in the html when a capture is threaded through the html branch', + ).toBe(true); + }); +}); diff --git a/hub-client/src/services/executableCells.test.ts b/hub-client/src/services/executableCells.test.ts new file mode 100644 index 000000000..7b2391510 --- /dev/null +++ b/hub-client/src/services/executableCells.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { hasExecutableCells } from './executableCells'; + +describe('hasExecutableCells', () => { + it('detects braced engine cells', () => { + expect(hasExecutableCells('```{r}\n1 + 1\n```')).toBe(true); + expect(hasExecutableCells('text\n\n```{python}\nprint(1)\n```\n')).toBe(true); + expect(hasExecutableCells('```{ojs}\nx = 1\n```')).toBe(true); + }); + + it('accepts cell options after the language', () => { + expect(hasExecutableCells('```{r echo=false}\n1\n```')).toBe(true); + }); + + it('accepts tilde fences and up to 3 spaces of indent', () => { + expect(hasExecutableCells('~~~{r}\n1\n~~~')).toBe(true); + expect(hasExecutableCells(' ```{python}\n1\n```')).toBe(true); + }); + + it('ignores the dotted display-class form (not executable)', () => { + expect(hasExecutableCells('```{.python}\nprint(1)\n```')).toBe(false); + expect(hasExecutableCells('```{.r .numberLines}\n1\n```')).toBe(false); + }); + + it('ignores plain fences and prose', () => { + expect(hasExecutableCells('```\nplain code\n```')).toBe(false); + expect(hasExecutableCells('```python\nprint(1)\n```')).toBe(false); // language, no braces + expect(hasExecutableCells('# A heading\n\nSome prose with `inline` code.')).toBe(false); + expect(hasExecutableCells('')).toBe(false); + }); +}); diff --git a/hub-client/src/services/executableCells.ts b/hub-client/src/services/executableCells.ts new file mode 100644 index 000000000..1c2cc65c8 --- /dev/null +++ b/hub-client/src/services/executableCells.ts @@ -0,0 +1,23 @@ +/** + * Detect whether a qmd document has executable code cells (bd-sfet3264, + * Phase 4b). + * + * Used to gate the preview's "Run" affordance: there's no point offering to + * execute a document with no executable cells. An executable cell is a fenced + * code block whose info string is a *braced* engine language — ```` ```{r} ````, + * ```` ```{python} ````, ```` ```{ojs} ````, etc. The dotted class form + * (```` ```{.python} ````) is a *display* class, not executable, so we require + * a letter (not a dot) right after the brace. + * + * This is a deliberately loose line-scan, not a full parse: it can over-report + * on a fenced *example* that itself contains a ```` ```{r} ```` line, which is + * harmless (the worst case is showing a Run button that produces no capture). + */ + +// A fence open (``` or ~~~, optionally indented up to 3 spaces per CommonMark) +// immediately followed by `{` + an ASCII letter (the engine language). +const EXECUTABLE_CELL = /^[ \t]{0,3}(?:`{3,}|~{3,})\{[a-zA-Z]/m; + +export function hasExecutableCells(content: string): boolean { + return EXECUTABLE_CELL.test(content); +} diff --git a/hub-client/src/services/executionChannel.test.ts b/hub-client/src/services/executionChannel.test.ts new file mode 100644 index 000000000..2363b1602 --- /dev/null +++ b/hub-client/src/services/executionChannel.test.ts @@ -0,0 +1,254 @@ +/** + * Unit tests for the execution-channel wire format + pure helpers + * (bd-sfet3264, Phase 2B). + * + * These cover the cross-language wire contract (the Rust executor mirrors it + * in Phase 4) and the pure live-executor bookkeeping, with no timers or + * DocHandles involved. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import type { DocHandleEphemeralMessagePayload } from '@automerge/automerge-repo'; +import { + BEACON_INTERVAL_MS, + BEACON_TIMEOUT_MS, + makeBeacon, + makeExecuteRequest, + parseExecMessage, + applyBeacon, + pruneExecutors, + createExecutionChannel, + type ExecMessage, + type ExecRequestMessage, + type LiveExecutor, +} from './executionChannel'; + +describe('execution-channel timing constants', () => { + it('beacon timeout is 1.5x the interval (D2 liveness contract)', () => { + expect(BEACON_TIMEOUT_MS).toBe(BEACON_INTERVAL_MS * 1.5); + }); +}); + +describe('message builders', () => { + it('makeBeacon builds a well-formed beacon', () => { + expect(makeBeacon('actor-1', ['knitr', 'jupyter'], 2)).toEqual({ + kind: 'exec/beacon', + actorId: 'actor-1', + engines: ['knitr', 'jupyter'], + generation: 2, + }); + }); + + it('makeExecuteRequest builds a well-formed request', () => { + expect(makeExecuteRequest('index.qmd', 'req-7', 'actor-9')).toEqual({ + kind: 'exec/request', + path: 'index.qmd', + requestId: 'req-7', + requesterActorId: 'actor-9', + }); + }); +}); + +describe('parseExecMessage', () => { + it('accepts a valid beacon and returns it typed', () => { + const raw = { kind: 'exec/beacon', actorId: 'a', engines: ['knitr'], generation: 0 }; + expect(parseExecMessage(raw)).toEqual(raw); + }); + + it('accepts a valid request and returns it typed', () => { + const raw = { kind: 'exec/request', path: 'p.qmd', requestId: 'r', requesterActorId: 'a' }; + expect(parseExecMessage(raw)).toEqual(raw); + }); + + it('rejects unknown / missing kind', () => { + expect(parseExecMessage({ kind: 'presence', x: 1 })).toBeNull(); + expect(parseExecMessage({ actorId: 'a' })).toBeNull(); + }); + + it('rejects a beacon with wrong field types', () => { + expect(parseExecMessage({ kind: 'exec/beacon', actorId: 'a', engines: 'knitr', generation: 0 })).toBeNull(); + expect(parseExecMessage({ kind: 'exec/beacon', actorId: 1, engines: [], generation: 0 })).toBeNull(); + expect(parseExecMessage({ kind: 'exec/beacon', actorId: 'a', engines: [1], generation: 0 })).toBeNull(); + }); + + it('rejects a request missing required fields', () => { + expect(parseExecMessage({ kind: 'exec/request', path: 'p.qmd' })).toBeNull(); + }); + + it('rejects non-objects', () => { + expect(parseExecMessage(null)).toBeNull(); + expect(parseExecMessage('exec/beacon')).toBeNull(); + expect(parseExecMessage(42)).toBeNull(); + }); +}); + +describe('applyBeacon', () => { + it('inserts a new executor with lastSeen set to now', () => { + const out = applyBeacon(new Map(), makeBeacon('a', ['knitr'], 0), 1000); + expect(out.get('a')).toEqual({ actorId: 'a', engines: ['knitr'], generation: 0, lastSeen: 1000 }); + }); + + it('refreshes lastSeen (and engines/generation) for an existing executor', () => { + const prev = new Map([ + ['a', { actorId: 'a', engines: ['knitr'], generation: 0, lastSeen: 1000 }], + ]); + const out = applyBeacon(prev, makeBeacon('a', ['knitr', 'jupyter'], 1), 5000); + expect(out.get('a')).toEqual({ actorId: 'a', engines: ['knitr', 'jupyter'], generation: 1, lastSeen: 5000 }); + // Does not mutate the input map. + expect(prev.get('a')!.lastSeen).toBe(1000); + }); +}); + +describe('pruneExecutors', () => { + const base = (): Map => + new Map([ + ['fresh', { actorId: 'fresh', engines: [], generation: 0, lastSeen: 10_000 }], + ['stale', { actorId: 'stale', engines: [], generation: 0, lastSeen: 1_000 }], + ]); + + it('drops executors older than the timeout and keeps fresh ones', () => { + // now = 12_000: fresh is 2_000ms old (<= 4_500, kept); stale is 11_000ms + // old (> 4_500, dropped). + const out = pruneExecutors(base(), 12_000); + expect(out.has('fresh')).toBe(true); + expect(out.has('stale')).toBe(false); + }); + + it('keeps an executor exactly at the timeout boundary (<= timeout)', () => { + const out = pruneExecutors(base(), 10_000 + BEACON_TIMEOUT_MS); + expect(out.has('fresh')).toBe(true); + }); +}); + +// ── Stub-responder service tests (Phase 2C) ───────────────────────────── + +/** A fake index DocHandle that records broadcasts and lets a test inject + * inbound ephemeral messages (simulating a remote executor). */ +function fakeHandle() { + const handlers = new Set<(p: DocHandleEphemeralMessagePayload) => void>(); + const broadcasts: unknown[] = []; + return { + handle: { + broadcast: (m: unknown) => broadcasts.push(m), + on: (_e: 'ephemeral-message', h: (p: DocHandleEphemeralMessagePayload) => void) => + handlers.add(h), + off: (_e: 'ephemeral-message', h: (p: DocHandleEphemeralMessagePayload) => void) => + handlers.delete(h), + }, + broadcasts, + /** Deliver a message as if a remote peer broadcast it. */ + inject: (message: unknown) => + handlers.forEach((h) => h({ message } as DocHandleEphemeralMessagePayload)), + handlerCount: () => handlers.size, + }; +} + +describe('createExecutionChannel (Phase 2C)', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('a remote executor beacon makes it appear, and stops listening on stop()', () => { + const fake = fakeHandle(); + let clock = 1000; + const onExecutorsChange = vi.fn(); + const ch = createExecutionChannel({ + getIndexHandle: () => fake.handle, + onExecutorsChange, + now: () => clock, + pruneIntervalMs: 1000, + }); + + ch.start(); + expect(fake.handlerCount()).toBe(1); + + fake.inject(makeBeacon('exec-1', ['knitr'], 0)); + expect(onExecutorsChange).toHaveBeenLastCalledWith([ + { actorId: 'exec-1', engines: ['knitr'], generation: 0, lastSeen: 1000 }, + ]); + expect(ch.getExecutors()).toHaveLength(1); + + ch.stop(); + expect(fake.handlerCount()).toBe(0); + expect(ch.getExecutors()).toHaveLength(0); + }); + + it('prunes an executor once its beacon goes stale (1.5x interval)', () => { + const fake = fakeHandle(); + let clock = 1000; + const onExecutorsChange = vi.fn(); + const ch = createExecutionChannel({ + getIndexHandle: () => fake.handle, + onExecutorsChange, + now: () => clock, + pruneIntervalMs: 1000, + }); + ch.start(); + fake.inject(makeBeacon('exec-1', ['knitr'], 0)); + expect(ch.getExecutors()).toHaveLength(1); + + // Advance the clock well past the beacon timeout, then let the prune + // timer fire. The executor should be dropped and a change emitted. + clock += BEACON_TIMEOUT_MS + 1; + onExecutorsChange.mockClear(); + vi.advanceTimersByTime(1000); + + expect(ch.getExecutors()).toHaveLength(0); + expect(onExecutorsChange).toHaveBeenLastCalledWith([]); + ch.stop(); + }); + + it('requestExecution broadcasts a well-formed exec/request and returns its id', () => { + const fake = fakeHandle(); + const ch = createExecutionChannel({ + getIndexHandle: () => fake.handle, + onExecutorsChange: vi.fn(), + selfActorId: 'me', + now: () => 5000, + generateRequestId: () => 'req-fixed', + }); + ch.start(); + + const id = ch.requestExecution('docs/index.qmd'); + expect(id).toBe('req-fixed'); + + const sent = fake.broadcasts.at(-1) as ExecRequestMessage; + expect(sent).toEqual({ + kind: 'exec/request', + path: 'docs/index.qmd', + requestId: 'req-fixed', + requesterActorId: 'me', + }); + // It must be a valid message by the shared parser (round-trip contract). + expect(parseExecMessage(sent)).toEqual(sent); + ch.stop(); + }); + + it("ignores the editor's own beacon (self actor)", () => { + const fake = fakeHandle(); + const onExecutorsChange = vi.fn(); + const ch = createExecutionChannel({ + getIndexHandle: () => fake.handle, + onExecutorsChange, + selfActorId: 'me', + now: () => 1000, + }); + ch.start(); + fake.inject(makeBeacon('me', ['knitr'], 0)); + expect(ch.getExecutors()).toHaveLength(0); + expect(onExecutorsChange).not.toHaveBeenCalled(); + ch.stop(); + }); + + it('requestExecution returns null when not connected (no index handle)', () => { + const ch = createExecutionChannel({ + getIndexHandle: () => null, + onExecutorsChange: vi.fn(), + }); + ch.start(); + expect(ch.requestExecution('doc.qmd')).toBeNull(); + }); +}); diff --git a/hub-client/src/services/executionChannel.ts b/hub-client/src/services/executionChannel.ts new file mode 100644 index 000000000..93b32b442 --- /dev/null +++ b/hub-client/src/services/executionChannel.ts @@ -0,0 +1,282 @@ +/** + * Execution channel (bd-sfet3264, Phase 2). + * + * The remote-execution-provider feature needs two ephemeral, project-scoped + * signals between the editor and a connected `q2` executor: + * + * - a **capability beacon** the executor re-broadcasts periodically + * ("I'm online and can run engines X, Y"), and + * - an **execute request** the editor sends ("please run this document now"). + * + * Both ride Automerge's ephemeral messaging on the **index** DocHandle + * (project-scoped), so a single channel reaches every peer regardless of the + * active file. Ephemeral messages are best-effort and not persisted — exactly + * right for liveness + "run now" nudges; durable status stays in the + * persisted `CaptureRef.state`/`staleness` sidecar (D2). + * + * This module is split into: + * - a cross-language **wire format** (the Rust executor mirrors it in + * Phase 4) + pure helpers (builders, parse, live-executor bookkeeping), + * all timer/handle-free and unit-tested in isolation; and + * - a small stateful **service** (`createExecutionChannel`) that wires the + * index handle's broadcast/subscribe to those helpers and prunes stale + * executors on a timer. + * + * Phase 2 builds only the editor side + a stub responder for tests. The real + * executor (which produces beacons and consumes requests) is Phase 4; claim / + * heartbeat / `--force` takeover (D5) also land then. + */ + +import type { DocHandle, DocHandleEphemeralMessagePayload } from '@automerge/automerge-repo'; + +/** How often the executor re-broadcasts its capability beacon. */ +export const BEACON_INTERVAL_MS = 3000; +/** + * Liveness window: an editor marks an executor offline if no beacon arrives + * within this long. Locked at 1.5x the interval (D2) — tolerates a late + * beacon and CRDT latency without flicker, but a genuinely disconnected + * executor disappears within ~1.5 intervals. + */ +export const BEACON_TIMEOUT_MS = BEACON_INTERVAL_MS * 1.5; + +/** Executor capability announcement (re-broadcast every `BEACON_INTERVAL_MS`). */ +export interface ExecBeaconMessage { + kind: 'exec/beacon'; + actorId: string; + engines: string[]; + /** + * Monotonic per-executor counter. Unused in Phase 2 (no claims); reserved + * for the Phase 4 `--force` takeover protocol (D5). + */ + generation: number; +} + +/** Editor → executor "run this document now" request. */ +export interface ExecRequestMessage { + kind: 'exec/request'; + path: string; + requestId: string; + requesterActorId: string; +} + +export type ExecMessage = ExecBeaconMessage | ExecRequestMessage; + +/** An executor the editor currently believes is online. */ +export interface LiveExecutor { + actorId: string; + engines: string[]; + generation: number; + /** Epoch-ms of the most recent beacon from this executor. */ + lastSeen: number; +} + +// ── Builders ──────────────────────────────────────────────────────────── + +export function makeBeacon( + actorId: string, + engines: string[], + generation: number, +): ExecBeaconMessage { + return { kind: 'exec/beacon', actorId, engines, generation }; +} + +export function makeExecuteRequest( + path: string, + requestId: string, + requesterActorId: string, +): ExecRequestMessage { + return { kind: 'exec/request', path, requestId, requesterActorId }; +} + +// ── Parsing / validation ──────────────────────────────────────────────── + +function isStringArray(v: unknown): v is string[] { + return Array.isArray(v) && v.every((x) => typeof x === 'string'); +} + +/** + * Validate and discriminate an untrusted ephemeral payload into a typed + * `ExecMessage`, or `null` if it isn't a well-formed execution message. The + * index handle may carry other ephemeral traffic in future, so anything that + * isn't an `exec/*` message we recognise is ignored. + */ +export function parseExecMessage(raw: unknown): ExecMessage | null { + if (!raw || typeof raw !== 'object') return null; + const m = raw as Record; + switch (m.kind) { + case 'exec/beacon': + if ( + typeof m.actorId === 'string' && + isStringArray(m.engines) && + typeof m.generation === 'number' + ) { + return { kind: 'exec/beacon', actorId: m.actorId, engines: m.engines, generation: m.generation }; + } + return null; + case 'exec/request': + if ( + typeof m.path === 'string' && + typeof m.requestId === 'string' && + typeof m.requesterActorId === 'string' + ) { + return { + kind: 'exec/request', + path: m.path, + requestId: m.requestId, + requesterActorId: m.requesterActorId, + }; + } + return null; + default: + return null; + } +} + +// ── Live-executor bookkeeping (pure) ──────────────────────────────────── + +/** + * Upsert the executor named by `beacon`, stamping `lastSeen = nowMs`. Returns + * a new map (does not mutate the input). + */ +export function applyBeacon( + executors: ReadonlyMap, + beacon: ExecBeaconMessage, + nowMs: number, +): Map { + const next = new Map(executors); + next.set(beacon.actorId, { + actorId: beacon.actorId, + engines: beacon.engines, + generation: beacon.generation, + lastSeen: nowMs, + }); + return next; +} + +/** + * Drop executors whose most recent beacon is older than `timeoutMs`. Returns + * a new map (does not mutate the input). An executor exactly at the boundary + * is kept (`nowMs - lastSeen <= timeoutMs`). + */ +export function pruneExecutors( + executors: ReadonlyMap, + nowMs: number, + timeoutMs: number = BEACON_TIMEOUT_MS, +): Map { + const next = new Map(); + for (const [id, ex] of executors) { + if (nowMs - ex.lastSeen <= timeoutMs) next.set(id, ex); + } + return next; +} + +// ── Stateful service ──────────────────────────────────────────────────── + +/** A DocHandle restricted to the ephemeral surface this channel needs. */ +export interface EphemeralHandle { + broadcast(message: unknown): void; + on(event: 'ephemeral-message', handler: (payload: DocHandleEphemeralMessagePayload) => void): void; + off(event: 'ephemeral-message', handler: (payload: DocHandleEphemeralMessagePayload) => void): void; +} + +export interface ExecutionChannelOptions { + /** Returns the index DocHandle to broadcast/subscribe on (null until connected). */ + getIndexHandle: () => EphemeralHandle | DocHandle | null; + /** Called whenever the set of live executors changes (added / refreshed / pruned). */ + onExecutorsChange: (executors: LiveExecutor[]) => void; + /** This client's actor id (so an editor that is also an executor can ignore self). Optional. */ + selfActorId?: string; + /** Injectable clock (ms). Defaults to `Date.now`. */ + now?: () => number; + /** Injectable id generator for request ids. Defaults to a random-ish token. */ + generateRequestId?: () => string; + /** Prune cadence (ms). Defaults to the beacon interval. */ + pruneIntervalMs?: number; +} + +export interface ExecutionChannel { + /** Begin listening for beacons + start the prune timer. */ + start(): void; + /** Stop listening and clear the prune timer. */ + stop(): void; + /** Broadcast an execute request for `path`. Returns the request id, or null if not connected. */ + requestExecution(path: string): string | null; + /** Current live executors (snapshot). */ + getExecutors(): LiveExecutor[]; +} + +export function createExecutionChannel(opts: ExecutionChannelOptions): ExecutionChannel { + const now = opts.now ?? (() => Date.now()); + const pruneIntervalMs = opts.pruneIntervalMs ?? BEACON_INTERVAL_MS; + const genId = + opts.generateRequestId ?? + (() => `req-${now().toString(36)}-${Math.floor(Math.random() * 1e9).toString(36)}`); + + let executors: Map = new Map(); + let handle: EphemeralHandle | null = null; + let pruneTimer: ReturnType | null = null; + let messageHandler: + | ((payload: DocHandleEphemeralMessagePayload) => void) + | null = null; + + function emit(): void { + opts.onExecutorsChange(Array.from(executors.values())); + } + + function handleMessage(payload: DocHandleEphemeralMessagePayload): void { + const msg = parseExecMessage(payload.message); + if (!msg) return; + if (msg.kind === 'exec/beacon') { + // An editor that is also an executor ignores its own beacon. + if (opts.selfActorId && msg.actorId === opts.selfActorId) return; + const before = executors.get(msg.actorId); + executors = applyBeacon(executors, msg, now()); + // Fire on a newly-seen executor or on any refresh — the editor wants the + // freshest engine list / liveness. (A pure lastSeen bump still matters + // because it pushes back the prune deadline.) + if (!before || before.generation !== msg.generation || before.engines.join() !== msg.engines.join()) { + emit(); + } + } + // exec/request is consumed by the executor (Phase 4), not the editor. + } + + function prune(): void { + const before = executors.size; + executors = pruneExecutors(executors, now()); + if (executors.size !== before) emit(); + } + + function start(): void { + if (handle) return; // already started + const h = opts.getIndexHandle(); + if (!h) return; + handle = h as EphemeralHandle; + messageHandler = handleMessage; + handle.on('ephemeral-message', messageHandler); + pruneTimer = setInterval(prune, pruneIntervalMs); + } + + function stop(): void { + if (handle && messageHandler) handle.off('ephemeral-message', messageHandler); + if (pruneTimer !== null) clearInterval(pruneTimer); + handle = null; + messageHandler = null; + pruneTimer = null; + executors = new Map(); + } + + function requestExecution(path: string): string | null { + const h = handle ?? (opts.getIndexHandle() as EphemeralHandle | null); + if (!h) return null; + const requestId = genId(); + h.broadcast(makeExecuteRequest(path, requestId, opts.selfActorId ?? '')); + return requestId; + } + + function getExecutors(): LiveExecutor[] { + return Array.from(executors.values()); + } + + return { start, stop, requestExecution, getExecutors }; +} diff --git a/interop-repro/README.md b/interop-repro/README.md new file mode 100644 index 000000000..bb48b5b2b --- /dev/null +++ b/interop-repro/README.md @@ -0,0 +1,57 @@ +# JS ↔ Rust automerge interop repro (bd-bm0vaetl) + +Minimal, self-contained harness (no quarto-hub / hub-client / feature code) that +pinned why `q2 provide-hub` read **0 files** from a project created in +hub-client. + +## The finding + +hub-client uses `@automerge/automerge` 3.x, which stores a plain string +map-value (`doc.files[path] = docId`) as an automerge **`Text` object**, not a +scalar string. quarto's Rust `IndexDocument::get_all_files()` read values with +`Value::to_str()`, which returns `None` for `Text` — so a hub-client-authored +`files` map read as empty and the provider materialized nothing. + +The document itself syncs **fine** JS→Rust (this was NOT a samod sync bug — an +early wrong theory). Proof, via `crates/quarto-hub-provider/tests/integration/sync_probe.rs::probe_root_keys`: + +``` +files=map{index.qmd=Some("Object(Text)")} value=Scalar(Int(42)) +``` + +Fix: `crates/quarto-hub/src/index.rs` reads `Str` **or** `Text` values. + +## Pieces + +- `js-peer/server.mjs` — a minimal JS `automerge-repo` sync server (a control: + removes samod-as-server from the equation). +- `js-peer/peer.mjs` — create/read a `{ value, files }` doc. +- `js-peer/create-project.mjs` — create a realistic project (index with a + `Text`-valued `files` map + a `.qmd` file doc), like hub-client. +- `js-peer/e2e-run.mjs` — full editor-side stand-in: create a project, broadcast + an `exec/request` (the Run button), and read back + decode the capture. +- `js-peer/dump.mjs` / `js-peer/decode-capture.mjs` — inspect the index / + decode a capture doc's engine output. +- Rust side: `sync_probe.rs` (ignored, network) + `relay_sync.rs` (self-contained). + +## Run the full end-to-end (JS project → Rust provider → executed output) + +Needs R (`engine: knitr`) or edit the qmd to `engine: jupyter` + Python. + +```bash +# 1. a local storage hub +cargo run --bin q2 -- hub --no-project --data-dir /tmp/hd --port 3055 & + +# 2. editor side: create a project + click "Run" (broadcast exec/request), read the capture +node interop-repro/js-peer/e2e-run.mjs ws://127.0.0.1:3055 & +# → prints INDEX_DOC_ID= + +# 3. the executor (any non-empty --token; the no-auth hub ignores it) +cargo run --bin q2 -- provide-hub --server ws://127.0.0.1:3055 --allow-all --token dev +``` + +Observed: the provider lists the file, runs knitr, writes the capture; the JS +side prints `ENGINE=knitr` and `STDOUT=1 2 3` (from `cat(1, 2, 3)`). + +> Node scripts must be run from the repo root (ESM resolves the repo's +> `node_modules`). diff --git a/interop-repro/js-peer/create-project.mjs b/interop-repro/js-peer/create-project.mjs new file mode 100644 index 000000000..a93d50871 --- /dev/null +++ b/interop-repro/js-peer/create-project.mjs @@ -0,0 +1,25 @@ +// Create a realistic Quarto project on a sync server, the way hub-client does: +// an index doc with files[path]=fileDocId, and a text file doc per file. +// node create-project.mjs +import { Repo } from '@automerge/automerge-repo'; +import { WebSocketClientAdapter } from '@automerge/automerge-repo-network-websocket'; + +const url = process.argv[2]; +const repo = new Repo({ network: [new WebSocketClientAdapter(url)], sharePolicy: async () => true }); + +// A qmd file document: { text: "" }. +const qmd = `---\ntitle: JS project\nengine: knitr\n---\n\n## Hello from a hub-client-style project\n\n\`\`\`{r}\ncat(1, 2, 3)\n\`\`\`\n`; +const fileHandle = repo.create(); +fileHandle.change((d) => { d.text = qmd; }); +await fileHandle.whenReady(); + +const idx = repo.create(); +idx.change((d) => { + d.files = { 'index.qmd': fileHandle.documentId }; // plain string -> automerge Text + d.identities = {}; + d.version = 2; +}); +await idx.whenReady(); +console.log('INDEX_DOC_ID=' + idx.documentId); +console.log('FILE_DOC_ID=' + fileHandle.documentId); +setInterval(() => {}, 1000); diff --git a/interop-repro/js-peer/decode-capture.mjs b/interop-repro/js-peer/decode-capture.mjs new file mode 100644 index 000000000..93f3766c5 --- /dev/null +++ b/interop-repro/js-peer/decode-capture.mjs @@ -0,0 +1,17 @@ +import { Repo } from '@automerge/automerge-repo'; +import { WebSocketClientAdapter } from '@automerge/automerge-repo-network-websocket'; +import zlib from 'node:zlib'; +const [url, capId] = process.argv.slice(2); +const repo = new Repo({ network: [new WebSocketClientAdapter(url)], sharePolicy: async () => true }); +const h = await repo.find(capId); +for (let i = 0; i < 20; i++) { + const content = h.doc()?.content; + if (content) { + const caps = JSON.parse(zlib.gunzipSync(Buffer.from(content)).toString()); + console.log('ENGINE=' + caps[0]?.engine_name); + console.log('RESULT=' + JSON.stringify(caps[0]?.result)); + break; + } + await new Promise(r => setTimeout(r, 500)); +} +process.exit(0); diff --git a/interop-repro/js-peer/dump.mjs b/interop-repro/js-peer/dump.mjs new file mode 100644 index 000000000..fb82160c5 --- /dev/null +++ b/interop-repro/js-peer/dump.mjs @@ -0,0 +1,12 @@ +import { Repo } from '@automerge/automerge-repo'; +import { WebSocketClientAdapter } from '@automerge/automerge-repo-network-websocket'; +const [url, docId] = process.argv.slice(2); +const repo = new Repo({ network: [new WebSocketClientAdapter(url)], sharePolicy: async () => true }); +const h = await repo.find(docId); +for (let i = 0; i < 12; i++) { + const d = h.doc(); + console.log(`[t=${i*500}ms] keys=${JSON.stringify(Object.keys(d||{}))} captures=${JSON.stringify(d?.captures||null)}`); + if (d?.captures && Object.keys(d.captures).length) break; + await new Promise(r => setTimeout(r, 500)); +} +process.exit(0); diff --git a/interop-repro/js-peer/e2e-run.mjs b/interop-repro/js-peer/e2e-run.mjs new file mode 100644 index 000000000..2adb95f44 --- /dev/null +++ b/interop-repro/js-peer/e2e-run.mjs @@ -0,0 +1,44 @@ +// Full hub-client-side e2e stand-in: create a project, broadcast an exec/request +// (like the Run button), and read back the capture the provider writes. +// node e2e-run.mjs +import { Repo } from '@automerge/automerge-repo'; +import { WebSocketClientAdapter } from '@automerge/automerge-repo-network-websocket'; +import zlib from 'node:zlib'; + +const url = process.argv[2]; +const repo = new Repo({ network: [new WebSocketClientAdapter(url)], sharePolicy: async () => true }); + +const qmd = `---\ntitle: JS project\nengine: knitr\n---\n\n\`\`\`{r}\ncat(1, 2, 3)\n\`\`\`\n`; +const fileHandle = repo.create(); +fileHandle.change((d) => { d.text = qmd; }); +await fileHandle.whenReady(); +const idx = repo.create(); +idx.change((d) => { d.files = { 'index.qmd': fileHandle.documentId }; d.identities = {}; d.version = 2; }); +await idx.whenReady(); +console.log('INDEX_DOC_ID=' + idx.documentId); + +// Broadcast an exec/request every 1s (ephemeral, like the Run button) and poll +// the captures sidecar for the provider's write-back. +let done = false; +const onChange = async ({ doc }) => { + const cap = doc?.captures?.['index.qmd']; + if (done || !(cap?.captureDocId && cap.state === 'idle')) return; + done = true; + console.log(`CAPTURE state=${cap.state} docId=${cap.captureDocId}`); + const capHandle = await repo.find(cap.captureDocId); + const content = capHandle.doc()?.content; + if (content) { + const json = JSON.parse(zlib.gunzipSync(Buffer.from(content)).toString()); + console.log('ENGINE=' + json[0]?.engine_name); + // The stdout of `cat(1,2,3)` appears in the captured markdown. + const md = json[0]?.result?.markdown ?? ''; + const m = md.match(/cell-output-stdout[\s\S]*?```\n([\s\S]*?)\n```/); + console.log('STDOUT=' + (m ? m[1].trim() : '(see result.markdown)')); + } + process.exit(0); +}; +idx.on('change', onChange); +const timer = setInterval(() => { + idx.broadcast({ kind: 'exec/request', path: 'index.qmd', requestId: 'r1', requesterActorId: 'js-e2e' }); +}, 1000); +setTimeout(() => { if (!done) { console.log('NO_CAPTURE_WITHIN_TIMEOUT'); process.exit(1); } clearInterval(timer); }, 60000); diff --git a/interop-repro/js-peer/peer.mjs b/interop-repro/js-peer/peer.mjs new file mode 100644 index 000000000..2df2be220 --- /dev/null +++ b/interop-repro/js-peer/peer.mjs @@ -0,0 +1,29 @@ +// Minimal JS automerge-repo peer: create or read a doc { value: 42 }. +// node peer.mjs create +// node peer.mjs read +import { Repo } from '@automerge/automerge-repo'; +import { WebSocketClientAdapter } from '@automerge/automerge-repo-network-websocket'; + +const [cmd, url, docId] = process.argv.slice(2); +const repo = new Repo({ network: [new WebSocketClientAdapter(url)], sharePolicy: async () => true }); + +if (cmd === 'create') { + const handle = repo.create(); + handle.change((d) => { d.value = 42; d.files = { 'index.qmd': 'x', 'about.qmd': 'y' }; }); + await handle.whenReady(); + console.log('DOC_ID=' + handle.documentId); + setInterval(() => {}, 1000); // stay alive so the doc stays served +} else if (cmd === 'read') { + const handle = await repo.find(docId); + for (let i = 0; i < 30; i++) { + const doc = handle.doc(); + const v = doc && doc.value; + console.log(`[JS read t=${i * 500}ms] value=${v === undefined ? 'none' : v}`); + if (v !== undefined) break; + await new Promise((r) => setTimeout(r, 500)); + } + process.exit(0); +} else { + console.error('usage: node peer.mjs create|read [docId]'); + process.exit(1); +} diff --git a/interop-repro/js-peer/server.mjs b/interop-repro/js-peer/server.mjs new file mode 100644 index 000000000..38b10011d --- /dev/null +++ b/interop-repro/js-peer/server.mjs @@ -0,0 +1,15 @@ +// Minimal JS automerge-repo sync server (control for the interop matrix). +import { WebSocketServer } from 'ws'; +import { Repo } from '@automerge/automerge-repo'; +import { NodeWSServerAdapter } from '@automerge/automerge-repo-network-websocket'; +import { NodeFSStorageAdapter } from '@automerge/automerge-repo-storage-nodefs'; + +const port = Number(process.env.PORT || 3044); +const wss = new WebSocketServer({ port }); +// eslint-disable-next-line no-unused-vars +const repo = new Repo({ + network: [new NodeWSServerAdapter(wss)], + storage: new NodeFSStorageAdapter(process.env.STORAGE || '/tmp/js-sync-storage'), + sharePolicy: async () => true, +}); +console.log('JS sync server listening on ws://127.0.0.1:' + port); diff --git a/ts-packages/preview-runtime/src/automergeSync.ts b/ts-packages/preview-runtime/src/automergeSync.ts index 7295e83f9..0e66acdd6 100644 --- a/ts-packages/preview-runtime/src/automergeSync.ts +++ b/ts-packages/preview-runtime/src/automergeSync.ts @@ -249,6 +249,18 @@ export function deleteFile(path: string): void { // VFS is updated via callback } +/** + * Clear the recorded engine capture for a document (D6 / bd-sfet3264). + * + * Removes the `CaptureRef` sidecar entry so the preview falls back to + * source-only rendering. Pure CRDT map-key delete — needs no executor + * and no server round-trip; the removal syncs to every peer and fires + * `onCapturesChange`. No-op when the path has no capture. + */ +export function clearCapture(path: string): void { + ensureClient().clearCapture(path); +} + /** * Rename a file in the project. */ @@ -295,6 +307,16 @@ export function getFileHandle(path: string) { return ensureClient().getFileHandle(path); } +/** + * Get the index DocHandle for project-scoped ephemeral messaging + * (bd-sfet3264). The execution beacon/request channel broadcasts here so a + * single channel reaches every peer regardless of the active file. Returns + * null before connect. + */ +export function getIndexHandle() { + return client?.getIndexHandle() ?? null; +} + /** * Get all current file paths that have handles. */ diff --git a/ts-packages/preview-runtime/src/wasmRenderer.ts b/ts-packages/preview-runtime/src/wasmRenderer.ts index 20e3ff0f6..f0f27179d 100644 --- a/ts-packages/preview-runtime/src/wasmRenderer.ts +++ b/ts-packages/preview-runtime/src/wasmRenderer.ts @@ -75,6 +75,10 @@ interface WasmModuleExtended { path: string, user_grammars?: unknown, attribution_json?: string, + // bd-sfet3264 (Phase 1A): optional gzipped-JSON `EngineCapture[]`. When + // present, the q2-preview pipeline's CaptureSpliceStage folds the recorded + // engine output into the AST — alongside attribution, not instead of it. + capture_gz_json?: Uint8Array, ) => Promise; get_builtin_template: (name: string) => string; get_project_choices: () => string; @@ -455,8 +459,14 @@ export async function renderQmd( export async function renderPageInProject( path: string, userGrammars?: unknown, + // bd-uy4uygha: optional gzipped-JSON `EngineCapture[]` from the project's + // capture sidecar. Forwarded to the capture-ready + // `renderPageInProjectWithAttribution`; the WASM HTML branch splices it so + // hub-client's default `format: html` preview shows executed output. Omit to + // render code cells as source. + captureGzJson?: Uint8Array, ): Promise { - return renderPageInProjectWithAttribution(path, userGrammars, null); + return renderPageInProjectWithAttribution(path, userGrammars, null, captureGzJson); } /** @@ -488,6 +498,11 @@ export async function renderPageInProjectWithAttribution( path: string, userGrammars: unknown, attributionJson: string | null, + // bd-sfet3264 (Phase 1A): optional gzipped-JSON `EngineCapture[]` from the + // project's capture sidecar. When provided, the q2-preview pipeline splices + // the recorded engine output into the AST alongside attribution. Omit (or + // `undefined`) to render code cells as source. + captureGzJson?: Uint8Array, ): Promise { const wasm = getWasm(); return JSON.parse( @@ -495,6 +510,7 @@ export async function renderPageInProjectWithAttribution( path, userGrammars, attributionJson ?? undefined, + captureGzJson, ), ); } @@ -969,6 +985,15 @@ export interface RenderToHtmlOptions { * Phase 4.5 of `claude-notes/plans/2026-04-21-syntax-highlighting-phase-4.md`. */ userGrammars?: UserGrammarDiscoveryContext; + + /** + * Optional gzipped-JSON `EngineCapture[]` from the project's capture sidecar + * (bd-uy4uygha). When present, the WASM HTML render splices the recorded + * engine output into the page, so hub-client's default `format: html` preview + * shows the output of a document executed by a connected `q2 provide-hub`. + * Absent → code cells render as source. + */ + captureGzJson?: Uint8Array; } /** @@ -1161,7 +1186,7 @@ async function renderToHtmlInner( try { await initWasm(); - const { documentPath, userGrammars } = options; + const { documentPath, userGrammars, captureGzJson } = options; // Resolve and register any user-defined tree-sitter grammars // before the render. Cache + bridge live at module scope so @@ -1179,6 +1204,7 @@ async function renderToHtmlInner( const result: RenderResponse = await renderPageInProject( documentPath, grammarsHandle, + captureGzJson, ); if (result.success) { diff --git a/ts-packages/quarto-hub-mcp/scripts/bundle.mjs b/ts-packages/quarto-hub-mcp/scripts/bundle.mjs index ab7ce73cf..9a565c70b 100644 --- a/ts-packages/quarto-hub-mcp/scripts/bundle.mjs +++ b/ts-packages/quarto-hub-mcp/scripts/bundle.mjs @@ -78,13 +78,14 @@ const automergeBase64Plugin = { rmSync(outDir, { recursive: true, force: true }); mkdirSync(outDir, { recursive: true }); -await esbuild.build({ - entryPoints: [join(pkgRoot, 'src/index.ts')], +// Shared esbuild options for every entry we bundle (the MCP server and the +// `q2 provide-hub` auth bridge). Both embed into the same dist-bundle/ that +// the q2 binary include_dir!s, so they must use identical bundling rules. +const sharedOptions = { bundle: true, platform: 'node', format: 'esm', target: NODE_TARGET, - outfile: join(outDir, 'index.mjs'), conditions: ['source'], external: ['@napi-rs/keyring'], // Minify to shrink both the standalone tarball and the copy embedded @@ -107,6 +108,21 @@ await esbuild.build({ }, plugins: [automergeBase64Plugin], logLevel: 'info', +}; + +// The MCP server (`q2 mcp`). +await esbuild.build({ + ...sharedOptions, + entryPoints: [join(pkgRoot, 'src/index.ts')], + outfile: join(outDir, 'index.mjs'), +}); + +// The `q2 provide-hub` auth bridge (bd-sfet3264): authenticates and streams +// Bearer tokens to stdout. Shares the auth/* modules with the server. +await esbuild.build({ + ...sharedOptions, + entryPoints: [join(pkgRoot, 'src/auth-stream.ts')], + outfile: join(outDir, 'auth-stream.mjs'), }); // --- ship the keyring addon as a mini node_modules --------------------- diff --git a/ts-packages/quarto-hub-mcp/src/auth-stream.ts b/ts-packages/quarto-hub-mcp/src/auth-stream.ts new file mode 100644 index 000000000..f6080807c --- /dev/null +++ b/ts-packages/quarto-hub-mcp/src/auth-stream.ts @@ -0,0 +1,99 @@ +/** + * Auth bridge entry for `q2 provide-hub` (bd-sfet3264, Phase 3C). + * + * Authenticates via the shared hub-mcp OAuth machinery (loopback + PKCE + + * keyring + refresh) and streams Bearer tokens — the OIDC `id_token`, the + * exact credential the hub validates — to **stdout** as newline-delimited + * JSON for the Rust `q2 provide-hub` parent. Logs and the interactive sign-in + * URL go to **stderr**; `{"type":"refresh"}` on **stdin** pulls a fresh token. + * + * The token-stream control flow lives in `./auth-stream/protocol.ts` (unit + * tested); this entry is the thin wiring to the real auth modules and process + * stdio. It deliberately does NOT connect to the hub — that's the Rust side's + * job (it dials with a BearerDialer fed by this stream). + */ + +import { createInterface } from 'node:readline'; + +import { AuthToolsState } from './auth/auth-tools.js'; +import { CredentialStore } from './auth/credential-store.js'; +import { + discoverAuthorizationServer, + loadOAuthConfigFromEnv, + resolveIssuer, +} from './auth/oauth-config.js'; +import { ReauthRequired, RefreshManager } from './auth/refresh-manager.js'; +import { runTokenStream, type OutFrame, type Token } from './auth-stream/protocol.js'; + +function emit(frame: OutFrame): void { + process.stdout.write(`${JSON.stringify(frame)}\n`); +} + +function log(msg: string): void { + process.stderr.write(`[provide-hub-auth] ${msg}\n`); +} + +async function main(): Promise { + const issuer = resolveIssuer(); + const authServer = () => discoverAuthorizationServer(issuer); + // Throws MissingOAuthConfigError (→ caught below) if the client id/secret + // env vars are absent. The Rust launcher injects them (bundled or user env). + const flowConfig = loadOAuthConfigFromEnv(); + + const store = new CredentialStore({ issuer, clientId: flowConfig.clientId }); + const refreshManager = new RefreshManager({ + authServer, + config: { clientId: flowConfig.clientId, clientSecret: flowConfig.clientSecret }, + store, + }); + const authTools = new AuthToolsState({ + credentialStore: store, + refreshManager, + // We always want the interactive sign-in path available; report + // requires-auth so handleAuthenticate never short-circuits to no-auth. + connectionManager: { lastObservedAuthMode: () => 'requires-auth' }, + flowConfig: { + clientId: flowConfig.clientId, + clientSecret: flowConfig.clientSecret, + issuer, + }, + authServer, + logger: log, + }); + + async function tokenFromStore(bearer: string): Promise { + const bundle = await store.read(); + return { bearer, expiresAt: bundle?.idTokenExpiresAt.toISOString() ?? '' }; + } + + // Initial token: use a cached/refreshed one if present, else sign in. + async function getToken(): Promise { + try { + return await tokenFromStore(await refreshManager.getValidIdToken()); + } catch (e) { + if (e instanceof ReauthRequired) { + log('sign-in required — opening browser (URL also printed to stderr)'); + const result = await authTools.handleAuthenticate(); + if (result.isError) { + throw new Error('interactive sign-in failed'); + } + return await tokenFromStore(await refreshManager.getValidIdToken()); + } + throw e; + } + } + + async function forceRefresh(): Promise { + return tokenFromStore(await refreshManager.forceRefresh()); + } + + // readline's Interface is an AsyncIterable of input lines. + const input = createInterface({ input: process.stdin }); + + await runTokenStream({ getToken, forceRefresh, input, emit }); +} + +main().catch((e) => { + emit({ type: 'error', message: e instanceof Error ? e.message : String(e) }); + process.exit(1); +}); diff --git a/ts-packages/quarto-hub-mcp/src/auth-stream/protocol.test.ts b/ts-packages/quarto-hub-mcp/src/auth-stream/protocol.test.ts new file mode 100644 index 000000000..4a18299c7 --- /dev/null +++ b/ts-packages/quarto-hub-mcp/src/auth-stream/protocol.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect, vi } from 'vitest'; +import { parseCommand, runTokenStream, type OutFrame, type Token } from './protocol.js'; + +/** Build an async iterable from a fixed list of lines. */ +async function* lines(...ls: string[]): AsyncIterable { + for (const l of ls) yield l; +} + +describe('parseCommand', () => { + it('recognizes a refresh command', () => { + expect(parseCommand('{"type":"refresh"}')).toEqual({ type: 'refresh' }); + expect(parseCommand(' {"type":"refresh"} \n')).toEqual({ type: 'refresh' }); + }); + + it('ignores blank, non-JSON, and unknown commands', () => { + expect(parseCommand('')).toBeNull(); + expect(parseCommand(' ')).toBeNull(); + expect(parseCommand('not json')).toBeNull(); + expect(parseCommand('{"type":"explode"}')).toBeNull(); + expect(parseCommand('42')).toBeNull(); + }); +}); + +describe('runTokenStream', () => { + const tok = (bearer: string): Token => ({ bearer, expiresAt: '2026-07-01T00:00:00.000Z' }); + + it('emits the initial token before any command', async () => { + const frames: OutFrame[] = []; + await runTokenStream({ + getToken: async () => tok('initial'), + forceRefresh: vi.fn(), + input: lines(), + emit: (f) => frames.push(f), + }); + expect(frames).toEqual([ + { type: 'token', bearer: 'initial', expiresAt: '2026-07-01T00:00:00.000Z' }, + ]); + }); + + it('forces a refresh and emits the new token on a refresh command', async () => { + const frames: OutFrame[] = []; + const forceRefresh = vi.fn(async () => tok('refreshed')); + await runTokenStream({ + getToken: async () => tok('initial'), + forceRefresh, + input: lines('{"type":"refresh"}'), + emit: (f) => frames.push(f), + }); + expect(forceRefresh).toHaveBeenCalledTimes(1); + expect(frames.map((f) => (f.type === 'token' ? f.bearer : f.type))).toEqual([ + 'initial', + 'refreshed', + ]); + }); + + it('ignores unrecognized input lines (no extra frames, no refresh)', async () => { + const frames: OutFrame[] = []; + const forceRefresh = vi.fn(async () => tok('refreshed')); + await runTokenStream({ + getToken: async () => tok('initial'), + forceRefresh, + input: lines('', 'garbage', '{"type":"other"}'), + emit: (f) => frames.push(f), + }); + expect(forceRefresh).not.toHaveBeenCalled(); + expect(frames).toHaveLength(1); + expect(frames[0]).toMatchObject({ type: 'token', bearer: 'initial' }); + }); + + it('emits an error frame and stops if the initial token cannot be obtained', async () => { + const frames: OutFrame[] = []; + const forceRefresh = vi.fn(); + await runTokenStream({ + getToken: async () => { + throw new Error('reauth required'); + }, + forceRefresh, + input: lines('{"type":"refresh"}'), + emit: (f) => frames.push(f), + }); + expect(frames).toEqual([{ type: 'error', message: 'reauth required' }]); + // Fatal: we never start servicing commands. + expect(forceRefresh).not.toHaveBeenCalled(); + }); + + it('reports a failed refresh but keeps the stream alive', async () => { + const frames: OutFrame[] = []; + let calls = 0; + const forceRefresh = vi.fn(async () => { + calls += 1; + if (calls === 1) throw new Error('network blip'); + return tok('recovered'); + }); + await runTokenStream({ + getToken: async () => tok('initial'), + forceRefresh, + input: lines('{"type":"refresh"}', '{"type":"refresh"}'), + emit: (f) => frames.push(f), + }); + expect(frames).toEqual([ + { type: 'token', bearer: 'initial', expiresAt: '2026-07-01T00:00:00.000Z' }, + { type: 'error', message: 'network blip' }, + { type: 'token', bearer: 'recovered', expiresAt: '2026-07-01T00:00:00.000Z' }, + ]); + }); +}); diff --git a/ts-packages/quarto-hub-mcp/src/auth-stream/protocol.ts b/ts-packages/quarto-hub-mcp/src/auth-stream/protocol.ts new file mode 100644 index 000000000..6e7269d24 --- /dev/null +++ b/ts-packages/quarto-hub-mcp/src/auth-stream/protocol.ts @@ -0,0 +1,91 @@ +/** + * Token-stream protocol for `q2 provide-hub`'s auth bridge (bd-sfet3264, + * Phase 3C). + * + * The Rust `q2 provide-hub` process spawns this helper as a child and reads + * Bearer tokens from its **stdout** (newline-delimited JSON), writing + * `{"type":"refresh"}` to its **stdin** to pull a fresh token before a + * reconnect. Logs and the interactive auth URL go to **stderr**. stdio pipes + * are identical on Windows/macOS/Linux, so this hand-off is cross-platform. + * + * This module is the transport-agnostic core: it is driven by abstract + * `getToken`/`forceRefresh` callbacks and an async line input, so it is fully + * unit-testable without real OAuth, a keyring, or process stdio. + */ + +/** A token the parent can use as `Authorization: Bearer `. */ +export interface Token { + bearer: string; + /** ISO-8601 expiry, so the parent can refresh ahead of time. */ + expiresAt: string; +} + +/** Outbound stdout frame. */ +export type OutFrame = + | { type: 'token'; bearer: string; expiresAt: string } + | { type: 'error'; message: string }; + +/** Inbound stdin command. */ +export type InCommand = { type: 'refresh' }; + +export interface TokenStreamDeps { + /** Get the current valid token (used for the initial emit). */ + getToken: () => Promise; + /** Force a fresh token (in response to a `refresh` command). */ + forceRefresh: () => Promise; + /** Inbound stdin lines (newline-delimited). */ + input: AsyncIterable; + /** Emit one outbound frame (the caller serializes it to a stdout line). */ + emit: (frame: OutFrame) => void; +} + +/** + * Parse one stdin line into a recognized command, or `null` for blank lines, + * non-JSON, or anything we don't understand (forward-compatible: unknown + * commands are ignored rather than fatal). + */ +export function parseCommand(line: string): InCommand | null { + const trimmed = line.trim(); + if (!trimmed) return null; + try { + const v: unknown = JSON.parse(trimmed); + if (v && typeof v === 'object' && (v as { type?: unknown }).type === 'refresh') { + return { type: 'refresh' }; + } + } catch { + // not JSON — ignore + } + return null; +} + +function errorMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +/** + * Emit the initial token, then service `refresh` commands until the input + * ends. A failure to obtain the *initial* token is fatal (we emit an error + * frame and return — the parent cannot authenticate). A failed *refresh* is + * reported as an error frame but the stream keeps running (the previous token + * may still be valid; the parent decides). + */ +export async function runTokenStream(deps: TokenStreamDeps): Promise { + try { + const t = await deps.getToken(); + deps.emit({ type: 'token', bearer: t.bearer, expiresAt: t.expiresAt }); + } catch (e) { + deps.emit({ type: 'error', message: errorMessage(e) }); + return; + } + + for await (const line of deps.input) { + const cmd = parseCommand(line); + if (cmd?.type !== 'refresh') continue; + try { + const t = await deps.forceRefresh(); + deps.emit({ type: 'token', bearer: t.bearer, expiresAt: t.expiresAt }); + } catch (e) { + deps.emit({ type: 'error', message: errorMessage(e) }); + } + } +} diff --git a/ts-packages/quarto-sync-client/src/client.test.ts b/ts-packages/quarto-sync-client/src/client.test.ts index 3c5484e8c..a593888c2 100644 --- a/ts-packages/quarto-sync-client/src/client.test.ts +++ b/ts-packages/quarto-sync-client/src/client.test.ts @@ -248,6 +248,78 @@ describe('createSyncClient captures (Phase C.3)', () => { }); }); +describe('createSyncClient getIndexHandle (bd-sfet3264 Phase 2A)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns null before connect and the index DocHandle after connect', async () => { + const indexDoc: IndexDocument = { files: {}, version: 2, identities: {} }; + const { handle } = createMockHandle(indexDoc); + installMockRepo(handle, handle); + + const client = createSyncClient(noopCallbacks()); + // Before connect there is no index handle to broadcast on. + expect(client.getIndexHandle()).toBeNull(); + + await client.connect('ws://localhost:9999', 'mock-doc-id', 'actor-1', 'Alice', '#FF0000'); + + // After connect the index handle is exposed (the ephemeral channel + // carrier for the execution beacon/request protocol). + expect(client.getIndexHandle()).toBe(handle); + }); +}); + +describe('createSyncClient clearCapture (D6 / bd-sfet3264)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('clearCapture removes the CaptureRef sidecar entry for the path, leaving siblings intact', async () => { + const indexDoc: IndexDocument = { + files: { 'index.qmd': 'doc1', 'other.qmd': 'doc2' }, + version: 2, + identities: {}, + captures: { + 'index.qmd': { captureDocId: 'cap-1', state: 'idle' }, + 'other.qmd': { captureDocId: 'cap-2', state: 'idle' }, + }, + }; + const { handle, getDoc } = createMockHandle(indexDoc); + installMockRepo(handle, handle); + + const client = createSyncClient(noopCallbacks()); + await client.connect('ws://localhost:9999', 'mock-doc-id', 'actor-1', 'Alice', '#FF0000'); + + client.clearCapture('index.qmd'); + + const doc = getDoc(); + expect(doc.captures?.['index.qmd']).toBeUndefined(); + // The sibling's capture must be untouched — clear is per-document. + expect(doc.captures?.['other.qmd']).toEqual({ captureDocId: 'cap-2', state: 'idle' }); + }); + + it('clearCapture is a no-op (no throw, no sibling change) when the path has no capture', async () => { + const indexDoc: IndexDocument = { + files: { 'index.qmd': 'doc1' }, + version: 2, + identities: {}, + captures: { + 'index.qmd': { captureDocId: 'cap-1', state: 'idle' }, + }, + }; + const { handle, getDoc } = createMockHandle(indexDoc); + installMockRepo(handle, handle); + + const client = createSyncClient(noopCallbacks()); + await client.connect('ws://localhost:9999', 'mock-doc-id', 'actor-1', 'Alice', '#FF0000'); + + expect(() => client.clearCapture('nonexistent.qmd')).not.toThrow(); + // The existing capture is left intact. + expect(getDoc().captures?.['index.qmd']).toEqual({ captureDocId: 'cap-1', state: 'idle' }); + }); +}); + // ─── bd-4uvv: getBinaryDocById prefix normalization ────────────────── // // samod's TS `repo.find()` rejects bare docIds with diff --git a/ts-packages/quarto-sync-client/src/client.ts b/ts-packages/quarto-sync-client/src/client.ts index 17cd49f84..6e600e044 100644 --- a/ts-packages/quarto-sync-client/src/client.ts +++ b/ts-packages/quarto-sync-client/src/client.ts @@ -1266,6 +1266,33 @@ export function createSyncClient(callbacks: SyncClientCallbacks, astOptions?: AS callbacks.onFileRemoved(path); } + /** + * Clear the recorded engine capture for `path` (D6 / bd-sfet3264). + * + * Removes the `CaptureRef` sidecar entry from the index document so + * the editor falls back to source-only rendering (the splice has + * nothing to apply). This is a pure CRDT map-key delete — it touches + * only the index doc, never the capture binary doc (samod has no + * document-delete API; the orphaned bytes are a separate server-GC + * concern). It needs no executor and no server round-trip; the + * removal syncs to every connected peer and fires `onCapturesChange` + * for all of them. + * + * No-op when `path` has no capture entry. Throws only if not + * connected (no index handle to mutate). + */ + function clearCapture(path: string): void { + if (!state.indexHandle) { + throw new Error('Not connected'); + } + const indexHandle = state.indexHandle; + indexHandle.change(doc => { + if (doc.captures && doc.captures[path] !== undefined) { + delete doc.captures[path]; + } + }); + } + /** * Rename a file. */ @@ -1338,6 +1365,18 @@ export function createSyncClient(callbacks: SyncClientCallbacks, astOptions?: AS return state.fileHandles.get(path) ?? null; } + /** + * Get the index DocHandle for project-scoped ephemeral messaging + * (bd-sfet3264). Unlike `getFileHandle` (per-file), the index handle is + * the natural carrier for the execution beacon/request protocol: every + * peer subscribes to the index doc, so a single ephemeral channel reaches + * all of them regardless of which file is active. Returns null before + * connect. + */ + function getIndexHandle(): DocHandle | null { + return state.indexHandle; + } + /** * Get all file paths. */ @@ -1588,8 +1627,10 @@ export function createSyncClient(callbacks: SyncClientCallbacks, astOptions?: AS createBinaryFile, deleteFile, renameFile, + clearCapture, isConnected, getFileHandle, + getIndexHandle, getFilePaths, getUnavailableFiles, createNewProject,