diff --git a/.env.example b/.env.example index d8d210d..baf98b9 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,8 @@ LEAPFLOW_LLM_API_KEY= LEAPFLOW_LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 LEAPFLOW_LLM_MODEL=qwen3.7-plus LEAPFLOW_LLM_MAX_RETRIES=3 +# Runtime context budget in tokens. Set to the usable limit for your provider/model. +LEAPFLOW_LLM_CONTEXT_LENGTH=256000 # ═══════════════════════════════════════════════════════════════════════ # Bridge / Host — communication between Python Brain and platform Host diff --git a/README.md b/README.md index c9b155c..88d8930 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ The `.env` file lives in your project root (or `~/.leapflow/.env` for global def | `LEAPFLOW_LLM_API_KEY` | **Yes** | — | Your LLM provider API key | | `LEAPFLOW_LLM_BASE_URL` | No | DashScope endpoint | OpenAI-compatible base URL | | `LEAPFLOW_LLM_MODEL` | No | `qwen3.7-plus` | Model identifier | +| `LEAPFLOW_LLM_CONTEXT_LENGTH` | No | `256000` | Runtime context budget shown in the TUI status bar | | `LEAPFLOW_MOCK_HOST` | No | `0` | Set `1` to use in-process mock (no execution backend) | | `LEAPFLOW_RECORDING_MODE` | No | `video` | `video` / `default` / `vision_only` | | `LEAPFLOW_LOG_LEVEL` | No | `INFO` | `DEBUG` / `INFO` / `WARNING` | @@ -171,6 +172,7 @@ The `.env` file lives in your project root (or `~/.leapflow/.env` for global def | `LEAPFLOW_LLM_BASE_URL` | DashScope endpoint | OpenAI-compatible base URL | | `LEAPFLOW_LLM_MODEL` | `qwen3.7-plus` | Model identifier | | `LEAPFLOW_LLM_MAX_RETRIES` | `3` | Retry attempts on transient LLM errors | +| `LEAPFLOW_LLM_CONTEXT_LENGTH` | `256000` | Runtime context budget in tokens; explicit config wins over static model hints | | **Platform** | | | | `LEAPFLOW_MOCK_HOST` | `0` | `1` to use in-process mock (no execution backend) | | **Storage** | | | @@ -240,13 +242,13 @@ The `.env` file lives in your project root (or `~/.leapflow/.env` for global def ## Quick Start — First Experience -### Step 1: Launch Interactive Mode +### Step 1: Launch the Interactive TUI ```bash -uv run leap --mock-host +leap ``` -> You'll see the LeapFlow banner, session info (model, platform, cwd), and a `❯` prompt — you're in the rich interactive TUI. +> You'll see the LeapFlow banner, session info (model, context budget, platform, cwd), and a `❯` prompt — you're in the rich interactive TUI. If you do not have a native execution backend available yet, use `leap --mock-host` for a safe first run. ### Step 2: Have a Conversation @@ -289,16 +291,35 @@ uv run leap skills show "skill-name" # Inspect a specific skill --- -## Usage Patterns +## Recommended Entry Point — Interactive TUI -### Interactive Mode — Conversational Agent +LeapFlow is designed to be used primarily through the **interactive terminal UI**. Start here for day-to-day work: chat, inspect status, trigger tools, resume sessions, and progressively teach or execute workflows from one conversational surface. ```bash -uv run leap # Enter REPL -uv run leap "summarize this PDF" # Single-turn (answer + exit) +leap # Recommended: launch the interactive TUI +leap --mock-host # Safe first run without native OS backend +leap "summarize this repo" # Single-turn prompt, then exit ``` -The REPL supports multi-turn conversation with tool use, memory, and real-time streaming. +Why we recommend the TUI first: + +- **One surface for the whole loop**: conversation, tool execution, skill learning, status, and session continuity. +- **Real-time transparency**: streaming output, tool progress, daemon status, model name, and context budget are visible while work is running. +- **Lower setup friction**: first-run config is generated automatically, and API key/context changes are hot-reloaded in active sessions. +- **Best default mental model**: use `leap` like an always-available agent console; reach for subcommands only when scripting or automating. + +### TUI Status Bar + +The bottom toolbar shows the active model and context usage, for example: + +```text +qwen3.7-plus │ 0/256K │ [░░░░░░░░░░] 0% +``` + +The max value comes from `LEAPFLOW_LLM_CONTEXT_LENGTH` — LeapFlow's runtime context budget. Configure it in `~/.leapflow/.env`, project `./.env`, `~/.leapflow/config.yaml`, or real environment variables. Explicit config always wins over static model capability hints. + +
+More commands — teaching, execution, skills, host, daemon ### Teach Mode — Learn from Demonstration @@ -323,18 +344,27 @@ uv run leap run --auto "routine task" # Skip confirmations Skills start at `STEP` tier (human confirms each action) and graduate to `AUTO` as confidence grows. -
-CLI Command Reference +### Command Reference | Command | Syntax | Description | |---------|--------|-------------| -| _(default)_ | `leap` | Launch interactive REPL with multi-turn conversation | +| _(default)_ | `leap` | Launch the interactive TUI with multi-turn conversation | | _(prompt)_ | `leap "question"` | Single-turn chat (answer + exit) | | `teach` | `leap teach [goal] [options]` | Record a demonstration and distill into a skill | | `run` | `leap run [prompt] [options]` | Execute a matched skill | | `skills` | `leap skills [action] [name]` | Manage the skill library | | `relearn` | `leap relearn ` | Re-run learning pipeline on a saved trajectory | | `host` | `leap host ` | Manage execution backend connection and diagnostics | +| `daemon` | `leap daemon ` | Manage the persistent leapd runtime | + +**`leap daemon` actions:** + +| Action | Description | +|--------|-------------| +| `status` | Show leapd PID, socket, runtime source, Python executable, model, context usage, config paths, and DB path | +| `start` | Start leapd for the active profile, or connect to the healthy running daemon | +| `stop` | Stop the running leapd process for the active profile | +| `restart` | Stop then start leapd so code/config changes take effect after reinstalling or upgrading LeapFlow | **Global Flags:** @@ -400,11 +430,31 @@ LeapFlow provides a rich interactive terminal experience built on [Rich](https:/ | **Persistent history** | Input history saved to `~/.leapflow/history` (Up/Down to navigate) | | **Command completion** | Tab-completion for all REPL commands | | **Multiline editing** | Alt+Enter inserts a newline for multi-line prompts | -| **Status bar** | Live bottom toolbar: mode, skills, platform, model, context usage %, turn elapsed | +| **Status bar** | Live bottom toolbar: mode, skills, platform, model, context usage, turn elapsed | | **Adaptive theming** | Automatic light/dark detection via `COLORFGBG` / `LEAPFLOW_TUI_THEME` | | **Session info** | Startup display showing model, platform status, cwd, and skill count | | **Mode indicators** | Prompt character changes with session mode (idle ❯ / recording ● / paused ⏸) | +The context maximum shown in the status bar is the active runtime budget from `LEAPFLOW_LLM_CONTEXT_LENGTH`. In daemon mode, the TUI synchronizes this value from the daemon runtime so multiple terminal clients show the same budget. + +### Daemon-backed TUI Lifecycle + +By default, `leap` uses `leapd`, a per-profile background runtime shared across terminal clients. Exiting the TUI closes the current client; before returning, LeapFlow asks whether to stop `leapd` and defaults to stopping it. Keep it running when you want another terminal to reuse the same runtime. + +After reinstalling or upgrading LeapFlow, restart the daemon so the background process loads the new code: + +```bash +leap daemon restart +``` + +Use diagnostics when the TUI appears stale: + +```bash +leap daemon status +``` + +`status` prints the daemon PID, socket, runtime source path, Python executable, model, context usage, config paths, and DB path. + ### Theme Configuration The TUI auto-detects your terminal background. Override with: @@ -428,6 +478,174 @@ All output flows through `LeapConsole`, ensuring consistent theming. All input f --- +## External Platform Integration (Gateway) + +LeapFlow can connect to external messaging platforms — **Feishu (Lark)**, **DingTalk**, **Telegram**, and more — turning any IM channel into a natural-language interface to the agent. Platforms are integrated through a declarative **manifest** system and configured via a conversational setup flow, with no source-code changes required. + +### Design Highlights + +| Aspect | Approach | +|--------|----------| +| **Config-as-Conversation** | Say *"connect to Feishu"* in the REPL. The agent walks you through credential setup in 1–2 turns — no config files to edit by hand. | +| **Declarative Manifests** | Each platform is defined by a YAML manifest (credentials, setup guide, adapter module). Add a new platform by dropping a `.yaml` file. | +| **Credential Security** | Secrets are encrypted at rest (Fernet AES-128-CBC), never appear in LLM context or logs, and can be overridden via environment variables (`LEAPFLOW__`). | +| **Lazy Loading** | Platform SDK dependencies are imported only when a platform is first connected, keeping CLI startup instant. | +| **Adapter Protocol** | Platform adapters implement a simple Python `Protocol` — `connect()`, `disconnect()`, `send()`, `on_message` callback — extensible via `PlatformAdapterMixin` for graceful degradation. | +| **Auto-Reconnect** | Previously configured platforms are automatically reconnected on startup. Connection state persists across sessions via `gateway.yaml`. | +| **Bidirectional** | Inbound: platform messages are processed through LLM with safe tool access. Outbound: the agent can proactively send messages via `gateway_send`. | +| **Independent Sessions** | Each external chat gets its own conversation history with a restricted tool set (read-only), isolated from the CLI session. | +| **Event-Driven** | Inbound messages are logged to episodic memory and emitted as typed events (`GatewayMessageReceived`, `GatewaySessionCreated`, `GatewaySessionEnded`) for downstream subscribers. | + +### Architecture + +``` + ┌──────────────────┐ + │ CLI Agent │ + │ (AgentEngine) │ + │ │ + │ gateway_send ──▶│──┐ + └──────────────────┘ │ + │ send_message() + ┌─────────────┐ ┌──────────────┐ ┌───▼─────────┐ + │ Platform │───▶│ Gateway │───▶│ Gateway │───▶ LLM + safe tools + │ Adapter │ │ Server │ │ Router │◀─── reply + │ (Protocol) │◀───│ (lifecycle) │◀───│ (per- │ + └─────────────┘ │ │ │ session) │ + send reply │ on_event ──▶│ └─────────────┘ + └──────┬───────┘ + │ + ┌────────▼────────┐ + │ Episodic Memory │ + │ (event logging) │ + └─────────────────┘ +``` + +`Context` is the sole integration point — gateway modules have no dependency on engine or CLI. + +### Quick Start + +``` +❯ connect to Telegram +# Agent shows setup steps, asks for Bot Token +❯ +# Agent validates, encrypts, connects — done. +``` + +Or use the `/gateway` slash command to check connection status: + +``` +❯ /gateway +┌── Gateway ─────────────────────┐ +│ Connected │ +│ ● Telegram (5m 32s) │ +│ Available │ +│ 飞书, DingTalk, Webhook │ +│ │ +│ Say "connect to " │ +│ to set up a new integration. │ +└────────────────────────────────┘ +``` + +### Adding a Custom Platform + +1. Create a YAML manifest in `~/.leapflow/profiles//gateway/manifests/`: + +```yaml +platform_id: my_platform +display_name: My Platform +category: im + +credentials: + - key: api_key + label: API Key + required: true + secret: true + +setup_guide: + summary_en: "Provide your API key to connect." + +adapter: + module: my_package.adapter + class: MyAdapter + dependencies: [my-sdk] +``` + +2. Implement the adapter: + +```python +from leapflow.gateway.protocol import PlatformAdapter + +class MyAdapter: + def __init__(self, api_key: str, **kwargs): ... + async def connect(self, *, is_reconnect: bool = False) -> None: ... + async def send(self, target, content) -> SendResult: ... + async def disconnect(self) -> None: ... +``` + +3. Say *"connect to my_platform"* in the REPL — the agent handles the rest. + +### Environment Variable Overrides + +For deployment environments (CI/CD, containers), set credentials as environment variables instead of interactive configuration: + +```bash +export LEAPFLOW_TELEGRAM_BOT_TOKEN=your_token_here +export LEAPFLOW_FEISHU_APP_ID=cli_xxx +export LEAPFLOW_FEISHU_APP_SECRET=xxx +``` + +Environment variables take precedence over values stored in `gateway.yaml`. + +--- + +## Safety & Approval + +LeapFlow enforces a **layered safety architecture** that balances autonomy with human oversight. The goal is minimal interruption — the agent asks for permission only when an action carries real consequences. + +### Multi-Layer Defense + +``` + ┌───────────────────────────────┐ + │ Hardline Block (always) │ rm -rf /, mkfs, fork bomb + ├───────────────────────────────┤ + │ Dangerous Detection │ sudo, chmod, kill -9 ... + │ → Approval Gate (prompt) │ [y]es / [a]lways / [n]o + ├───────────────────────────────┤ + │ Safe Path / Size Bypass │ .md, .json, < 500 chars + ├───────────────────────────────┤ + │ Output Redaction │ Secrets stripped from results + ├───────────────────────────────┤ + │ Untrusted Result Wrapping │ MCP/web tool output delimited + └───────────────────────────────┘ +``` + +### Approval System + +| Feature | Behavior | +|---------|----------| +| **Unified Gate** | A single `ApprovalGate` protocol handles shell commands, file writes, and outbound messages — swappable for TUI, Web UI, or CI modes. | +| **Session Memory** | Choose **[a]lways** once and the same category won't prompt again for the rest of the session. | +| **Per-Category Scoping** | Shell commands, file writes, and each gateway platform are tracked independently. | +| **Smart Approval** | When an auxiliary LLM is configured, low-risk commands (risk < 0.3) are auto-approved; medium/high-risk still prompt. | +| **Fail-Closed** | In non-interactive environments (pipes, CI), all dangerous actions are denied by default. | +| **Rich TUI Display** | Approval prompts render as styled panels in the terminal — not raw text — with full action detail. | +| **Gateway Send** | First outbound message to each platform requires explicit approval; subsequent sends are auto-approved for the session. | +| **Audit Trail** | Every approval decision (allow/deny/session) is logged with timestamp and category. | + +### What Gets Approved + +| Action | Default | Approval Needed? | +|--------|---------|-----------------| +| Safe shell commands (`ls`, `cat`, `git status`) | Auto-execute | No | +| Dangerous shell (`sudo`, `rm -r`, `kill -9`) | Prompt | Yes (once per session) | +| Destructive shell (`rm -rf /`, `mkfs`) | Always blocked | Cannot override | +| File write (`.md`, `.json`, small files) | Auto-approve | No | +| File write (large, non-safe extensions) | Prompt | Yes (once per session) | +| Gateway send (first message to platform) | Prompt | Yes (once per platform per session) | +| Gateway inbound (external messages) | Restricted tools | Only safe read-only tools | + +--- + ## Workflow Copilot (Preview) LeapFlow includes a **Workflow Copilot** that predicts your next action and offers proactive suggestions — like GitHub Copilot, but for any workflow on your computer. @@ -641,6 +859,7 @@ uv run pytest -k "test_world_model" -q # By keyword | `engine/` | Session orchestration and ReAct execution loop | | `memory/` | Three-tier event-driven memory (working → episodic → long-term) | | `platform/` | Platform adaptation layer — protocol abstraction for pluggable execution backends | +| `gateway/` | External platform integration — declarative manifests, credential vault, adapter lifecycle | | `hub/` | Multi-source skill hub (ModelScope, GitHub, local) |
@@ -665,6 +884,7 @@ uv run pytest -k "test_world_model" -q # By keyword | Tools | `src/leapflow/tools/` | registry, builtins | Built-in tool definitions for the ReAct loop | | CLI | `src/leapflow/cli/` | cli, commands/, banner | Argument parsing, subcommand dispatch, interactive REPL | | Storage | `src/leapflow/storage/` | duckdb, skill_library | DuckDB-backed persistent storage for skills, trajectories, audit | +| Gateway | `src/leapflow/gateway/` | server, manifest, protocol, credential_vault | External platform integration — manifest discovery, adapter lifecycle, credential encryption | | Execution Backend | external (pluggable) | — | OS interaction: screen capture, AX tree, input injection (default: `cua-driver` via MCP) |
@@ -678,6 +898,7 @@ uv run pytest -k "test_world_model" -q # By keyword | `PredictorLayer` | `copilot/types.py` | Prediction algorithm interface (predict, on_feedback, priority, timeout) | | `SignalChannel` | `copilot/types.py` | Dynamically registerable signal source (start, stop, subscribe) | | `HintRenderer` | `copilot/types.py` | Ghost-hint display abstraction (show, dismiss, is_visible) | +| `PlatformAdapter` | `gateway/protocol.py` | External platform adapter contract (connect, disconnect, send, on_message) | **Core Data Types:** diff --git a/src/leapflow/_env_template.py b/src/leapflow/_env_template.py index f324446..b9b9639 100644 --- a/src/leapflow/_env_template.py +++ b/src/leapflow/_env_template.py @@ -21,6 +21,8 @@ LEAPFLOW_LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 LEAPFLOW_LLM_MODEL=qwen3.7-plus LEAPFLOW_LLM_MAX_RETRIES=3 +# Runtime context budget in tokens. Set to the usable limit for your provider/model. +LEAPFLOW_LLM_CONTEXT_LENGTH=256000 # ═══════════════════════════════════════════════════════════════════════ # Platform — CuaDriver communication @@ -32,7 +34,8 @@ # ═══════════════════════════════════════════════════════════════════════ # Storage — persistent state (memory, trajectories, skill library) # ═══════════════════════════════════════════════════════════════════════ -LEAPFLOW_DUCKDB_PATH=~/.leapflow/memory.duckdb +# LEAPFLOW_DUCKDB_PATH=~/.leapflow/profiles/default/db/leap.duckdb +# LEAPFLOW_PROFILE=default # ═══════════════════════════════════════════════════════════════════════ # Memory Providers — storage and retrieval configuration @@ -46,7 +49,7 @@ # Logging & Audit # ═══════════════════════════════════════════════════════════════════════ LEAPFLOW_LOG_LEVEL=INFO -LEAPFLOW_AUDIT_LOG_PATH=~/.leapflow/audit.jsonl +LEAPFLOW_AUDIT_LOG_PATH=~/.leapflow/profiles/default/audit.jsonl # ═══════════════════════════════════════════════════════════════════════ # Learning & Execution — SessionController behavior @@ -96,7 +99,7 @@ # Max seconds per video segment file before auto-splitting. # LEAPFLOW_VIDEO_MAX_SEGMENT_S=600 # Directory for video segment cache. -# LEAPFLOW_VIDEO_CACHE_DIR=~/.leapflow/cache/video +# LEAPFLOW_VIDEO_CACHE_DIR=~/.leapflow/profiles/default/cache/video # ── Video Analysis ── # LEAPFLOW_VIDEO_VLM_URL_SCHEME=base64 @@ -128,9 +131,11 @@ # ═══════════════════════════════════════════════════════════════════════ # Visual Track — screenshot-based perception (default/vision_only modes) +# Disabled by default on first run. Enable only after configuring +# LEAPFLOW_LLM_API_KEY or LEAPFLOW_VLM_API_KEY. # ═══════════════════════════════════════════════════════════════════════ -# LEAPFLOW_VISUAL_TRACK_ENABLED=true -# LEAPFLOW_VISUAL_FRAME_CACHE_DIR=~/.leapflow/cache/frames +# LEAPFLOW_VISUAL_TRACK_ENABLED=false +# LEAPFLOW_VISUAL_FRAME_CACHE_DIR=~/.leapflow/profiles/default/cache/frames # LEAPFLOW_VLM_MODEL= # LEAPFLOW_VLM_API_KEY= # LEAPFLOW_VLM_BASE_URL= @@ -149,7 +154,7 @@ # Perceptual Field — fine-grained per-context perception control # ═══════════════════════════════════════════════════════════════════════ # LEAPFLOW_PERCEPTUAL_FIELD_ENABLED=false -# LEAPFLOW_PERCEPTUAL_FIELD_CONFIG=~/.leapflow/perceptual_fields.yaml +# LEAPFLOW_PERCEPTUAL_FIELD_CONFIG=~/.leapflow/profiles/default/perceptual_fields.yaml # ═══════════════════════════════════════════════════════════════════════ # Context Learning Attention — signal/noise filtering during recording diff --git a/src/leapflow/cli/banner.py b/src/leapflow/cli/banner.py index f86c51e..87959a2 100644 --- a/src/leapflow/cli/banner.py +++ b/src/leapflow/cli/banner.py @@ -15,6 +15,7 @@ import time from typing import Any, Dict, List, Optional, Sequence +from leapflow.cli.tui_app.theme import ResolvedTheme, Theme, detect_theme from leapflow.version import __version__ VERSION = __version__ @@ -38,6 +39,8 @@ "hub_push": "hub", "hub_pull": "hub", "hub_search": "hub", + "gateway_connect": "gateway", + "gateway_send": "gateway", } @@ -68,13 +71,38 @@ def _categorize_skills( # ── Rich banner (interactive REPL) ─────────────────────────────────── -# Warm gold/amber/bronze palette (inspired by Hermes, adapted for LeapFlow) -_GOLD = "#FFD700" -_AMBER = "#FFBF00" -_BRONZE = "#CD7F32" -_CREAM = "#FFF8DC" -_DIM_GOLD = "#B8860B" -_WARM_GRAY = "#8B8682" +_MAX_PANEL_WIDTH = 132 +_NARROW_WIDTH = 70 +_MEDIUM_WIDTH = 100 + + +class _BannerPalette: + def __init__(self, theme: Theme | ResolvedTheme) -> None: + self.accent = theme.accent + self.accent_dim = theme.accent_dim + self.text = theme.text + self.muted = theme.text_muted + self.border = theme.border + self.title = theme.panel_title + self.success = theme.success + + +def _trim(text: str, limit: int) -> str: + if limit <= 1: + return "…" + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _category_limit(width: int) -> int: + if width < _NARROW_WIDTH: + return 44 + if width < _MEDIUM_WIDTH: + return 56 + return 72 + + +def _render_width(term_width: int) -> int: + return max(40, min(term_width, _MAX_PANEL_WIDTH)) def display_rich_banner( @@ -87,34 +115,36 @@ def display_rich_banner( skills: Optional[Sequence[Any]] = None, context_length: int = 0, mcp_tools: int = 0, + gateway_connected: Sequence[str] = (), show_welcome: bool = True, + theme: Theme | ResolvedTheme | None = None, ) -> None: - """Print the full-width Rich banner panel with tools/skills catalog.""" + """Print the adaptive Rich banner panel with tools/skills catalog.""" try: from rich.console import Console from rich.panel import Panel from rich.table import Table - from rich.text import Text except ImportError: display_welcome() return - console = Console(highlight=False, soft_wrap=True) + palette = _BannerPalette(theme or detect_theme()) term_width = shutil.get_terminal_size().columns + render_width = _render_width(term_width) + console = Console(highlight=False, soft_wrap=True, width=render_width) # ── Left column: branding + session metadata ── left_lines: list[str] = [] - left_lines.append("") left_lines.append( - f"[bold {_GOLD}]L[/] [dim {_DIM_GOLD}].[/] " - f"[bold {_GOLD}]E[/] [dim {_DIM_GOLD}].[/] " - f"[bold {_GOLD}]A[/] [dim {_DIM_GOLD}].[/] " - f"[bold {_GOLD}]P[/]" + f"[bold {palette.accent}]L[/] [{palette.accent_dim}].[/] " + f"[bold {palette.accent}]E[/] [{palette.accent_dim}].[/] " + f"[bold {palette.accent}]A[/] [{palette.accent_dim}].[/] " + f"[bold {palette.accent}]P[/]" ) - left_lines.append(f"[{_CREAM}]Learning and Evolving from Actual Practice[/]") - left_lines.append(f"[bold {_AMBER}]ModelScope[/]") + left_lines.append(f"[{palette.text}]Learning and Evolving from Actual Practice[/]") + left_lines.append(f"[bold {palette.accent}]ModelScope[/]") if session_id: - left_lines.append(f"[{_WARM_GRAY}]Session: {session_id}[/]") + left_lines.append(f"[{palette.muted}]Session: {_trim(session_id, 24)}[/]") left_lines.append("") if model: @@ -125,47 +155,57 @@ def display_rich_banner( ctx_label = f"{context_length // 1000}K" else: ctx_label = str(context_length) - ctx_str = f" [dim {_DIM_GOLD}]({ctx_label} ctx)[/]" if context_length else "" - left_lines.append(f"[bold {_AMBER}]{model_short}[/]{ctx_str}") + ctx_str = f" [{palette.muted}]({ctx_label} ctx)[/]" if context_length else "" + left_lines.append(f"[bold {palette.accent}]{model_short}[/]{ctx_str}") if cwd: short_cwd = cwd.replace(os.path.expanduser("~"), "~") - left_lines.append(f"[{_WARM_GRAY}]{short_cwd}[/]") + left_lines.append(f"[{palette.muted}]{_trim(short_cwd, 40)}[/]") - left_lines.append("") left_content = "\n".join(left_lines) - # ── Right column: tools + skills catalog ── + # ── Right column: compact capability overview ── right_lines: list[str] = [] - if tool_defs: - tool_groups = _categorize_tools(tool_defs) - right_lines.append(f"[bold {_AMBER}]Available Tools[/]") - for cat, names in tool_groups.items(): - names_str = ", ".join(sorted(names)) - if len(names_str) > 52: - names_str = names_str[:49] + "…" - right_lines.append(f"[{_DIM_GOLD}]{cat}:[/] [{_CREAM}]{names_str}[/]") + tool_count = len(tool_defs) if tool_defs else 0 + skill_count = len(skills) if skills else 0 + if tool_count: + tool_groups = _categorize_tools(tool_defs) + categories = ", ".join( + f"{cat}({len(names)})" for cat, names in tool_groups.items() + ) + category_text = _trim(categories, _category_limit(render_width)) + right_lines.append( + f"[bold {palette.accent}]Tools[/] [{palette.text}]{tool_count} available[/]" + ) + right_lines.append(f"[{palette.accent_dim}]{category_text}[/]") if mcp_tools > 0: - right_lines.append(f"[{_DIM_GOLD}]mcp:[/] [{_CREAM}]{mcp_tools} platform tools[/]") + right_lines.append( + f"[{palette.accent_dim}]mcp:[/] [{palette.text}]{mcp_tools} platform tools[/]" + ) - if skills: - right_lines.append("") - right_lines.append(f"[bold {_AMBER}]Available Skills[/]") + if skill_count: skill_groups = _categorize_skills(skills) - for cat, names in skill_groups.items(): - names_str = ", ".join(sorted(names)) - if len(names_str) > 52: - names_str = names_str[:49] + "…" - right_lines.append(f"[{_DIM_GOLD}]{cat}:[/] [{_CREAM}]{names_str}[/]") + categories = ", ".join( + f"{cat}({len(names)})" for cat, names in skill_groups.items() + ) + category_text = _trim(categories, _category_limit(render_width)) + right_lines.append( + f"[bold {palette.accent}]Skills[/] [{palette.text}]{skill_count} available[/]" + ) + right_lines.append(f"[{palette.accent_dim}]{category_text}[/]") + + if gateway_connected: + right_lines.append("") + right_lines.append(f"[bold {palette.accent}]Gateway[/]") + names_str = _trim(", ".join(gateway_connected), 52) + right_lines.append(f"[{palette.success}]●[/] [{palette.text}]{names_str}[/]") if not tool_defs and not skills: - right_lines.append(f"[{_WARM_GRAY}]No tools or skills loaded[/]") + right_lines.append(f"[{palette.muted}]No tools or skills loaded[/]") # Summary line - tool_count = len(tool_defs) if tool_defs else 0 - skill_count = len(skills) if skills else 0 summary_parts = [] if tool_count: summary_parts.append(f"{tool_count} tools") @@ -173,39 +213,39 @@ def display_rich_banner( summary_parts.append(f"{skill_count} skills") if mcp_tools: summary_parts.append(f"{mcp_tools} mcp") - summary_parts.append("/help for commands") + if gateway_connected: + summary_parts.append(f"{len(gateway_connected)} gateway") + summary_parts.append("/help · /tools · /skills") right_lines.append("") - right_lines.append(f"[{_WARM_GRAY}]{' · '.join(summary_parts)}[/]") + right_lines.append(f"[{palette.muted}]{' · '.join(summary_parts)}[/]") right_content = "\n".join(right_lines) - # ── Assembly — full-width panel ── - if term_width < 70: + # ── Assembly — width-capped panel ── + version_label = f"LeapFlow v{VERSION}" + conn = f"[{palette.success}]●[/]" if platform_online else f"[{palette.muted}]○[/]" + if render_width < _NARROW_WIDTH: # Narrow: single-column compact combined = left_content + "\n" + right_content - version_label = f"LeapFlow v{VERSION}" - conn = "[green]●[/]" if platform_online else f"[{_WARM_GRAY}]○[/]" panel = Panel( combined, - title=f"[bold {_GOLD}]{version_label}[/] {conn}", - border_style=_BRONZE, - padding=(0, 2), + title=f"[{palette.title}]{version_label}[/] {conn}", + border_style=palette.border, + padding=(0, 1), expand=True, ) else: - layout = Table.grid(padding=(0, 3)) - left_min = max(38, term_width // 3) + layout = Table.grid(padding=(0, 2)) + left_min = max(28, min(44, render_width // 3)) layout.add_column("left", justify="center", min_width=left_min) layout.add_column("right", justify="left", ratio=1) layout.add_row(left_content, right_content) - version_label = f"LeapFlow v{VERSION}" - conn = "[green]●[/]" if platform_online else f"[{_WARM_GRAY}]○[/]" panel = Panel( layout, - title=f"[bold {_GOLD}]{version_label}[/] {conn}", - border_style=_BRONZE, - padding=(0, 2), + title=f"[{palette.title}]{version_label}[/] {conn}", + border_style=palette.border, + padding=(0, 1), expand=True, ) @@ -214,8 +254,8 @@ def display_rich_banner( if show_welcome: console.print( - f"\n[{_CREAM}]Welcome to LeapFlow! " - f"Type your message or [bold {_AMBER}]/help[/bold {_AMBER}] for commands.[/]\n", + f"\n[{palette.text}]Welcome to LeapFlow! " + f"Type your message or [bold {palette.accent}]/help[/bold {palette.accent}] for commands.[/]\n", ) diff --git a/src/leapflow/cli/cli.py b/src/leapflow/cli/cli.py index 2932335..720d7d2 100644 --- a/src/leapflow/cli/cli.py +++ b/src/leapflow/cli/cli.py @@ -12,6 +12,7 @@ import argparse import asyncio +import os import sys try: @@ -38,7 +39,7 @@ async def _async_main(args: argparse.Namespace) -> int: if cmd == "interactive": from leapflow.cli.commands.interactive import cmd_interactive - return await cmd_interactive(ctx) + return await cmd_interactive(ctx, resume_id=getattr(args, "resume", None)) elif cmd == "chat": from leapflow.cli.commands.chat import cmd_chat return await cmd_chat(ctx, args.prompt, getattr(args, "thinking", False)) @@ -86,6 +87,56 @@ async def _async_host(args: argparse.Namespace) -> int: return await cmd_host(args) +async def _async_daemon_main(args: argparse.Namespace) -> int: + """Run chat/interactive through a shared leapd daemon.""" + from leapflow.daemon.client import DaemonUnavailableError, ensure_daemon_client + + settings = load_config() + mock_host = getattr(args, "mock_host", False) + + def _status(message: str) -> None: + sys.stderr.write(f"\033[2m→ {message}\033[0m\n") + sys.stderr.flush() + + try: + client = await ensure_daemon_client( + settings, + mock_host=mock_host, + status_callback=_status, + ) + except DaemonUnavailableError as exc: + sys.stderr.write( + "\033[33m→ leapd unavailable; falling back to local volatile-capable mode.\033[0m\n" + ) + sys.stderr.write(f"\033[2m {exc}\033[0m\n") + sys.stderr.flush() + return await _async_main(args) + + cmd = args.command + if cmd == "interactive": + from leapflow.cli.commands.interactive import cmd_interactive_daemon + + return await cmd_interactive_daemon( + client, + settings, + resume_id=getattr(args, "resume", None), + ) + if cmd == "chat": + from leapflow.cli.commands.chat import cmd_chat_daemon + + return await cmd_chat_daemon(client, args.prompt, getattr(args, "thinking", False)) + + return await _async_main(args) + + +def _daemon_enabled(args: argparse.Namespace) -> bool: + """Return whether chat/interactive should use leapd.""" + if getattr(args, "no_daemon", False): + return False + raw = os.getenv("LEAPFLOW_DAEMON", "1").strip().lower() + return raw not in {"0", "false", "no", "off"} + + def main(argv: list[str] | None = None) -> int: common = argparse.ArgumentParser(add_help=False) common.add_argument( @@ -93,6 +144,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Force in-process mock platform (overrides LEAPFLOW_MOCK_HOST).", ) + common.add_argument( + "--no-daemon", + action="store_true", + help="Run chat/interactive in the legacy in-process mode.", + ) parser = argparse.ArgumentParser( prog="leap", @@ -101,6 +157,7 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--thinking", action="store_true", help="Enable LLM reasoning mode") + parser.add_argument("--resume", metavar="ID", help="Resume a previous chat session") subparsers = parser.add_subparsers(dest="command") @@ -148,24 +205,56 @@ def main(argv: list[str] | None = None) -> int: host_sub.add_parser("doctor", help="Run cua-driver connectivity health check") host_sub.add_parser("install", help="Install cua-driver") + # leap daemon + daemon_parser = subparsers.add_parser("daemon", help="Manage leapd daemon") + daemon_sub = daemon_parser.add_subparsers(dest="daemon_action") + daemon_sub.add_parser("status", help="Show daemon status") + daemon_sub.add_parser("start", help="Start daemon for the active profile") + daemon_sub.add_parser("stop", help="Stop running daemon") + daemon_sub.add_parser("restart", help="Restart daemon for the active profile") + serve_parser = daemon_sub.add_parser("serve", help=argparse.SUPPRESS) + serve_parser.add_argument("--internal", action="store_true", help=argparse.SUPPRESS) + # ── Pre-parse: detect if first non-flag arg is a known subcommand ── # If not, treat everything non-flag as a chat prompt. - known_commands = {"teach", "run", "skills", "relearn", "host"} + known_commands = {"teach", "run", "skills", "relearn", "host", "daemon"} effective_argv = list(argv) if argv is not None else sys.argv[1:] - # Find first non-flag argument + # Find first non-flag argument, skipping values owned by global options. + value_options = {"--resume"} prompt_words: list[str] = [] first_pos = None + skip_next = False for i, tok in enumerate(effective_argv): + if skip_next: + skip_next = False + continue + if tok in value_options: + skip_next = True + continue if tok.startswith("-"): continue first_pos = i break if first_pos is not None and effective_argv[first_pos] not in known_commands: - # Collect all non-flag tokens as prompt, remove them from argv for argparse - flags = [t for t in effective_argv if t.startswith("-")] - prompt_words = [t for t in effective_argv if not t.startswith("-")] + # Collect all non-option prompt tokens while preserving global option values. + flags: list[str] = [] + prompt_words = [] + skip_next = False + for tok in effective_argv: + if skip_next: + flags.append(tok) + skip_next = False + continue + if tok in value_options: + flags.append(tok) + skip_next = True + continue + if tok.startswith("-"): + flags.append(tok) + else: + prompt_words.append(tok) effective_argv = flags args = parser.parse_args(effective_argv) @@ -195,7 +284,14 @@ def main(argv: list[str] | None = None) -> int: sys.stderr.write("\n\033[2m→ Interrupted\033[0m\n") return 130 + # Daemon command does not need Context initialization + if args.command == "daemon": + from leapflow.cli.commands.daemon import cmd_daemon + return cmd_daemon(args) + try: + if args.command in {"interactive", "chat"} and _daemon_enabled(args): + return asyncio.run(_async_daemon_main(args)) return asyncio.run(_async_main(args)) except KeyboardInterrupt: sys.stderr.write("\n\033[2m→ Interrupted\033[0m\n") diff --git a/src/leapflow/cli/commands/chat.py b/src/leapflow/cli/commands/chat.py index 8115116..f61b470 100644 --- a/src/leapflow/cli/commands/chat.py +++ b/src/leapflow/cli/commands/chat.py @@ -2,18 +2,17 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, AsyncIterator from leapflow.cli.helpers import require_initialized -from leapflow.engine import StreamEvent if TYPE_CHECKING: from leapflow.cli.context import Context + from leapflow.daemon.client import DaemonClient -async def cmd_chat(ctx: "Context", prompt: str, thinking: bool) -> int: - require_initialized(ctx) - +async def render_chat_stream(events: AsyncIterator[object]) -> int: + """Render a stream of engine events to the terminal.""" from leapflow.cli.tui_app import detect_theme, LeapConsole, StreamRenderer theme = detect_theme() @@ -23,7 +22,7 @@ async def cmd_chat(ctx: "Context", prompt: str, thinking: bool) -> int: renderer.start() try: - async for event in ctx.engine.run_stream(prompt, enable_thinking=thinking): + async for event in events: if isinstance(event, str): renderer.feed(event) elif event.type == "chunk": @@ -36,7 +35,23 @@ async def cmd_chat(ctx: "Context", prompt: str, thinking: bool) -> int: renderer.tool_finished(event.content) elif event.type == "final" and not renderer.text: renderer.feed(event.content) + elif event.type == "error": + renderer.feed(event.content) finally: renderer.finish() return 0 + + +async def cmd_chat_daemon(client: "DaemonClient", prompt: str, thinking: bool) -> int: + """Single-turn conversational mode backed by leapd.""" + return await render_chat_stream( + client.engine_chat(prompt, enable_thinking=thinking) + ) + + +async def cmd_chat(ctx: "Context", prompt: str, thinking: bool) -> int: + require_initialized(ctx) + return await render_chat_stream( + ctx.engine.run_stream(prompt, enable_thinking=thinking) + ) diff --git a/src/leapflow/cli/commands/daemon.py b/src/leapflow/cli/commands/daemon.py new file mode 100644 index 0000000..a1c4070 --- /dev/null +++ b/src/leapflow/cli/commands/daemon.py @@ -0,0 +1,166 @@ +"""CLI commands for leapd daemon management. + +``leap daemon status`` — show whether leapd is running +``leap daemon start`` — start leapd for the active profile +``leap daemon stop`` — send SIGTERM to a running leapd +``leap daemon restart`` — restart leapd so code/config changes take effect +""" +from __future__ import annotations + +import asyncio +import signal +import sys +import time +from argparse import Namespace +from pathlib import Path + + +def cmd_daemon(args: Namespace) -> int: + """Route daemon subcommands.""" + from leapflow.config import load_config + + settings = load_config() + run_dir = settings.profile_dir / "run" + + action = getattr(args, "daemon_action", None) or "status" + + if action == "status": + return _status(run_dir) + if action == "start": + return _start(settings, getattr(args, "mock_host", False)) + if action == "stop": + return _stop(run_dir) + if action == "restart": + return _restart(settings, getattr(args, "mock_host", False)) + if action == "serve": + if not getattr(args, "internal", False): + sys.stderr.write("'leap daemon serve' is an internal command. Use 'leap daemon start'.\n") + return 2 + return asyncio.run(_serve(settings, getattr(args, "mock_host", False))) + + sys.stderr.write(f"Unknown daemon action: {action}\n") + return 1 + + +def _status(run_dir: Path) -> int: + from leapflow.daemon.lifecycle import DaemonInfo + + info = DaemonInfo.discover(run_dir) + print(info.format_status()) + if info.sock_path is not None: + print(f"socket: {info.sock_path}") + if info.is_healthy and info.sock_path is not None: + try: + details = asyncio.run(_runtime_status(info.sock_path)) + except Exception as exc: + sys.stderr.write(f"Could not fetch daemon runtime details: {exc}\n") + else: + _print_runtime_status(details) + return 0 if info.is_healthy else 1 + + +async def _runtime_status(sock_path: Path) -> dict: + from leapflow.daemon.client import DaemonClient + + return await DaemonClient(sock_path).status() + + +def _print_runtime_status(status: dict) -> None: + print( + "runtime: " + f"profile={status.get('profile')} " + f"clients={status.get('active_clients')} " + f"volatile={status.get('volatile')}" + ) + print( + "model: " + f"{status.get('model')} " + f"context={status.get('context_used', 0)}/{status.get('llm_context_length', 0)}" + ) + if status.get("session_id"): + print(f"session: {status['session_id']}") + if status.get("runtime_version"): + print(f"version: {status['runtime_version']}") + if status.get("runtime_source"): + print(f"source: {status['runtime_source']}") + if status.get("runtime_executable"): + print(f"python: {status['runtime_executable']}") + if status.get("config_path"): + print(f"config: {status['config_path']}") + if status.get("project_env_path"): + print(f"project_env: {status['project_env_path']}") + if status.get("db_path"): + print(f"db: {status['db_path']}") + + +def _start(settings: object, mock_host: bool) -> int: + from leapflow.daemon.client import DaemonUnavailableError, ensure_daemon_client + + async def _run() -> int: + try: + client = await ensure_daemon_client( + settings, + mock_host=mock_host, + status_callback=lambda msg: print(f"→ {msg}"), + ) + status = await client.status() + except DaemonUnavailableError as exc: + sys.stderr.write(f"Failed to start leapd: {exc}\n") + return 1 + print( + "leapd ready " + f"(pid={status.get('pid')}, profile={status.get('profile')}, " + f"clients={status.get('active_clients')})" + ) + return 0 + + return asyncio.run(_run()) + + +def _stop(run_dir: Path) -> int: + from leapflow.daemon.lifecycle import DaemonInfo, send_signal, cleanup_stale + + info = DaemonInfo.discover(run_dir) + if not info.is_running: + if info.pid is not None: + cleanup_stale(run_dir) + print("Cleaned up stale daemon files.") + else: + print("leapd is not running.") + return 0 + + if send_signal(run_dir, signal.SIGTERM): + print(f"Sent SIGTERM to leapd (pid={info.pid}).") + return 0 + + sys.stderr.write("Failed to stop leapd.\n") + return 1 + + +def _restart(settings: object, mock_host: bool) -> int: + run_dir = settings.profile_dir / "run" + print("Restarting leapd...") + stop_code = _stop(run_dir) + if stop_code != 0: + return stop_code + if not _wait_stopped(run_dir): + sys.stderr.write("Timed out waiting for leapd to stop.\n") + return 1 + return _start(settings, mock_host) + + +def _wait_stopped(run_dir: Path, *, timeout_s: float = 10.0) -> bool: + from leapflow.daemon.lifecycle import DaemonInfo + + deadline = time.time() + timeout_s + while time.time() < deadline: + if not DaemonInfo.discover(run_dir).is_running: + return True + time.sleep(0.1) + return not DaemonInfo.discover(run_dir).is_running + + +async def _serve(settings: object, mock_host: bool) -> int: + from leapflow.daemon.server import serve_daemon + + return await serve_daemon(settings, mock_host=mock_host) diff --git a/src/leapflow/cli/commands/interactive.py b/src/leapflow/cli/commands/interactive.py index fad8e70..5119594 100644 --- a/src/leapflow/cli/commands/interactive.py +++ b/src/leapflow/cli/commands/interactive.py @@ -8,17 +8,23 @@ from __future__ import annotations import asyncio +import logging import os +import signal +import sys import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Optional -from leapflow.cli.helpers import require_initialized from leapflow.cli.commands.run import _print_execution_result +from leapflow.cli.helpers import require_initialized from leapflow.engine import StreamEvent +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from leapflow.cli.context import Context from leapflow.copilot.types import PredictionCandidate + from leapflow.daemon.client import DaemonClient _CLI_INTERACTION_EVENT = "cli.interaction" _CLI_EVENT_SOURCE = "interactive_repl" @@ -26,11 +32,62 @@ _last_hint: Optional["PredictionCandidate"] = None -async def cmd_interactive(ctx: "Context") -> int: +async def _prompt_stop_daemon_on_exit( + client: "DaemonClient", + settings: Any, + console: Any, +) -> None: + """Ask whether to stop leapd after a daemon-backed TUI exits.""" + try: + daemon_status = await client.status() + except Exception: + daemon_status = {} + pid = daemon_status.get("pid") or "unknown" + console.system( + "leapd runs in the background; stop/restart it after reinstalling LeapFlow " + "to load new code." + ) + stop = await _ask_yes_no_default_yes(f"Stop leapd now (pid={pid})? [Y/n]: ") + if not stop: + console.system("leapd kept running. Use `leap daemon restart` when needed.") + return + + from leapflow.daemon.lifecycle import send_signal + + run_dir = settings.profile_dir / "run" + if send_signal(run_dir, signal.SIGTERM): + console.system(f"Sent SIGTERM to leapd (pid={pid}).") + else: + console.warning("Could not stop leapd; run `leap daemon stop` manually.") + + +async def _ask_yes_no_default_yes(prompt: str) -> bool: + """Return True by default, including non-interactive or interrupted prompts.""" + if not sys.stdin.isatty(): + return True + try: + answer = await asyncio.get_running_loop().run_in_executor( + None, + lambda: input(prompt).strip().lower(), + ) + except (EOFError, KeyboardInterrupt): + return True + return answer not in {"n", "no"} + + +async def cmd_interactive(ctx: "Context", *, resume_id: Optional[str] = None) -> int: """Persistent REPL session with hybrid TUI (Application + Rich).""" require_initialized(ctx) - from leapflow.cli.tui_app import detect_theme, LeapConsole, StreamRenderer, LeapApp + from leapflow.cli.tui_app import ( + LeapApp, + LeapConsole, + SessionExitStats, + StreamRenderer, + build_exit_summary_lines, + detect_theme, + summarize_messages, + ) from leapflow.cli.tui_app.status import StatusBar from leapflow.cli.banner import display_rich_banner from leapflow.cli.commands.registry import completion_entries, resolve_command @@ -40,6 +97,7 @@ async def cmd_interactive(ctx: "Context") -> int: handle_usage, handle_model, handle_clear, + handle_gateway, ) from leapflow.utils.terminal_io import TerminalIOProvider from leapflow.engine.session import SessionMode @@ -49,6 +107,9 @@ async def cmd_interactive(ctx: "Context") -> int: console = LeapConsole(theme) status = StatusBar(theme) io = TerminalIOProvider() + exit_stats = SessionExitStats() + active_resume_id = "" + storage_volatile = bool(getattr(ctx, "storage_volatile", False)) # ── Session callbacks ──────────────────────────────────────────── @@ -100,10 +161,6 @@ def _update_status() -> None: engine = ctx.engine if engine is not None: ctx_used = getattr(engine, "context_token_count", 0) - cap_registry = getattr(engine, "model_capabilities", None) - if cap_registry is not None: - caps = cap_registry.resolve(ctx.settings.llm_model) - ctx_max = caps.context_length mode = _mode_name() status.update( mode=mode, @@ -124,12 +181,19 @@ def _update_status() -> None: if hasattr(ctx, "platform_tools") else 0 ) - cap_registry = getattr(ctx.engine, "model_capabilities", None) if ctx.engine else None - ctx_len = ( - cap_registry.resolve(ctx.settings.llm_model).context_length - if cap_registry is not None - else ctx.settings.llm_context_length - ) + ctx_len = ctx.settings.llm_context_length + + def _gateway_connected_names() -> list[str]: + gw = getattr(ctx, "gateway_server", None) + if gw is None: + return [] + return [ + (gw.manifests[s.platform_id].display_name + if s.platform_id in gw.manifests + else s.platform_id) + for s in gw.platform_status() + if s.connected + ] def _render_banner() -> None: display_rich_banner( @@ -141,11 +205,64 @@ def _render_banner() -> None: skills=all_skills, context_length=ctx_len, mcp_tools=mcp_count, + gateway_connected=_gateway_connected_names(), + theme=theme, ) + if storage_volatile: + console.warning( + "Primary database is locked by another LeapFlow instance; " + "this window is using volatile storage." + ) + console.system("New memory, session history, and learned skills will not persist here.") + + def _active_chat_session_id() -> str: + engine = getattr(ctx, "engine", None) + current = getattr(engine, "_current_session_id", "") if engine else "" + return str(current or active_resume_id or "") + + def _stored_message_counts(session_id: str) -> tuple[int, int, int] | None: + if not session_id: + return None + store = getattr(ctx, "_conversation_store", None) + if store is None: + engine = getattr(ctx, "engine", None) + store = getattr(engine, "_conversation_store", None) if engine else None + if store is None: + return None + try: + messages = store.get_messages(session_id, limit=10_000) + except Exception: + logger.debug("session summary message lookup failed", exc_info=True) + return None + return summarize_messages(messages) + + def _print_exit_summary() -> None: + session_id = _active_chat_session_id() + counts = _stored_message_counts(session_id) + if counts is None: + message_count = exit_stats.message_count + user_messages = exit_stats.user_messages + tool_calls = exit_stats.tool_calls + else: + message_count, user_messages, stored_tool_calls = counts + tool_calls = max(stored_tool_calls, exit_stats.tool_calls) + if not session_id and message_count == 0: + return + console.newline() + for line in build_exit_summary_lines( + session_id=session_id, + duration_s=exit_stats.duration_s, + message_count=message_count, + user_messages=user_messages, + tool_calls=tool_calls, + resumable=not storage_volatile, + ): + console.print(line) # ── Stream response ────────────────────────────────────────────── async def _stream_response(prompt_text: str) -> None: + exit_stats.record_user_message() status.mark_turn_start() app.agent_running = True app.spinner_text = "Thinking…" @@ -171,9 +288,13 @@ async def _stream_response(prompt_text: str) -> None: renderer.feed(str(event)) finally: renderer.finish() + if renderer.has_output: + exit_stats.record_assistant_message() + exit_stats.record_tool_calls(renderer.tool_count) app.spinner_text = "" app.agent_running = False status.mark_turn_end() + _update_status() # ── Input handler (business logic) ─────────────────────────────── @@ -181,6 +302,15 @@ async def handle_input(text: str) -> None: """Dispatch one user input — slash commands or natural language.""" global _last_hint + try: + if ctx.reload_runtime_config_if_changed(): + console.success( + "Configuration reloaded — LLM settings updated for this session." + ) + except Exception as exc: + logger.warning("Runtime config reload failed: %s", exc) + console.warning(f"Configuration reload failed: {exc}") + _learning = ctx.session and ctx.session.mode == SessionMode.LEARNING if _learning: ctx.imitation.end_control_input() @@ -225,7 +355,6 @@ async def handle_input(text: str) -> None: pass elif _learning: ctx.imitation.end_control_input() - console.system("Bye!") app.exit() return @@ -246,6 +375,10 @@ async def handle_input(text: str) -> None: handle_tools(ctx, console, cmd_args) return + if canonical == "gateway": + handle_gateway(ctx, console, cmd_args) + return + if canonical == "usage": handle_usage(ctx, console, cmd_args) return @@ -376,9 +509,274 @@ async def _after_dispatch(text: str) -> None: on_input=handle_input, ) + # Auto-connect previously configured gateway platforms + gw = getattr(ctx, "gateway_server", None) + if gw is not None: + try: + gw_count = await gw.start() + if gw_count > 0: + console.system(f" Gateway: {gw_count} platform(s) reconnected") + except Exception: + logger.debug("Gateway auto-connect failed", exc_info=True) + + if resume_id and ctx.engine is not None: + if ctx.engine.load_session(resume_id): + active_resume_id = resume_id + console.success(f"Resumed session {resume_id}") + else: + console.warning(f"Session '{resume_id}' not found; starting a new session.") + + _render_banner() + _update_status() + exit_code = 0 + try: + exit_code = await app.run() + finally: + _print_exit_summary() + return exit_code + + +async def cmd_interactive_daemon( + client: "DaemonClient", + settings: Any, + *, + resume_id: Optional[str] = None, +) -> int: + """Persistent REPL backed by leapd thin-client RPC.""" + from leapflow.cli.banner import display_rich_banner + from leapflow.cli.commands.registry import completion_entries, resolve_command + from leapflow.cli.tui_app import ( + LeapApp, + LeapConsole, + SessionExitStats, + StreamRenderer, + build_exit_summary_lines, + detect_theme, + ) + from leapflow.cli.tui_app.status import StatusBar + from leapflow.tools.registry_bootstrap import TOOL_DEFINITIONS + + theme = detect_theme() + console = LeapConsole(theme) + status = StatusBar(theme) + exit_stats = SessionExitStats() + active_session_id = str(resume_id or "") + turn_count = 0 + runtime_model_name = str(getattr(settings, "llm_model", "")) + runtime_context_length = int(getattr(settings, "llm_context_length", 0) or 0) + runtime_context_used = 0 + runtime_daemon_pid = "" + + def _apply_daemon_runtime_metadata(metadata: dict[str, Any]) -> None: + nonlocal active_session_id, runtime_model_name, runtime_context_length + nonlocal runtime_context_used, runtime_daemon_pid + if metadata.get("pid"): + runtime_daemon_pid = str(metadata["pid"]) + if metadata.get("session_id"): + active_session_id = str(metadata["session_id"]) + if metadata.get("model"): + runtime_model_name = str(metadata["model"]) + if metadata.get("llm_model"): + runtime_model_name = str(metadata["llm_model"]) + if metadata.get("llm_context_length") is not None: + try: + runtime_context_length = max(1, int(metadata["llm_context_length"])) + except (TypeError, ValueError): + pass + if metadata.get("context_used") is not None: + try: + runtime_context_used = max(0, int(metadata["context_used"])) + except (TypeError, ValueError): + pass + + def _update_status() -> None: + status.update( + mode="daemon", + skill_count=0, + platform_online=True, + model_name=runtime_model_name, + session_turns=turn_count, + context_used=runtime_context_used, + context_max=runtime_context_length, + ) + app.prompt_mode = "daemon" + + def _render_banner() -> None: + display_rich_banner( + model=runtime_model_name, + cwd=os.getcwd(), + session_id=active_session_id, + platform_online=True, + tool_defs=TOOL_DEFINITIONS, + skills=[], + context_length=runtime_context_length, + mcp_tools=0, + gateway_connected=[], + theme=theme, + ) + daemon_suffix = f" pid={runtime_daemon_pid}" if runtime_daemon_pid else "" + console.system(f"Daemon mode{daemon_suffix}: shared runtime across terminals.") + console.system("After reinstalling LeapFlow, use `leap daemon restart` to load new code.") + + async def _print_daemon_status() -> None: + try: + daemon_status = await client.status() + except Exception as exc: + console.warning(f"Daemon status unavailable: {exc}") + return + _apply_daemon_runtime_metadata(daemon_status) + _update_status() + console.system( + "leapd " + f"pid={daemon_status.get('pid')} " + f"profile={daemon_status.get('profile')} " + f"clients={daemon_status.get('active_clients')} " + f"volatile={daemon_status.get('volatile')}" + ) + db_path = daemon_status.get("db_path") + if db_path: + console.system(f"DB: {db_path}") + config_path = daemon_status.get("config_path") + if config_path: + console.system(f"Config: {config_path}") + project_env_path = daemon_status.get("project_env_path") + if project_env_path: + console.system(f"Project override: {project_env_path}") + runtime_version = daemon_status.get("runtime_version") + if runtime_version: + console.system(f"Runtime version: {runtime_version}") + runtime_source = daemon_status.get("runtime_source") + if runtime_source: + console.system(f"Runtime source: {runtime_source}") + runtime_executable = daemon_status.get("runtime_executable") + if runtime_executable: + console.system(f"Python: {runtime_executable}") + context_length = daemon_status.get("llm_context_length") + if context_length: + console.system(f"Context budget: {int(context_length):,} tokens") + context_used = daemon_status.get("context_used") + if context_used is not None: + console.system(f"Context used: {int(context_used):,} tokens") + + async def _stream_response(prompt_text: str) -> None: + nonlocal active_session_id, turn_count, runtime_model_name + nonlocal runtime_context_length, runtime_context_used + exit_stats.record_user_message() + status.mark_turn_start() + app.agent_running = True + app.spinner_text = "Thinking…" + + renderer = StreamRenderer(console) + renderer.start() + try: + async for event in client.engine_chat(prompt_text): + metadata = event.metadata or {} + _apply_daemon_runtime_metadata(metadata) + if event.type == "chunk": + renderer.feed(event.content) + elif event.type == "thinking": + renderer.feed_thinking(event.content) + elif event.type == "tool_start": + app.spinner_text = renderer.tool_started(event.content) + elif event.type == "tool_complete": + renderer.tool_finished(event.content) + app.spinner_text = "Thinking…" + elif event.type == "final": + if not renderer.text: + renderer.feed(event.content) + elif event.type == "error": + renderer.feed(event.content) + elif event.type == "status": + console.system(event.content) + finally: + renderer.finish() + if renderer.has_output: + exit_stats.record_assistant_message() + exit_stats.record_tool_calls(renderer.tool_count) + turn_count += 1 + app.spinner_text = "" + app.agent_running = False + status.mark_turn_end() + _update_status() + + async def handle_input(text: str) -> None: + cmd_text = text.lstrip("/") if text.startswith("/") else text + cmd_def = resolve_command(cmd_text) + if cmd_def is not None: + canonical = cmd_def.name + if canonical == "exit": + app.exit() + return + if canonical == "help": + _show_help(console) + return + if canonical == "status": + await _print_daemon_status() + return + if canonical == "clear": + _render_banner() + return + if canonical in {"tools", "usage", "model"}: + console.system( + f"/{canonical} is not available in daemon mode yet; " + "chat and resume are daemon-backed in this phase." + ) + return + console.warning( + f"/{canonical} is not available in daemon mode yet. " + "Use --no-daemon for legacy in-process commands." + ) + return + + console.rule() + await _stream_response(text) + + def _print_exit_summary() -> None: + if not active_session_id and exit_stats.message_count == 0: + return + console.newline() + for line in build_exit_summary_lines( + session_id=active_session_id, + duration_s=exit_stats.duration_s, + message_count=exit_stats.message_count, + user_messages=exit_stats.user_messages, + tool_calls=exit_stats.tool_calls, + resumable=True, + ): + console.print(line) + + app = LeapApp( + console=console, + theme=theme, + status=status, + commands=completion_entries(), + data_dir=getattr(settings, "data_dir", None), + on_input=handle_input, + ) + + if resume_id: + result = await client.session_resume(resume_id) + if result.get("found"): + active_session_id = str(result.get("session_id") or resume_id) + console.success(f"Resumed session {active_session_id}") + else: + console.warning(f"Session '{resume_id}' not found; starting a new session.") + active_session_id = "" + + try: + _apply_daemon_runtime_metadata(await client.status()) + except Exception as exc: + console.warning(f"Daemon status unavailable: {exc}") + _render_banner() _update_status() - return await app.run() + exit_code = 0 + try: + exit_code = await app.run() + finally: + _print_exit_summary() + await _prompt_stop_daemon_on_exit(client, settings, console) + return exit_code # ── Command handlers ───────────────────────────────────────────────── diff --git a/src/leapflow/cli/commands/registry.py b/src/leapflow/cli/commands/registry.py index 4053899..9edc1e8 100644 --- a/src/leapflow/cli/commands/registry.py +++ b/src/leapflow/cli/commands/registry.py @@ -66,6 +66,9 @@ class CommandDef: CommandDef("shortcut add", "Add a quick-reply shortcut", "Shortcuts", args_hint=" = "), CommandDef("shortcut remove", "Remove a shortcut", "Shortcuts", args_hint=""), + # Gateway + CommandDef("gateway", "Show connected platforms and gateway status", "Gateway"), + # Scheduler CommandDef("arm", "Schedule a skill for timed execution", "Scheduler", args_hint=" "), CommandDef("tasks", "List scheduled tasks", "Scheduler"), diff --git a/src/leapflow/cli/commands/scheduler.py b/src/leapflow/cli/commands/scheduler.py index 8c59271..c3f8bc9 100644 --- a/src/leapflow/cli/commands/scheduler.py +++ b/src/leapflow/cli/commands/scheduler.py @@ -65,8 +65,7 @@ def _get_coordinator(ctx: "Context"): return ctx.coordinator # Build one from settings - db_path = ctx.settings.data_dir / "scheduler.duckdb" - store = TaskStore(db_path) + store = TaskStore(ctx.settings.duckdb_path) # Local scheduler with a simple skill executor class _SimpleExecutor: diff --git a/src/leapflow/cli/commands/slash_handlers.py b/src/leapflow/cli/commands/slash_handlers.py index e2bf795..326f3b0 100644 --- a/src/leapflow/cli/commands/slash_handlers.py +++ b/src/leapflow/cli/commands/slash_handlers.py @@ -52,6 +52,14 @@ def handle_status(ctx: "Context", console: "LeapConsole", args: str) -> None: info.append("CWD: ", style="dim") info.append(f"{cwd}\n") + config_path = ctx.settings.data_dir / ".env" + info.append("Config: ", style="dim") + info.append(f"{str(config_path).replace(os.path.expanduser('~'), '~')}\n") + + project_env = os.path.join(os.getcwd(), ".env") + info.append("Override: ", style="dim") + info.append(f"{project_env.replace(os.path.expanduser('~'), '~')}\n") + session_id = getattr(ctx.session, "session_id", "") if session_id: info.append("Session: ", style="dim") @@ -67,6 +75,20 @@ def handle_status(ctx: "Context", console: "LeapConsole", args: str) -> None: info.append("Mode: ", style="dim") info.append(f"{mode}\n") + gw = getattr(ctx, "gateway_server", None) + if gw is not None: + statuses = gw.platform_status() + connected = [s for s in statuses if s.connected] + info.append("Gateway: ", style="dim") + if connected: + names = [] + for s in connected: + m = gw.manifests.get(s.platform_id) + names.append(m.display_name if m else s.platform_id) + info.append(f"{', '.join(names)}\n", style="green") + else: + info.append("no connections\n", style="dim") + console.print(Panel( info, title="[bold cyan]LeapFlow Status[/]", @@ -172,6 +194,67 @@ def handle_model(ctx: "Context", console: "LeapConsole", args: str) -> None: console.system(f" Example: LEAPFLOW_LLM_MODEL={model_arg} leap") +def handle_gateway(ctx: "Context", console: "LeapConsole", args: str) -> None: + """Display gateway status: connected platforms, available integrations.""" + from rich.panel import Panel + from rich.text import Text + + gw = getattr(ctx, "gateway_server", None) + if gw is None: + console.warning("Gateway not initialised.") + return + + statuses = gw.platform_status() + if not statuses: + console.system("No platform manifests discovered.") + return + + info = Text() + connected = [s for s in statuses if s.connected] + configured = [s for s in statuses if not s.connected and s.error == "configured but not connected"] + available = [s for s in statuses if not s.connected and not s.error] + + import time + + if connected: + info.append("Connected\n", style="bold green") + for s in connected: + m = gw.manifests.get(s.platform_id) + name = m.display_name if m else s.platform_id + uptime = "" + if s.connected_since > 0: + secs = int(time.time() - s.connected_since) + if secs < 60: + uptime = f" ({secs}s)" + elif secs < 3600: + uptime = f" ({secs // 60}m)" + else: + uptime = f" ({secs // 3600}h {(secs % 3600) // 60}m)" + info.append(f" ● {name}{uptime}\n", style="green") + + if configured: + info.append("Configured (not connected)\n", style="bold yellow") + for s in configured: + m = gw.manifests.get(s.platform_id) + name = m.display_name if m else s.platform_id + info.append(f" ○ {name}\n", style="yellow") + + if available: + info.append("Available\n", style="bold dim") + names = [gw.manifests[s.platform_id].display_name for s in available if s.platform_id in gw.manifests] + info.append(f" {', '.join(names)}\n", style="dim") + + info.append("\n", style="dim") + info.append('Say "connect to " to set up a new integration.', style="dim italic") + + console.print(Panel( + info, + title="[bold cyan]Gateway[/]", + border_style="bright_black", + padding=(0, 2), + )) + + def handle_clear(ctx: "Context", console: "LeapConsole", args: str) -> None: """Clear the terminal screen.""" os.system("cls" if os.name == "nt" else "clear") diff --git a/src/leapflow/cli/context.py b/src/leapflow/cli/context.py index 9cc5cad..f100a61 100644 --- a/src/leapflow/cli/context.py +++ b/src/leapflow/cli/context.py @@ -8,13 +8,16 @@ import re import sys import time +from dataclasses import replace from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional + +from dotenv import dotenv_values from leapflow.platform.cua_client import CuaDriverClient from leapflow.platform.event_bus import EventBus from leapflow.platform.mock import MockBridge -from leapflow.config import Settings, load_config +from leapflow.config import Settings, _load_yaml_overlay from leapflow.engine.engine import AgentEngine, build_default_registry from leapflow.engine.graph_planner import GraphPlanner from leapflow.engine.intent_classifier import ( @@ -32,13 +35,13 @@ from leapflow.llm.provider_chain import ( AuxiliaryClient, FailoverChain, - ProviderConfig, parse_credential_pools, parse_provider_configs, ) from leapflow.memory import ( MemoryManager, WorkingMemoryProvider, EpisodicMemoryProvider, - SemanticMemoryProvider, EvolutionMemoryProvider, MemoryFragment, + SemanticMemoryProvider, EvolutionMemoryProvider, NarrativeProvider, + MemoryFragment, ) from leapflow.skills.activator import SkillActivator from leapflow.skills.evolution import EMAConfidencePolicy @@ -50,6 +53,7 @@ from leapflow.learning.distiller import LLMSkillDistiller, SkillDistiller from leapflow.learning.doc_generator import CompositeSkillDocGenerator, LLMSkillDocGenerator from leapflow.storage.skill_docs import SkillDocStore +from leapflow.storage.connection import LocalConnectionHolder from leapflow.learning.feedback import FeedbackEvaluator from leapflow.storage.skill_library import SkillLibraryStore from leapflow.skills.registry import SkillRegistry @@ -65,26 +69,89 @@ logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from leapflow.platform.observers import RecordingProfile + from leapflow.security.approval import ApprovalDecision, ApprovalRequest + from leapflow.storage.skill_library import StoredSkill + -class _InteractiveApprovalGate: - """CLI-mode interactive approval for dangerous shell commands.""" +class _TUIApprovalGate: + """Rich-styled approval gate for the interactive TUI. + + Displays a styled panel with the action details and accepts: + - ``y``/``yes`` → allow this one time + - ``a``/``always`` → allow and skip future prompts for this category + - ``n``/``no``/Enter → deny + + Implements both ``ApprovalGate`` (unified) and ``CommandApprovalGate`` + (backward-compatible with ``shell_tools.py``). + """ + + _CATEGORY_LABELS = { + "shell_dangerous": ("Shell Command", "yellow"), + "file_write": ("File Write", "yellow"), + "gateway_send": ("External Message", "cyan"), + } + + async def request_approval( + self, request: "ApprovalRequest", + ) -> "ApprovalDecision": + from leapflow.security.approval import ApprovalDecision - async def check(self, command: str) -> bool: if not sys.stdin.isatty(): - return False + return ApprovalDecision.DENY + + label, color = self._CATEGORY_LABELS.get( + request.category, ("Action", "yellow"), + ) + try: - if sys.stderr.isatty(): - sys.stderr.write(f"\033[33m⚠ Dangerous command: {command}\033[0m\n") - else: - sys.stderr.write(f"WARNING: Dangerous command: {command}\n") - sys.stderr.write("Approve? [y/N]: ") + from rich.console import Console + from rich.panel import Panel + from rich.text import Text + + console = Console(stderr=True, highlight=False) + body = Text() + body.append(f" {request.detail}\n\n", style="bold") + body.append(" [y]es — allow once\n", style="dim") + body.append(" [a]lways — allow for this session\n", style="dim") + body.append(" [n]o / Enter — deny\n", style="dim") + console.print(Panel( + body, + title=f"[bold {color}]⚠ {label} Approval[/]", + border_style=color, + padding=(0, 1), + )) + sys.stderr.write(" → ") + sys.stderr.flush() + except ImportError: + sys.stderr.write(f"⚠ {label}: {request.detail}\n") + sys.stderr.write("Approve? [y/a(lways)/N]: ") sys.stderr.flush() + + try: answer = await asyncio.get_running_loop().run_in_executor( - None, lambda: input().strip().lower() + None, lambda: input().strip().lower(), ) - return answer in ("y", "yes") except (EOFError, KeyboardInterrupt): - return False + return ApprovalDecision.DENY + + if answer in ("a", "always"): + return ApprovalDecision.ALLOW_SESSION + if answer in ("y", "yes"): + return ApprovalDecision.ALLOW + return ApprovalDecision.DENY + + async def check(self, command: str) -> bool: + """``CommandApprovalGate`` compatibility — shell_tools calls this.""" + from leapflow.security.approval import ApprovalDecision, ApprovalRequest + + decision = await self.request_approval(ApprovalRequest( + category="shell_dangerous", + detail=command, + risk_hint=0.7, + )) + return decision in (ApprovalDecision.ALLOW, ApprovalDecision.ALLOW_SESSION) def _default_recording_profile(settings: Settings) -> Optional["RecordingProfile"]: @@ -133,6 +200,15 @@ def _build_visual_components( from leapflow.perception.session import PerceptionSession vlm_api_key = settings.vlm_api_key or settings.llm_api_key + if not vlm_api_key.strip(): + message = ( + "Visual perception disabled: LEAPFLOW_VLM_API_KEY or " + "LEAPFLOW_LLM_API_KEY is required when visual track is enabled." + ) + logger.warning(message) + _emit_status(message) + return None + vlm_base_url = settings.vlm_base_url or settings.llm_base_url vlm_model = settings.vlm_model or settings.llm_model vlm_provider = OpenAIChat( @@ -265,7 +341,6 @@ def _register_stored_skill_fallbacks( llm: Any, ) -> int: """Register StoredSkills that lack a parameterized or doc counterpart.""" - from leapflow.storage.skill_library import StoredSkill from leapflow.skills.registry import Skill, SkillMetadata registered_names = set(registry.names()) if hasattr(registry, 'names') else {s.name for s in registry.list_all()} @@ -307,19 +382,33 @@ def __init__(self, settings: Settings, mock_host: bool) -> None: self.settings = settings self.effective_mock = bool(mock_host or settings.mock_host) - # Memory subsystem — provider-based architecture + # Shared DuckDB connection holder — single leap.duckdb for all stores (P1). + # Created here (lazy, not yet opened) so __init__-time providers such as + # SemanticMemoryProvider can bind to it. Eager open + lock detection + # happens later in initialize(). + self._db_holder = LocalConnectionHolder( + settings.duckdb_path, + volatile_on_lock=True, + ) + + # Memory subsystem — provider-based architecture (dual-layer) working = WorkingMemoryProvider(max_tokens=settings.memory_working_max_tokens) - semantic = SemanticMemoryProvider(db_path=settings.duckdb_path) + semantic = SemanticMemoryProvider(source=self._db_holder) episodic = EpisodicMemoryProvider( ttl=settings.memory_episodic_ttl_s, max_entries=settings.memory_episodic_max_entries, on_promote=_build_promotion_callback(semantic), ) evolution = EvolutionMemoryProvider(max_episodes=settings.memory_evolution_max_episodes) + narrative = NarrativeProvider( + memory_dir=settings.profile_dir / "memory", + workspace_path=str(Path.cwd()), + ) self.memory = MemoryManager() self.memory.add_provider(working) self.memory.add_provider(episodic) + self.memory.add_provider(narrative) self.memory.add_provider(semantic) self.memory.add_provider(evolution) @@ -365,7 +454,51 @@ def __init__(self, settings: Settings, mock_host: bool) -> None: ) self.rpc = CuaDriverClient() - # ── Multi-provider LLM chain (failover + credential rotation) ── + self._config_signature = self._runtime_config_signature(settings) + self._configure_llm_clients(settings) + + self.audit = AuditLogger(settings.audit_log_path) + + self.shortcuts = ShortcutStore(Path.cwd() / ".leapflow" / "shortcuts.yaml") + self.assessor: Optional[LLMSituationalAssessor] = None + + self.perception_session: Optional[Any] = None + self.registry: Optional[SkillRegistry] = None + self.imitation: Optional[ImitationPipeline] = None + self.skill_lib: Optional[SkillLibraryStore] = None + self.doc_store: Optional[SkillDocStore] = None + self.session: Optional[SessionController] = None + self.session_store: Optional[LearningSessionStore] = None + self.engine: Optional[AgentEngine] = None + self.intent_classifier: Optional[IntentClassifier] = None + + # World Model components (wired during initialize) + self.learning_budget: Optional[Any] = None + self.experience_store: Optional[Any] = None + self.prediction_loop: Optional[Any] = None + self.curiosity: Optional[Any] = None + self.replay_engine: Optional[Any] = None + self.snapshot_service: Optional[Any] = None + self.trajectory_grader: Optional[Any] = None + self.active_observer: Optional[Any] = None + + # Daemon / Observer + self._observation_daemon: Optional[Any] = None + self._pipeline_observer: Optional[Any] = None + + # Skill evolution & PatternMiner + self._evolution_policy: Optional[EMAConfidencePolicy] = None + self._pattern_miner: Optional[Any] = None + + # Unified approval gate is resource-free; create it in __init__ so all + # initialize() wiring paths can safely reference the same session gate. + from leapflow.security.approval import SessionAwareGate + + self._tui_approval = _TUIApprovalGate() + self._approval_gate = SessionAwareGate(self._tui_approval) + + def _configure_llm_clients(self, settings: Settings) -> None: + """Build LLM/VLM clients from a settings snapshot.""" provider_configs = parse_provider_configs( settings.llm_api_key or "missing", settings.llm_base_url, @@ -379,7 +512,8 @@ def __init__(self, settings: Settings, mock_host: bool) -> None: ) if len(provider_configs) > 1 or credential_pools: self.llm_chain = FailoverChain( - provider_configs, credential_pools=credential_pools, + provider_configs, + credential_pools=credential_pools, circuit_failure_threshold=settings.circuit_breaker_threshold, circuit_cooldown_s=settings.circuit_breaker_cooldown_s, ) @@ -397,7 +531,6 @@ def __init__(self, settings: Settings, mock_host: bool) -> None: max_retries=settings.llm_max_retries, ) - # ── Auxiliary LLM for cheap operations ── self.auxiliary: Optional[AuxiliaryClient] = None if settings.llm_aux_model: aux_llm = OpenAIChat( @@ -412,45 +545,198 @@ def __init__(self, settings: Settings, mock_host: bool) -> None: self.auxiliary = AuxiliaryClient(self.llm) self.vlm: Optional[OpenAIChat] = None - if settings.vlm_model and settings.vlm_model != settings.llm_model: + if ( + settings.vlm_model + and settings.vlm_model != settings.llm_model + and settings.has_vlm_credentials + ): self.vlm = OpenAIChat( - api_key=settings.vlm_api_key or settings.llm_api_key or "missing", + api_key=settings.vlm_api_key or settings.llm_api_key, base_url=settings.vlm_base_url or settings.llm_base_url, model=settings.vlm_model, ) - self.audit = AuditLogger(settings.audit_log_path) + def _effective_llm_context_length(self, settings: Settings) -> int: + """Return the configured runtime context budget for the active provider.""" + if self.llm_chain is not None: + return max(1, int(self.llm_chain.context_length)) + return max(1, int(settings.llm_context_length)) - self.shortcuts = ShortcutStore(Path.cwd() / ".leapflow" / "shortcuts.yaml") - self.assessor: Optional[LLMSituationalAssessor] = None + def _build_model_capability_registry(self, settings: Settings) -> Any: + """Build model capabilities where explicit runtime config wins over static hints.""" + from leapflow.llm.model_capabilities import ModelCapabilities, ModelCapabilityRegistry - self.perception_session: Optional[Any] = None - self.registry: Optional[SkillRegistry] = None - self.imitation: Optional[ImitationPipeline] = None - self.skill_lib: Optional[SkillLibraryStore] = None - self.doc_store: Optional[SkillDocStore] = None - self.session: Optional[SessionController] = None - self.session_store: Optional[LearningSessionStore] = None - self.engine: Optional[AgentEngine] = None - self.intent_classifier: Optional[IntentClassifier] = None + cap_registry = ModelCapabilityRegistry() + base_caps = cap_registry.resolve(settings.llm_model) + cap_registry.register( + settings.llm_model, + ModelCapabilities( + context_length=self._effective_llm_context_length(settings), + max_output_tokens=base_caps.max_output_tokens, + supports_tools=settings.native_tool_calling_enabled, + supports_vision=base_caps.supports_vision, + supports_thinking=base_caps.supports_thinking, + supports_streaming_tools=base_caps.supports_streaming_tools, + tokens_per_image=base_caps.tokens_per_image, + ), + ) + return cap_registry - # World Model components (wired during initialize) - self.learning_budget: Optional[Any] = None - self.experience_store: Optional[Any] = None - self.prediction_loop: Optional[Any] = None - self.curiosity: Optional[Any] = None - self.replay_engine: Optional[Any] = None - self.snapshot_service: Optional[Any] = None - self.trajectory_grader: Optional[Any] = None - self.active_observer: Optional[Any] = None + def _sync_engine_runtime_budget(self, settings: Settings) -> None: + """Sync engine-visible budgets and model capability metadata from settings.""" + if self.engine is None: + return - # Daemon / Observer - self._observation_daemon: Optional[Any] = None - self._pipeline_observer: Optional[Any] = None + context_length = self._effective_llm_context_length(settings) + dynamic_result_budget = min(settings.max_tool_result_chars, context_length // 20) + if dynamic_result_budget != settings.max_tool_result_chars: + logger.info( + "Dynamic tool result budget: %d (context=%d)", + dynamic_result_budget, + context_length, + ) + self.engine.set_tool_result_budget(dynamic_result_budget) + self.engine.set_model_capabilities(self._build_model_capability_registry(settings)) + logger.debug("Model capability registry wired") + + @staticmethod + def _runtime_config_signature(settings: Settings) -> tuple: + """Return a stable signature for user-editable runtime config files.""" + paths = ( + settings.data_dir / ".env", + settings.data_dir / "config.yaml", + Path.cwd() / ".env", + ) + signature = [] + for path in paths: + try: + stat = path.stat() + except OSError: + signature.append((str(path), 0, 0)) + else: + signature.append((str(path), stat.st_mtime_ns, stat.st_size)) + return tuple(signature) + + @staticmethod + def _bool_env(value: str, default: bool) -> bool: + text = value.strip().lower() + if not text: + return default + return text in {"1", "true", "yes", "on"} + + def _load_runtime_settings_from_files(self) -> Settings: + """Reload hot-swappable LLM/VLM settings directly from config files.""" + values: dict[str, str] = {} + data_env = self.settings.data_dir / ".env" + cwd_env = Path.cwd() / ".env" + if data_env.exists(): + values.update({ + key: value + for key, value in dotenv_values(data_env).items() + if value is not None + }) + values.update(_load_yaml_overlay(self.settings.data_dir)) + if cwd_env.exists(): + values.update({ + key: value + for key, value in dotenv_values(cwd_env).items() + if value is not None + }) + + def _value(key: str, current: str) -> str: + env_value = os.environ.get(key, "").strip() + if env_value: + return env_value + return str(values.get(key, current)).strip() + + max_retries_raw = _value( + "LEAPFLOW_LLM_MAX_RETRIES", + str(self.settings.llm_max_retries), + ) + try: + max_retries = max(1, int(max_retries_raw)) + except ValueError: + max_retries = self.settings.llm_max_retries - # Skill evolution & PatternMiner - self._evolution_policy: Optional[EMAConfidencePolicy] = None - self._pattern_miner: Optional[Any] = None + context_length_raw = _value( + "LEAPFLOW_LLM_CONTEXT_LENGTH", + str(self.settings.llm_context_length), + ) + try: + llm_context_length = max(1, int(context_length_raw)) + except ValueError: + llm_context_length = self.settings.llm_context_length + + return replace( + self.settings, + llm_api_key=_value("LEAPFLOW_LLM_API_KEY", self.settings.llm_api_key), + llm_base_url=_value("LEAPFLOW_LLM_BASE_URL", self.settings.llm_base_url).rstrip("/"), + llm_model=_value("LEAPFLOW_LLM_MODEL", self.settings.llm_model), + llm_max_retries=max_retries, + llm_context_length=llm_context_length, + vlm_api_key=_value("LEAPFLOW_VLM_API_KEY", self.settings.vlm_api_key), + vlm_base_url=_value("LEAPFLOW_VLM_BASE_URL", self.settings.vlm_base_url).rstrip("/"), + vlm_model=_value("LEAPFLOW_VLM_MODEL", self.settings.vlm_model), + visual_track_enabled=self._bool_env( + _value( + "LEAPFLOW_VISUAL_TRACK_ENABLED", + "1" if self.settings.visual_track_enabled else "0", + ), + self.settings.visual_track_enabled, + ), + ) + + def reload_runtime_config_if_changed(self) -> bool: + """Hot-reload LLM/VLM config when user-editable config files changed.""" + signature = self._runtime_config_signature(self.settings) + if signature == self._config_signature: + return False + + previous = self.settings + updated = self._load_runtime_settings_from_files() + self._config_signature = signature + llm_changed = ( + previous.llm_api_key != updated.llm_api_key + or previous.llm_base_url != updated.llm_base_url + or previous.llm_model != updated.llm_model + or previous.llm_max_retries != updated.llm_max_retries + or previous.llm_context_length != updated.llm_context_length + or previous.vlm_api_key != updated.vlm_api_key + or previous.vlm_base_url != updated.vlm_base_url + or previous.vlm_model != updated.vlm_model + or previous.visual_track_enabled != updated.visual_track_enabled + ) + self.settings = updated + if not llm_changed: + return False + + self._configure_llm_clients(updated) + classifier: IntentClassifier = ( + LLMIntentClassifier(self.llm) + if updated.has_llm_credentials + else FallbackClassifier() + ) + self.intent_classifier = classifier + self.assessor = ( + LLMSituationalAssessor(self.llm) + if updated.has_llm_credentials + else None + ) + if self.engine is not None: + self.engine.reconfigure_runtime( + settings=updated, + llm=self.llm, + vlm=self.vlm, + classifier=classifier, + ) + self._sync_engine_runtime_budget(updated) + logger.info("Runtime LLM configuration reloaded") + return True + + @property + def storage_volatile(self) -> bool: + """Return True when this process uses non-persistent fallback storage.""" + return bool(getattr(self._db_holder, "is_volatile", False)) async def initialize(self) -> None: """Async initialization: VSI handshake, pipeline assembly. @@ -463,6 +749,13 @@ async def initialize(self) -> None: settings = self.settings await self.memory.initialize_all() + if self.storage_volatile: + _emit_status( + "Primary database is locked; running with volatile session storage." + ) + _emit_status( + "This window can chat, but new memory/session data will not persist." + ) vsi = VirtualSystemInterface(self.rpc) bridge_online = False @@ -525,8 +818,17 @@ async def initialize(self) -> None: if settings.has_llm_credentials: codegen = CompositeSkillCodeGenerator(LLMSkillCodeGenerator(self.llm)) - traj_db_path = settings.duckdb_path.parent / "trajectories.duckdb" - traj_store = TrajectoryStore(traj_db_path) + # Holder was created in __init__ and may already have opened during + # memory initialization. Access once here to preserve early lock/fallback + # detection before persistent stores are assembled. + _ = self._db_holder.connection + + try: + traj_store = TrajectoryStore(self._db_holder) + except Exception as exc: + logger.error("TrajectoryStore init failed: %s", exc) + self._db_holder.close() + raise SystemExit(f"\nFailed to initialize trajectory store: {exc}") from exc distiller: SkillDistiller if settings.has_llm_credentials: distiller = LLMSkillDistiller(self.llm) @@ -551,9 +853,17 @@ async def initialize(self) -> None: video_segmenter = None signal_timeline = None if settings.recording_mode.uses_video and settings.visual_track_enabled: - video_recorder, video_analyzer, video_segmenter, signal_timeline = ( - _build_video_components(settings, self.rpc, self.vlm or self.llm) - ) + if settings.has_vlm_credentials: + video_recorder, video_analyzer, video_segmenter, signal_timeline = ( + _build_video_components(settings, self.rpc, self.vlm or self.llm) + ) + else: + message = ( + "Video analysis disabled: LEAPFLOW_VLM_API_KEY or " + "LEAPFLOW_LLM_API_KEY is required for visual recording mode." + ) + logger.warning(message) + _emit_status(message) platform_hint = manifest.platform_id.value @@ -629,8 +939,7 @@ async def initialize(self) -> None: "(video mode active but timeline unavailable)" ) - skill_lib_path = settings.duckdb_path.parent / "skill_library.duckdb" - self.skill_lib = SkillLibraryStore(skill_lib_path, audit_logger=self.audit) + self.skill_lib = SkillLibraryStore(self._db_holder, audit_logger=self.audit) scorer = HeuristicSimilarityScorer() llm_scorer = LLMSimilarityScorer(self.llm) if settings.has_llm_credentials else None feedback_evaluator = FeedbackEvaluator( @@ -879,7 +1188,7 @@ def _on_prediction_outcome(outcome: Any) -> None: from leapflow.engine.confirmation import ConfirmationHandler confirmation = ConfirmationHandler(skill_store=self.skill_lib) - self.session_store = LearningSessionStore(traj_db_path) + self.session_store = LearningSessionStore(self._db_holder) # Learnability assessor learnability_assessor = None @@ -912,7 +1221,7 @@ def _on_prediction_outcome(outcome: Any) -> None: auto_learn=settings.learn_auto_distill, confirmation=confirmation, audit=self.audit, - storage_path=str(skill_lib_path), + storage_path=str(settings.duckdb_path), audit_log_path=str(settings.audit_log_path), active_learning_observer=observer, session_store=self.session_store, @@ -976,7 +1285,7 @@ def _on_prediction_outcome(outcome: Any) -> None: ] # L2/L3: wire Memory adapters when ExperienceStore is available if hasattr(self, 'experience_store') and self.experience_store is not None: - from leapflow.copilot.adapters import ExperienceEmbedAdapter, MemoryRAGAdapter + from leapflow.copilot.adapters import ExperienceEmbedAdapter from leapflow.copilot.predictors.l2_embed import L2EmbeddingPredictor from leapflow.copilot.predictors.l3_llm import L3LLMPredictor @@ -1051,6 +1360,78 @@ async def _copilot_on_idle(duration_ms: int) -> None: from leapflow.tools.registry_bootstrap import set_memory_manager set_memory_manager(self.memory) + # ── Gateway server (late-bound tool wiring) ── + from leapflow.gateway.server import GatewayServer + from leapflow.gateway.router import GatewayRouter + from leapflow.gateway.events import ( + GatewayMessageReceived, + GatewaySessionCreated, + GatewaySessionEnded, + ) + from leapflow.tools.registry_bootstrap import set_gateway_server + from leapflow.tools.gateway_tool import set_gateway_approval_gate + + async def _on_gateway_event(event: object) -> None: + """Bridge gateway events to episodic memory and logging.""" + if isinstance(event, GatewayMessageReceived): + logger.info( + "gateway.inbound platform=%s session=%s len=%d", + event.source.platform, + event.session_key, + len(event.text), + ) + episodic = self.memory.get_provider("episodic") + if episodic is not None and hasattr(episodic, "ingest"): + episodic.ingest( + "gateway.message", + f"[{event.source.platform}:{event.source.user_name or event.source.user_id}] " + f"{event.text[:500]}", + metadata={ + "platform": event.source.platform, + "session": event.session_key, + }, + ) + elif isinstance(event, GatewaySessionCreated): + logger.info("gateway.session_created key=%s", event.session_key) + elif isinstance(event, GatewaySessionEnded): + logger.info( + "gateway.session_ended key=%s reason=%s", + event.session_key, + event.reason, + ) + router = getattr(self, "_gateway_router", None) + if router is not None: + router.clear_session(event.session_key) + + self.gateway_server = GatewayServer( + settings.profile_dir, + extra_manifest_dirs=[settings.profile_dir / "gateway" / "manifests"], + on_event=_on_gateway_event, + ) + self.gateway_server.discover_manifests() + set_gateway_server(self.gateway_server) + set_gateway_approval_gate(self._approval_gate) + + async def _gateway_send(source: Any, text: str) -> None: + await self.gateway_server.send_reply(source, text) + + from leapflow.tools.registry_bootstrap import TOOL_DEFINITIONS, TOOL_HANDLERS + + self._gateway_router = GatewayRouter( + llm=self.llm, + system_prompt=( + "You are LeapFlow, a helpful AI assistant responding " + "through an external messaging platform. Be concise " + "and conversational." + ), + send_fn=_gateway_send, + tool_definitions=TOOL_DEFINITIONS, + tool_handlers=TOOL_HANDLERS, + ) + self.gateway_server.set_message_handler( + self._gateway_router.handle_message, + ) + # ── Build CompressorConfig with LLM callbacks ── from leapflow.engine.context_compressor import CompressorConfig @@ -1070,15 +1451,12 @@ async def _summarize_via_llm(prompt: str) -> str: token_count_fn=lambda text: len(text) // 4, ) - # ── Initialize DuckDBConversationStore (with self-repair) ── + # ── Initialize DuckDBConversationStore ── self._conversation_store = None try: from leapflow.storage.conversation_store import DuckDBConversationStore - from leapflow.storage.db_repair import check_and_repair - conv_db_path = settings.duckdb_path.parent / "conversations.duckdb" - check_and_repair(conv_db_path) - self._conversation_store = DuckDBConversationStore(conv_db_path) - logger.info("ConversationStore initialized: %s", conv_db_path) + self._conversation_store = DuckDBConversationStore(self._db_holder) + logger.info("ConversationStore initialized") except Exception: logger.warning("ConversationStore initialization failed", exc_info=True) @@ -1136,12 +1514,11 @@ async def _handler(params: dict) -> dict: except Exception: logger.debug("MCP Manager initialization skipped", exc_info=True) - # ── Wire Approval Gate for shell dangerous commands ── + # ── Unified approval gate wiring (shell, file, gateway) ── try: from leapflow.tools.shell_tools import set_approval_gate - - set_approval_gate(_InteractiveApprovalGate()) - logger.debug("Shell approval gate: interactive CLI mode") + set_approval_gate(self._approval_gate) + logger.debug("Shell approval gate: unified TUI mode") except Exception: logger.debug("Shell approval gate setup skipped", exc_info=True) @@ -1215,10 +1592,7 @@ async def _archive_to_semantic(messages: List[Dict[str, Any]]) -> None: self._evolution_store = None try: from leapflow.storage.evolution_store import DuckDBEvolutionStore - from leapflow.storage.db_repair import check_and_repair - evo_db_path = settings.duckdb_path.parent / "evolution.duckdb" - check_and_repair(evo_db_path) - self._evolution_store = DuckDBEvolutionStore(evo_db_path) + self._evolution_store = DuckDBEvolutionStore(self._db_holder) # Hydrate in-memory provider from persisted episodes persisted = self._evolution_store.load_recent_episodes( limit=settings.memory_evolution_max_episodes, @@ -1250,15 +1624,18 @@ async def _archive_to_semantic(messages: List[Dict[str, Any]]) -> None: # ── Wire Smart Approval (auxiliary LLM for command risk) ── if self.auxiliary is not None: try: - from leapflow.tools.shell_tools import CommandApprovalGate aux = self.auxiliary class _SmartApprovalGate: - """LLM-assisted approval for dangerous commands. + """LLM-assisted approval that auto-approves low-risk commands. - Low-risk commands (score < 0.3) are auto-approved. - Medium-risk prompts the user. High-risk always prompts. + Uses the auxiliary LLM to score risk (0.0–1.0). + Low-risk (< 0.3) auto-approved; others prompt via TUI gate. + Wraps the unified approval gate — session memory still works. """ + def __init__(self, delegate: Any) -> None: + self._delegate = delegate + async def check(self, command: str) -> bool: try: risk = await aux.classify_risk(command) @@ -1267,10 +1644,10 @@ async def check(self, command: str) -> bool: if risk < 0.3: logger.debug("smart_approval: auto-approved (risk=%.2f)", risk) return True - return await _InteractiveApprovalGate().check(command) + return await self._delegate.check(command) from leapflow.tools.shell_tools import set_approval_gate - set_approval_gate(_SmartApprovalGate()) + set_approval_gate(_SmartApprovalGate(self._approval_gate)) logger.debug("Smart approval gate enabled (auxiliary LLM)") except Exception: logger.debug("Smart approval setup skipped", exc_info=True) @@ -1278,79 +1655,53 @@ async def check(self, command: str) -> bool: # ── Wire File Write Approval Gate ── try: from leapflow.tools.registry_bootstrap import set_file_write_gate + from leapflow.security.approval import ApprovalDecision, ApprovalRequest + + _SAFE_EXTENSIONS = frozenset({ + ".tmp", ".log", ".txt", ".md", ".json", ".yaml", ".yml", + ".csv", ".tsv", + }) + approval_gate = self._approval_gate class _FileWriteGate: - """Interactive approval for file writes to important paths.""" - _APPROVE_EXTENSIONS = frozenset({ - ".tmp", ".log", ".txt", ".md", ".json", ".yaml", ".yml", - ".csv", ".tsv", - }) + """File write approval via the unified gate. + + Auto-approves safe extensions and small writes; delegates + everything else to the session-aware approval gate. + """ async def check(self, path: str, content: str) -> bool: from pathlib import Path as _P p = _P(path) - if p.suffix.lower() in self._APPROVE_EXTENSIONS: + if p.suffix.lower() in _SAFE_EXTENSIONS: return True if len(content) < 500: return True - if not sys.stdin.isatty(): - return True - if sys.stderr.isatty(): - sys.stderr.write(f"\033[33m⚠ File write: {p.name} ({len(content)} chars)\033[0m\n") - else: - sys.stderr.write(f"WARNING: File write: {p.name} ({len(content)} chars)\n") - sys.stderr.write("Approve? [Y/n]: ") - sys.stderr.flush() - try: - answer = await asyncio.get_running_loop().run_in_executor( - None, lambda: input().strip().lower() - ) - return answer in ("", "y", "yes") - except (EOFError, KeyboardInterrupt): - return False + decision = await approval_gate.request_approval( + ApprovalRequest( + category="file_write", + detail=f"{p.name} ({len(content)} chars)", + risk_hint=0.4, + ), + ) + return decision in ( + ApprovalDecision.ALLOW, + ApprovalDecision.ALLOW_SESSION, + ) set_file_write_gate(_FileWriteGate()) - logger.debug("File write approval gate enabled") + logger.debug("File write approval gate: unified") except Exception: logger.debug("File write gate setup skipped", exc_info=True) - # ── Token budget dynamic linking (tool result budget ∝ model context) ── - context_length = settings.llm_context_length - if hasattr(self, 'llm_chain') and self.llm_chain is not None: - context_length = self.llm_chain.context_length - dynamic_result_budget = min(settings.max_tool_result_chars, context_length // 20) - if dynamic_result_budget != settings.max_tool_result_chars: - logger.info( - "Dynamic tool result budget: %d (context=%d)", - dynamic_result_budget, context_length, - ) - if self.engine is not None: - self.engine.set_tool_result_budget(dynamic_result_budget) - - # ── Wire new engine capabilities ── + # ── Token budget and model capability metadata ───────────────────── if self.engine is not None: + self._sync_engine_runtime_budget(settings) self.engine.set_stale_stream_timeout(settings.stale_stream_timeout_s) self.engine.set_default_tool_timeout(settings.default_tool_timeout_s) if self._evolution_store is not None: self.engine.set_evolution_store(self._evolution_store) - try: - from leapflow.llm.model_capabilities import ModelCapabilityRegistry - cap_registry = ModelCapabilityRegistry() - if hasattr(self, 'llm_chain') and self.llm_chain is not None: - from leapflow.llm.model_capabilities import ModelCapabilities - cap_registry.register( - settings.llm_model, - ModelCapabilities( - context_length=self.llm_chain.context_length, - supports_tools=settings.native_tool_calling_enabled, - ), - ) - self.engine.set_model_capabilities(cap_registry) - logger.debug("Model capability registry wired") - except Exception: - logger.debug("ModelCapabilityRegistry setup skipped", exc_info=True) - if hasattr(self, "doc_store") and self.doc_store is not None: self.engine.set_doc_store(self.doc_store) @@ -1824,6 +2175,14 @@ async def cleanup(self) -> None: except Exception: pass + # Stop gateway server + gw = getattr(self, "gateway_server", None) + if gw is not None: + try: + await gw.stop() + except Exception: + logger.debug("GatewayServer stop failed", exc_info=True) + # Cancel engine if running if self.engine is not None: self.engine.cancel() @@ -1876,6 +2235,12 @@ async def cleanup(self) -> None: self.rpc.stop() if self.skill_lib: self.skill_lib.close() + if self.session_store: + self.session_store.close() if self.imitation: self.imitation.store.close() + # Close the shared DuckDB connection last (after all stores are done) + db_holder = getattr(self, "_db_holder", None) + if db_holder is not None: + db_holder.close() self.audit.close() diff --git a/src/leapflow/cli/tui_app/__init__.py b/src/leapflow/cli/tui_app/__init__.py index 0f4dfb4..d310183 100644 --- a/src/leapflow/cli/tui_app/__init__.py +++ b/src/leapflow/cli/tui_app/__init__.py @@ -5,14 +5,42 @@ """ from leapflow.cli.tui_app.app import LeapApp +from leapflow.cli.tui_app.command import TuiCommand, TuiCommandStatus from leapflow.cli.tui_app.console import LeapConsole +from leapflow.cli.tui_app.session_summary import ( + SessionExitStats, + build_exit_summary_lines, + format_duration, + summarize_messages, +) from leapflow.cli.tui_app.stream import StreamRenderer -from leapflow.cli.tui_app.theme import Theme, detect_theme +from leapflow.cli.tui_app.theme import ( + ResolvedTheme, + Theme, + contrast_ratio, + detect_theme, + ensure_contrast, + parse_hex_color, + relative_luminance, + resolve_theme, +) __all__ = [ "LeapApp", + "TuiCommand", + "TuiCommandStatus", "LeapConsole", + "SessionExitStats", "StreamRenderer", + "build_exit_summary_lines", + "ResolvedTheme", "Theme", + "contrast_ratio", "detect_theme", + "ensure_contrast", + "format_duration", + "parse_hex_color", + "relative_luminance", + "resolve_theme", + "summarize_messages", ] diff --git a/src/leapflow/cli/tui_app/app.py b/src/leapflow/cli/tui_app/app.py index 598d5d3..0c222d1 100644 --- a/src/leapflow/cli/tui_app/app.py +++ b/src/leapflow/cli/tui_app/app.py @@ -24,6 +24,7 @@ import asyncio import os import time +from dataclasses import dataclass from pathlib import Path from typing import ( TYPE_CHECKING, @@ -37,7 +38,6 @@ from prompt_toolkit import Application from prompt_toolkit.auto_suggest import AutoSuggestFromHistory -from prompt_toolkit.filters import Condition from prompt_toolkit.history import FileHistory from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.keys import Keys @@ -48,6 +48,7 @@ from prompt_toolkit.styles import Style as PTStyle from prompt_toolkit.widgets import TextArea +from leapflow.cli.tui_app.command import TuiCommand from leapflow.cli.tui_app.input import build_completer from leapflow.cli.tui_app.status import StatusBar from leapflow.cli.tui_app.theme import Theme @@ -57,10 +58,21 @@ _HISTORY_FILENAME = "history" _REFRESH_INTERVAL_S = 0.5 +_LARGE_PASTE_CHAR_THRESHOLD = 4_000 +_LARGE_PASTE_LINE_THRESHOLD = 24 +_PASTE_COMPACTOR_ATTR = "_leapflow_paste_compactor_installed" InputHandler = Callable[[str], Union[Awaitable[None], None]] +@dataclass(frozen=True) +class _PasteBlock: + """Full pasted content stored outside the prompt buffer.""" + + marker: str + text: str + + class LeapApp: """Hybrid TUI: prompt_toolkit Application + Rich output. @@ -92,7 +104,11 @@ def __init__( self._prompt_mode = "idle" self._spinner_text = "" self._tool_start_time: float = 0.0 - self._pending_input: asyncio.Queue[str] = asyncio.Queue() + self._next_command_id = 1 + self._next_paste_id = 1 + self._paste_blocks: dict[str, _PasteBlock] = {} + self._active_command: Optional[TuiCommand] = None + self._pending_input: asyncio.Queue[TuiCommand] = asyncio.Queue() data_dir = data_dir or Path( os.environ.get("LEAPFLOW_DATA_DIR", "~/.leapflow") @@ -133,6 +149,83 @@ def spinner_text(self, value: str) -> None: self._tool_start_time = time.monotonic() if value else 0.0 self._invalidate() + def submit_text(self, text: str) -> TuiCommand: + """Submit user text into the serial TUI command queue.""" + normalized = self._resolve_paste_blocks(text).strip() + if not normalized: + raise ValueError("Cannot submit an empty TUI command") + command = TuiCommand.create(command_id=self._next_command_id, text=normalized) + self._next_command_id += 1 + self._pending_input.put_nowait(command) + self._sync_task_counts() + if self._active_command is not None or self._pending_input.qsize() > 1: + self._console.command_card(command) + self._invalidate() + return command + + def _should_compact_paste(self, text: str) -> bool: + """Return True when rendering pasted text directly would hurt TUI responsiveness.""" + if len(text) >= _LARGE_PASTE_CHAR_THRESHOLD: + return True + return text.count("\n") + 1 >= _LARGE_PASTE_LINE_THRESHOLD + + def _compact_tokens(self, text: str) -> str: + size = len(text) + if size < 1_000: + return f"{size} chars" + if size < 1_000_000: + return f"{size / 1000:.1f}K chars" + return f"{size / 1_000_000:.1f}M chars" + + def _paste_marker(self, text: str) -> str: + paste_id = self._next_paste_id + self._next_paste_id += 1 + line_count = text.count("\n") + 1 + marker = ( + f"[pasted block #{paste_id}: {self._compact_tokens(text)}, " + f"{line_count} lines; full text will be submitted]" + ) + self._paste_blocks[marker] = _PasteBlock(marker=marker, text=text) + return marker + + def _resolve_paste_blocks(self, text: str) -> str: + """Replace compact paste markers with the original full pasted content.""" + if not self._paste_blocks: + return text + resolved = text + for marker, block in self._paste_blocks.items(): + if marker in resolved: + resolved = resolved.replace(marker, block.text) + self._paste_blocks.clear() + return resolved + + def _insert_paste_text(self, buffer: Any, text: str) -> None: + """Insert pasted text, compacting large blocks to keep terminal rendering smooth.""" + if not text: + return + if self._should_compact_paste(text): + buffer.insert_text(self._paste_marker(text)) + self._invalidate() + return + buffer.insert_text(text) + + def _install_paste_compactor(self, buffer: Any) -> None: + """Compact bulk Buffer inserts even when paste bypasses key bindings.""" + if getattr(buffer, _PASTE_COMPACTOR_ATTR, False): + return + original_insert_text = buffer.insert_text + ref = self + + def insert_text(text: str, *args: Any, **kwargs: Any) -> None: + if isinstance(text, str) and ref._should_compact_paste(text): + original_insert_text(ref._paste_marker(text), *args, **kwargs) + ref._invalidate() + return + original_insert_text(text, *args, **kwargs) + + buffer.insert_text = insert_text + setattr(buffer, _PASTE_COMPACTOR_ATTR, True) + # ── Lifecycle ──────────────────────────────────────────────────── async def run(self) -> int: @@ -164,7 +257,7 @@ async def _process_loop(self) -> None: """Drain pending_input and dispatch to on_input handler.""" while not self._should_exit: try: - text = await asyncio.wait_for( + command = await asyncio.wait_for( self._pending_input.get(), timeout=0.1 ) except asyncio.TimeoutError: @@ -172,19 +265,33 @@ async def _process_loop(self) -> None: except asyncio.CancelledError: return - if not text: + if not command.text.strip(): + self._sync_task_counts() continue + self._active_command = command.mark_running() + self._agent_running = True + self._sync_task_counts() + self._console.command_card(self._active_command) try: if self._on_input is not None: - result = self._on_input(text) + result = self._on_input(command.text) if asyncio.iscoroutine(result): await result + if self._active_command is not None: + finished = self._active_command.mark_done() + self._console.command_card(finished) except Exception as exc: + if self._active_command is not None: + failed = self._active_command.mark_failed(f"{type(exc).__name__}: {exc}") + self._console.command_card(failed) self._console.error(f"{exc}") - self._agent_running = False self._spinner_text = "" self._tool_start_time = 0.0 + finally: + self._active_command = None + self._agent_running = False + self._sync_task_counts() self._invalidate() # ── Layout construction ────────────────────────────────────────── @@ -199,18 +306,19 @@ def get_prompt(): return ref._prompt_fragments() area = TextArea( - height=Dimension(min=1, max=8, preferred=1), + height=Dimension(min=1, max=4, preferred=1), prompt=get_prompt, - style=f"{self._theme.input_text} bold", + style="class:input-area", multiline=True, wrap_lines=True, - read_only=Condition(lambda: ref._agent_running), + dont_extend_height=True, history=FileHistory(str(self._history_path)), completer=completer, complete_while_typing=True, auto_suggest=AutoSuggestFromHistory(), ) area.buffer.tempfile_suffix = ".md" + self._install_paste_compactor(area.buffer) return area def _build_application(self) -> Application[Any]: @@ -257,13 +365,18 @@ def _(event): text = event.app.current_buffer.text.strip() if not text: return - ref._pending_input.put_nowait(text) - event.app.current_buffer.reset(append_to_history=True) + has_compacted_paste = bool(ref._paste_blocks) + ref.submit_text(text) + event.app.current_buffer.reset(append_to_history=not has_compacted_paste) @kb.add(Keys.Escape, Keys.Enter) def _(event): event.current_buffer.insert_text("\n") + @kb.add(Keys.BracketedPaste) + def _(event): + ref._insert_paste_text(event.current_buffer, event.data) + @kb.add(Keys.ControlD) def _(event): ref._should_exit = True @@ -271,16 +384,28 @@ def _(event): @kb.add(Keys.ControlC) def _(event): - if not ref._agent_running: - ref._should_exit = True - event.app.exit() + if event.current_buffer.text: + event.current_buffer.reset() + ref._paste_blocks.clear() + return + if ref._agent_running: + ref._console.system( + "Task is running. Keep typing to queue the next instruction; " + "use /exit after it finishes to leave." + ) + return + ref._should_exit = True + event.app.exit() return kb def _build_style(self) -> PTStyle: t = self._theme + input_style = f"bg:{t.input_bg} {t.input_text} bold" + disabled_style = f"bg:{t.input_bg} {t.input_disabled_text}" return PTStyle.from_dict({ - "input-area": f"{t.input_text} bold", + "input-area": input_style, + "input-area.disabled": disabled_style, "prompt": t.prompt_char, "prompt.working": t.accent_dim, "prompt.recording": t.recording, @@ -294,6 +419,8 @@ def _build_style(self) -> PTStyle: "status-bar.bad": f"bg:{t.toolbar_bg} {t.error}", "hint": t.text_dim, "auto-suggest": t.auto_suggest, + "placeholder": t.input_placeholder, + "selection": f"bg:{t.input_selection_bg} {t.input_selection_fg}", }) # ── Fragment providers ─────────────────────────────────────────── @@ -332,6 +459,12 @@ def _spinner_height(self) -> int: # ── Internal ───────────────────────────────────────────────────── + def _sync_task_counts(self) -> None: + self._status.update_task_counts( + running=1 if self._active_command is not None else 0, + queued=self._pending_input.qsize(), + ) + def _invalidate(self) -> None: if self._app.is_running: self._app.invalidate() diff --git a/src/leapflow/cli/tui_app/command.py b/src/leapflow/cli/tui_app/command.py new file mode 100644 index 0000000..222ba14 --- /dev/null +++ b/src/leapflow/cli/tui_app/command.py @@ -0,0 +1,101 @@ +"""Command lifecycle primitives for the interactive TUI. + +The module is intentionally UI-framework agnostic: it models submitted user +commands and their lifecycle so the prompt_toolkit application, Rich console, +and future schedulers can share the same contract. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, replace +from enum import Enum + +_DEFAULT_SUMMARY_LIMIT = 96 +_MAX_ERROR_LENGTH = 240 + + +def _single_line(text: str) -> str: + return " ".join(text.split()) + + +def _truncate(text: str, limit: int) -> str: + if limit <= 1: + return "…" + return text if len(text) <= limit else text[: limit - 1] + "…" + + +class TuiCommandStatus(str, Enum): + """Lifecycle states for a TUI-submitted command.""" + + QUEUED = "queued" + RUNNING = "running" + DONE = "done" + FAILED = "failed" + + +@dataclass(frozen=True) +class TuiCommand: + """Immutable command request tracked by the TUI scheduler.""" + + id: int + text: str + status: TuiCommandStatus + created_at: float + started_at: float = 0.0 + finished_at: float = 0.0 + error: str = "" + + @classmethod + def create(cls, *, command_id: int, text: str) -> "TuiCommand": + """Create a queued command from user input.""" + return cls( + id=command_id, + text=text, + status=TuiCommandStatus.QUEUED, + created_at=time.monotonic(), + ) + + @property + def label(self) -> str: + """Human-readable command label.""" + return f"#{self.id}" + + @property + def elapsed_s(self) -> float: + """Return command runtime when available, otherwise current age.""" + if self.started_at <= 0: + return 0.0 + end = self.finished_at if self.finished_at > 0 else time.monotonic() + return max(0.0, end - self.started_at) + + def summary(self, *, limit: int = _DEFAULT_SUMMARY_LIMIT) -> str: + """Return a single-line summary suitable for compact task cards.""" + return _truncate(_single_line(self.text), limit) + + def mark_running(self) -> "TuiCommand": + """Return a copy marked as running.""" + return replace( + self, + status=TuiCommandStatus.RUNNING, + started_at=time.monotonic(), + error="", + ) + + def mark_done(self) -> "TuiCommand": + """Return a copy marked as successfully completed.""" + return replace( + self, + status=TuiCommandStatus.DONE, + finished_at=time.monotonic(), + error="", + ) + + def mark_failed(self, error: str) -> "TuiCommand": + """Return a copy marked as failed with a concise error message.""" + return replace( + self, + status=TuiCommandStatus.FAILED, + finished_at=time.monotonic(), + error=_truncate(_single_line(error), _MAX_ERROR_LENGTH), + ) diff --git a/src/leapflow/cli/tui_app/console.py b/src/leapflow/cli/tui_app/console.py index 4cedca2..254f29d 100644 --- a/src/leapflow/cli/tui_app/console.py +++ b/src/leapflow/cli/tui_app/console.py @@ -9,21 +9,24 @@ import os import sys -from typing import Optional, Sequence, Tuple +from typing import Optional from rich.console import Console from rich.markdown import Markdown from rich.panel import Panel from rich.rule import Rule from rich.syntax import Syntax -from rich.table import Table from rich.text import Text from rich.theme import Theme as RichTheme -from leapflow.cli.tui_app.theme import Theme +from leapflow.cli.tui_app.command import TuiCommand, TuiCommandStatus +from leapflow.cli.tui_app.theme import ResolvedTheme, Theme +_COMMAND_CARD_MIN_SUMMARY = 32 +_COMMAND_CARD_PADDING = 24 -def _build_rich_theme(theme: Theme) -> RichTheme: + +def _build_rich_theme(theme: Theme | ResolvedTheme) -> RichTheme: """Map LeapFlow theme to a Rich style dict.""" return RichTheme({ "leap.accent": theme.accent, @@ -48,7 +51,7 @@ class LeapConsole: theming and preventing raw print() calls from breaking layout. """ - def __init__(self, theme: Theme) -> None: + def __init__(self, theme: Theme | ResolvedTheme) -> None: self._theme = theme self._console = Console( theme=_build_rich_theme(theme), @@ -57,7 +60,7 @@ def __init__(self, theme: Theme) -> None: ) @property - def theme(self) -> Theme: + def theme(self) -> Theme | ResolvedTheme: return self._theme @property @@ -77,6 +80,39 @@ def print(self, *args, **kwargs) -> None: """Pass-through to rich.Console.print.""" self._console.print(*args, **kwargs) + def command_card(self, command: TuiCommand) -> None: + """Render a compact lifecycle card for a queued or running command.""" + border_styles = { + TuiCommandStatus.QUEUED: "leap.border", + TuiCommandStatus.RUNNING: "leap.accent", + TuiCommandStatus.DONE: "leap.success", + TuiCommandStatus.FAILED: "leap.error", + } + title_styles = { + TuiCommandStatus.QUEUED: "leap.muted", + TuiCommandStatus.RUNNING: "leap.accent", + TuiCommandStatus.DONE: "leap.success", + TuiCommandStatus.FAILED: "leap.error", + } + summary_limit = max(_COMMAND_CARD_MIN_SUMMARY, self.width - _COMMAND_CARD_PADDING) + body = Text(command.summary(limit=summary_limit), style="leap.muted") + if command.error: + body.append("\n") + body.append(command.error, style="leap.error") + if command.elapsed_s > 0: + body.append("\n") + body.append(f"elapsed: {command.elapsed_s:.1f}s", style="leap.dim") + title = Text() + title.append(command.label, style="bold") + title.append(f" {command.status.value}", style=title_styles[command.status]) + self._console.print(Panel( + body, + title=title, + title_align="left", + border_style=border_styles[command.status], + padding=(0, 1), + )) + def markdown(self, text: str, *, code_theme: str = "monokai") -> None: """Render markdown content with syntax-highlighted code blocks.""" if not text.strip(): diff --git a/src/leapflow/cli/tui_app/session_summary.py b/src/leapflow/cli/tui_app/session_summary.py new file mode 100644 index 0000000..bc9f7ab --- /dev/null +++ b/src/leapflow/cli/tui_app/session_summary.py @@ -0,0 +1,107 @@ +"""Exit summary helpers for interactive TUI sessions.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Iterable, Protocol + + +class ConversationMessageLike(Protocol): + role: str + tool_name: str | None + tool_calls_json: str | None + + +@dataclass +class SessionExitStats: + """Small session-local counters for TUI exit reporting.""" + + started_at: float = field(default_factory=time.monotonic) + user_messages: int = 0 + assistant_messages: int = 0 + tool_calls: int = 0 + + @property + def message_count(self) -> int: + return self.user_messages + self.assistant_messages + + @property + def duration_s(self) -> float: + return max(0.0, time.monotonic() - self.started_at) + + def record_user_message(self) -> None: + self.user_messages += 1 + + def record_assistant_message(self) -> None: + self.assistant_messages += 1 + + def record_tool_calls(self, count: int) -> None: + self.tool_calls += max(0, count) + + +def format_duration(seconds: float) -> str: + """Format seconds as a compact human-readable duration.""" + total = max(0, int(seconds)) + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + if hours > 0: + return f"{hours}h {minutes}m {seconds}s" + if minutes > 0: + return f"{minutes}m {seconds}s" + return f"{seconds}s" + + +def summarize_messages( + messages: Iterable[ConversationMessageLike], +) -> tuple[int, int, int]: + """Return total, user, and tool-call counts from stored messages.""" + total = 0 + user_messages = 0 + tool_calls = 0 + for message in messages: + total += 1 + if message.role == "user": + user_messages += 1 + if message.role == "tool" or message.tool_name or message.tool_calls_json: + tool_calls += 1 + return total, user_messages, tool_calls + + +def build_exit_summary_lines( + *, + session_id: str, + duration_s: float, + message_count: int, + user_messages: int, + tool_calls: int, + resume_command: str = "leap --resume", + resumable: bool = True, +) -> list[str]: + """Build Hermes-style TUI exit summary lines.""" + normalized_session_id = session_id.strip() + if not normalized_session_id: + return ["Goodbye!"] + + stats = [ + f"Session: {normalized_session_id}", + f"Duration: {format_duration(duration_s)}", + ( + f"Messages: {message_count} " + f"({user_messages} user, {tool_calls} tool calls)" + ), + ] + if not resumable: + return [ + "Session not saved:", + " This window used volatile storage because the primary database was locked.", + "", + *stats, + ] + + return [ + "Resume this session with:", + f" {resume_command} {normalized_session_id}", + "", + *stats, + ] diff --git a/src/leapflow/cli/tui_app/status.py b/src/leapflow/cli/tui_app/status.py index 1fe5281..27e1a95 100644 --- a/src/leapflow/cli/tui_app/status.py +++ b/src/leapflow/cli/tui_app/status.py @@ -11,18 +11,25 @@ from __future__ import annotations +import shutil import time from typing import Optional -from leapflow.cli.tui_app.theme import Theme, detect_theme +from leapflow.cli.tui_app.theme import ResolvedTheme, Theme, detect_theme def _compact_tokens(n: int) -> str: - """Format token count: ``12.4K``, ``128K``, ``1.2M``.""" + """Format token count with adaptive precision. + + < 1K → ``0.1K``, ``0.9K`` (1 decimal) + 1–10K → ``1.2K``, ``9.8K`` (1 decimal) + 10–999K → ``12K``, ``256K`` (integer K) + ≥ 1M → ``1.2M`` (1 decimal) + """ if n < 0: return "?" - if n < 1_000: - return str(n) + if n == 0: + return "0" if n < 10_000: return f"{n / 1000:.1f}K" if n < 1_000_000: @@ -42,12 +49,32 @@ def _format_duration(seconds: float) -> str: return f"{hours}h{mins}m" if mins else f"{hours}h" -def _progress_bar(pct: int, width: int = 10) -> str: +def _progress_bar(pct: float, width: int = 10) -> str: """Unicode progress bar: ``████░░░░░░``.""" filled = int(pct / 100 * width) + if pct > 0 and filled == 0: + filled = 1 + filled = min(filled, width) return "█" * filled + "░" * (width - filled) +def _format_percent(used: int, total: int) -> str: + """Format utilization percentage with useful precision at low values.""" + if total <= 0: + return "0%" + pct = max(0.0, min(used * 100 / total, 100.0)) + if 0 < pct < 10: + return f"{pct:.1f}%" + return f"{int(pct)}%" + + +def _truncate(text: str, limit: int) -> str: + """Return text truncated to a single-cell-safe display budget.""" + if limit <= 1: + return "…" + return text if len(text) <= limit else text[: limit - 1] + "…" + + class StatusBar: """Produces status-bar fragments for ``FormattedTextControl``. @@ -55,7 +82,7 @@ class StatusBar: every redraw cycle to get fresh fragments. """ - def __init__(self, theme: Optional[Theme] = None) -> None: + def __init__(self, theme: Optional[Theme | ResolvedTheme] = None) -> None: self._theme = theme or detect_theme() self.mode: str = "idle" self.skill_count: int = 0 @@ -64,6 +91,8 @@ def __init__(self, theme: Optional[Theme] = None) -> None: self.session_turns: int = 0 self.context_used: int = 0 self.context_max: int = 0 + self.running_tasks: int = 0 + self.queued_tasks: int = 0 self.last_turn_elapsed: float = 0.0 self._turn_start: float = 0.0 self._session_start: float = time.monotonic() @@ -81,48 +110,62 @@ def mark_turn_end(self) -> None: def __call__(self) -> list[tuple[str, str]]: """Called by FormattedTextControl on every Application redraw.""" parts: list[tuple[str, str]] = [] + width = shutil.get_terminal_size().columns + compact = width < 72 + narrow = width < 52 _modes = { "idle": ("class:status-bar", " ⚡ "), "learning": ("class:status-bar.bad", " ● "), "paused": ("class:status-bar.warn", " ⏸ "), "executing": ("class:status-bar.good", " ▶ "), + "daemon": ("class:status-bar.good", " ◆ "), } style, icon = _modes.get(self.mode, _modes["idle"]) parts.append((style, icon)) - if self.model_name: - display = self.model_name - if len(display) > 20: - display = display[:18] + "…" + if self.running_tasks or self.queued_tasks: + if narrow: + task_text = f"r{self.running_tasks} q{self.queued_tasks} " + else: + task_text = f"running:{self.running_tasks} queued:{self.queued_tasks} " + parts.append(("class:status-bar.strong", task_text)) + parts.append(("class:status-bar.dim", "│ ")) + + if self.model_name and not narrow: + model_limit = 12 if compact else 20 + display = _truncate(self.model_name, model_limit) parts.append(("class:status-bar.strong", f"{display} ")) parts.append(("class:status-bar.dim", "│ ")) if self.context_max > 0: used = _compact_tokens(self.context_used) total = _compact_tokens(self.context_max) - pct = ( - min(int(self.context_used * 100 / self.context_max), 100) - if self.context_max - else 0 - ) - parts.append(("class:status-bar", f"{used}/{total} ")) - parts.append(("class:status-bar.dim", "│ ")) - - bar_cls = "class:status-bar" - if pct >= 90: - bar_cls = "class:status-bar.bad" - elif pct >= 75: - bar_cls = "class:status-bar.warn" - bar = _progress_bar(pct) - parts.append((bar_cls, f"[{bar}] ")) - parts.append(("class:status-bar", f"{pct}% ")) + pct = min(self.context_used * 100 / self.context_max, 100.0) + pct_text = _format_percent(self.context_used, self.context_max) + if narrow: + parts.append(("class:status-bar", f"{pct_text} ")) + else: + parts.append(("class:status-bar", f"{used}/{total} ")) + parts.append(("class:status-bar.dim", "│ ")) + + bar_cls = "class:status-bar" + if pct >= 90: + bar_cls = "class:status-bar.bad" + elif pct >= 75: + bar_cls = "class:status-bar.warn" + if compact: + parts.append(("class:status-bar", f"{pct_text} ")) + else: + bar = _progress_bar(pct) + parts.append((bar_cls, f"[{bar}] ")) + parts.append(("class:status-bar", f"{pct_text} ")) parts.append(("class:status-bar.dim", "│ ")) elapsed = time.monotonic() - self._session_start parts.append(("class:status-bar", f"{_format_duration(elapsed)} ")) - if self.last_turn_elapsed > 0: + if self.last_turn_elapsed > 0 and not narrow: parts.append(("class:status-bar.dim", "│ ")) if self.last_turn_elapsed < 1.0: e_str = f"{self.last_turn_elapsed * 1000:.0f}ms" @@ -130,8 +173,9 @@ def __call__(self) -> list[tuple[str, str]]: e_str = f"{self.last_turn_elapsed:.1f}s" parts.append(("class:status-bar", f"⏲ {e_str} ")) - parts.append(("class:status-bar.dim", "│ ")) - parts.append(("class:status-bar.good", f"✓ {_format_duration(elapsed)} ")) + if not compact: + parts.append(("class:status-bar.dim", "│ ")) + parts.append(("class:status-bar.good", f"✓ {_format_duration(elapsed)} ")) return parts @@ -145,6 +189,8 @@ def update( session_turns: Optional[int] = None, context_used: Optional[int] = None, context_max: Optional[int] = None, + running_tasks: Optional[int] = None, + queued_tasks: Optional[int] = None, ) -> None: """Selectively update status fields.""" if mode is not None: @@ -161,3 +207,12 @@ def update( self.context_used = context_used if context_max is not None: self.context_max = context_max + if running_tasks is not None: + self.running_tasks = running_tasks + if queued_tasks is not None: + self.queued_tasks = queued_tasks + + def update_task_counts(self, *, running: int, queued: int) -> None: + """Update task counters shown in the status bar.""" + self.running_tasks = max(0, running) + self.queued_tasks = max(0, queued) \ No newline at end of file diff --git a/src/leapflow/cli/tui_app/stream.py b/src/leapflow/cli/tui_app/stream.py index 5ab94dc..588ece2 100644 --- a/src/leapflow/cli/tui_app/stream.py +++ b/src/leapflow/cli/tui_app/stream.py @@ -60,6 +60,10 @@ def text(self) -> str: """Full accumulated response text.""" return self._buffer + @property + def has_output(self) -> bool: + return bool(self._buffer.strip() or self._thinking_buffer.strip()) + @property def elapsed(self) -> float: """Seconds since streaming started.""" diff --git a/src/leapflow/cli/tui_app/theme.py b/src/leapflow/cli/tui_app/theme.py index 3f6cccc..ad6dc27 100644 --- a/src/leapflow/cli/tui_app/theme.py +++ b/src/leapflow/cli/tui_app/theme.py @@ -1,67 +1,145 @@ -"""Adaptive theming with automatic light/dark detection. +"""Adaptive theming with contrast-aware input colors. -Detects terminal background color via environment heuristics and provides -a coherent color palette for the entire TUI. All colors are centralized -here — components never hardcode ANSI sequences. - -The dark theme uses a warm gold/amber palette inspired by premium -terminal tools. The light theme uses dark counterparts for readability. +Detects terminal background color via conservative environment heuristics and +resolves a coherent palette for the entire TUI. Components consume theme +tokens only; no widget should hardcode ANSI sequences or raw colors. """ from __future__ import annotations import os +import re import sys from dataclasses import dataclass +from typing import Mapping + +_HEX_RE = re.compile(r"^#[0-9a-fA-F]{6}$") +_INPUT_TEXT_MIN_CONTRAST = 7.0 +_SECONDARY_TEXT_MIN_CONTRAST = 4.5 +_PROMPT_MIN_CONTRAST = 5.0 + +_DARK_TERMINAL_BG = "#0B1F24" +_LIGHT_TERMINAL_BG = "#F8FAFC" + +_DARK_INPUT_CANDIDATES = ( + "#FFFFFF", + "#F8FAFC", + "#E5E7EB", + "#D1D5DB", +) +_LIGHT_INPUT_CANDIDATES = ( + "#111827", + "#1A1A1A", + "#000000", + "#374151", +) +_PLACEHOLDER_CANDIDATES = ( + "#94A3B8", + "#CBD5E1", + "#64748B", + "#475569", + "#808080", +) +_PROMPT_CANDIDATES = ( + "#FFD700", + "#FACC15", + "#FFBF00", + "#CC6600", + "#996600", + "#8A4B00", + "#78350F", + "#FFFFFF", + "#111827", +) +_BORDER_CANDIDATES = ( + "#CD7F32", + "#B45309", + "#996600", + "#8A4B00", + "#78350F", + "#64748B", + "#475569", + "#FFFFFF", + "#111827", +) +_TEXT_CANDIDATES = ( + "#FFF8DC", + "#F8FAFC", + "#E5E7EB", + "#111827", + "#1A1A1A", + "#000000", +) +_MUTED_TEXT_CANDIDATES = ( + "#B8860B", + "#94A3B8", + "#CBD5E1", + "#64748B", + "#475569", + "#808080", + "#78350F", +) +_ANSI_BG = { + 0: "#000000", + 1: "#800000", + 2: "#008000", + 3: "#808000", + 4: "#000080", + 5: "#800080", + 6: "#008080", + 7: "#C0C0C0", + 8: "#808080", + 9: "#FF0000", + 10: "#00FF00", + 11: "#FFFF00", + 12: "#0000FF", + 13: "#FF00FF", + 14: "#00FFFF", + 15: "#FFFFFF", +} @dataclass(frozen=True) class Theme: - """Immutable color palette consumed by all TUI components.""" + """Immutable base color palette consumed by TUI components.""" - # Identity name: str - - # Primary accent (brand color) accent: str accent_dim: str - - # Semantic colors success: str warning: str error: str info: str - - # Text hierarchy text: str text_dim: str text_muted: str - - # Structural border: str border_dim: str panel_title: str - - # Status indicators recording: str executing: str - - # Code code_bg: str - - # Prompt prompt_char: str - - # Input line - input_text: str # Default input text color (adaptive white) - - # Bottom toolbar (status bar) + input_text: str + input_bg: str + input_placeholder: str + input_border: str + input_focus_border: str + input_selection_bg: str + input_selection_fg: str + input_disabled_text: str toolbar_bg: str toolbar_fg: str + prompt_paused: str + auto_suggest: str - # Prompt modes - prompt_paused: str # paused mode color - auto_suggest: str # autosuggestion ghost text color + +@dataclass(frozen=True) +class ResolvedTheme(Theme): + """Theme after terminal-background and contrast normalization.""" + + terminal_bg: str + base_name: str _DARK = Theme( @@ -83,70 +161,357 @@ class Theme: code_bg="#1c1c1c", prompt_char="bold #FFD700", input_text="#FFFFFF", + input_bg=_DARK_TERMINAL_BG, + input_placeholder="#94A3B8", + input_border="#8B6914", + input_focus_border="#FFD700", + input_selection_bg="#334155", + input_selection_fg="#FFFFFF", + input_disabled_text="#94A3B8", toolbar_bg="#2a2418", toolbar_fg="#B8860B", prompt_paused="bold #FFD700", - auto_suggest="#8B8682", + auto_suggest="#94A3B8", ) _LIGHT = Theme( name="light", - accent="#996600", - accent_dim="#B8860B", + accent="#8A4B00", + accent_dim="#7A3E00", success="#007700", - warning="#996600", + warning="#8A4B00", error="#cc0000", info="#0055cc", - text="black", - text_dim="#8B6914", - text_muted="#808080", - border="#CD7F32", - border_dim="#808080", - panel_title="bold #996600", + text="#111827", + text_dim="#475569", + text_muted="#64748B", + border="#8A4B00", + border_dim="#64748B", + panel_title="bold #8A4B00", recording="bold #cc0000", executing="bold #007700", code_bg="#ededed", - prompt_char="bold #996600", + prompt_char="bold #8A4B00", input_text="#1A1A1A", - toolbar_bg="#f0e8d8", - toolbar_fg="#8B6914", - prompt_paused="bold #996600", - auto_suggest="#808080", + input_bg=_LIGHT_TERMINAL_BG, + input_placeholder="#64748B", + input_border="#8A4B00", + input_focus_border="#78350F", + input_selection_bg="#D6E4FF", + input_selection_fg="#111827", + input_disabled_text="#64748B", + toolbar_bg=_LIGHT_TERMINAL_BG, + toolbar_fg="#374151", + prompt_paused="bold #8A4B00", + auto_suggest="#64748B", ) -def detect_light_mode() -> bool: - """Heuristic detection of light terminal background. +def parse_hex_color(value: str) -> tuple[int, int, int]: + """Parse a #RRGGBB color into RGB channels.""" + if not _HEX_RE.match(value): + raise ValueError(f"Expected #RRGGBB color, got {value!r}") + return int(value[1:3], 16), int(value[3:5], 16), int(value[5:7], 16) - Checks (in order): - 1. ``LEAPFLOW_TUI_THEME`` env var (``light`` or ``dark``) - 2. ``COLORFGBG`` (set by many terminals: ``fg;bg`` where bg>=8 is dark) - 3. macOS Terminal.app defaults to light - 4. Default: dark - """ - explicit = os.environ.get("LEAPFLOW_TUI_THEME", "").lower() - if explicit == "light": - return True - if explicit == "dark": - return False - colorfgbg = os.environ.get("COLORFGBG", "") +def relative_luminance(color: str) -> float: + """Return WCAG relative luminance for a #RRGGBB color.""" + channels = [] + for channel in parse_hex_color(color): + value = channel / 255 + if value <= 0.03928: + channels.append(value / 12.92) + else: + channels.append(((value + 0.055) / 1.055) ** 2.4) + red, green, blue = channels + return 0.2126 * red + 0.7152 * green + 0.0722 * blue + + +def contrast_ratio(foreground: str, background: str) -> float: + """Return WCAG contrast ratio between two #RRGGBB colors.""" + fg_luminance = relative_luminance(foreground) + bg_luminance = relative_luminance(background) + lighter = max(fg_luminance, bg_luminance) + darker = min(fg_luminance, bg_luminance) + return (lighter + 0.05) / (darker + 0.05) + + +def is_light_color(color: str) -> bool: + """Return True when the color is perceived as a light background.""" + return relative_luminance(color) >= 0.5 + + +def ensure_contrast( + preferred: str, + background: str, + *, + min_ratio: float, + candidates: tuple[str, ...], +) -> str: + """Choose the preferred color or the closest candidate that is readable.""" + valid_candidates = (preferred, *candidates) + best_color = preferred + best_ratio = -1.0 + for color in valid_candidates: + try: + ratio = contrast_ratio(color, background) + except ValueError: + continue + if ratio >= min_ratio: + return color + if ratio > best_ratio: + best_color = color + best_ratio = ratio + return best_color + + +def _style_color(style: str) -> str | None: + for part in reversed(style.split()): + if _HEX_RE.match(part): + return part + return None + + +def _with_style_color(style: str, color: str) -> str: + parts = style.split() + replaced = False + next_parts: list[str] = [] + for part in parts: + if _HEX_RE.match(part): + next_parts.append(color) + replaced = True + else: + next_parts.append(part) + if not replaced: + next_parts.append(color) + return " ".join(next_parts) + + +def _env_value(env: Mapping[str, str] | None, key: str) -> str: + source = os.environ if env is None else env + return source.get(key, "") + + +def _explicit_theme_name(env: Mapping[str, str] | None = None) -> str: + explicit = _env_value(env, "LEAPFLOW_TUI_THEME").strip().lower() + return explicit if explicit in {"light", "dark"} else "" + + +def _background_from_env(env: Mapping[str, str] | None = None) -> str | None: + explicit_bg = _env_value(env, "LEAPFLOW_TUI_BG").strip() + if explicit_bg: + if _HEX_RE.match(explicit_bg): + return explicit_bg.upper() + return None + + colorfgbg = _env_value(env, "COLORFGBG") if ";" in colorfgbg: try: - bg = int(colorfgbg.rsplit(";", 1)[1]) - return bg >= 8 or bg == 7 + bg_index = int(colorfgbg.rsplit(";", 1)[1]) except (ValueError, IndexError): - pass + return None + return _ANSI_BG.get(bg_index) - term_program = os.environ.get("TERM_PROGRAM", "") - if term_program == "Apple_Terminal": - return True + return None + + +def _looks_light_from_name(value: str) -> bool: + normalized = value.strip().lower() + if not normalized: + return False + light_markers = ("light", "day", "paper", "latte", "cream", "solarized-light") + dark_markers = ("dark", "night", "black", "dim", "moon", "dracula") + if any(marker in normalized for marker in dark_markers): + return False + return any(marker in normalized for marker in light_markers) + + +def detect_light_mode(env: Mapping[str, str] | None = None) -> bool: + """Conservatively detect whether the terminal background is light.""" + explicit = _explicit_theme_name(env) + if explicit: + return explicit == "light" + + background = _background_from_env(env) + if background is not None: + return is_light_color(background) + + for key in ("ITERM_PROFILE", "TERMINAL_PROFILE", "WT_PROFILE_ID"): + if _looks_light_from_name(_env_value(env, key)): + return True return False -def detect_theme() -> Theme: - """Return the appropriate theme based on terminal environment.""" - if not sys.stdout.isatty(): +def _select_base_theme(env: Mapping[str, str] | None = None) -> Theme: + explicit = _explicit_theme_name(env) + if explicit == "light": + return _LIGHT + if explicit == "dark": return _DARK - return _LIGHT if detect_light_mode() else _DARK + return _LIGHT if detect_light_mode(env) else _DARK + + +def resolve_theme( + base: Theme, + *, + env: Mapping[str, str] | None = None, + terminal_bg: str | None = None, +) -> ResolvedTheme: + """Resolve a base theme into contrast-safe TUI colors.""" + background = terminal_bg or _background_from_env(env) or base.input_bg + if not _HEX_RE.match(background): + background = base.input_bg + background = background.upper() + light_background = is_light_color(background) + surface_bg = background if light_background else base.toolbar_bg + input_candidates = _LIGHT_INPUT_CANDIDATES if light_background else _DARK_INPUT_CANDIDATES + input_text = ensure_contrast( + base.input_text, + background, + min_ratio=_INPUT_TEXT_MIN_CONTRAST, + candidates=input_candidates, + ) + input_placeholder = ensure_contrast( + base.input_placeholder, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_PLACEHOLDER_CANDIDATES, + ) + auto_suggest = ensure_contrast( + base.auto_suggest, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_PLACEHOLDER_CANDIDATES, + ) + input_disabled_text = ensure_contrast( + base.input_disabled_text, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_PLACEHOLDER_CANDIDATES, + ) + text = ensure_contrast( + base.text, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_TEXT_CANDIDATES, + ) + text_dim = ensure_contrast( + base.text_dim, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_MUTED_TEXT_CANDIDATES, + ) + text_muted = ensure_contrast( + base.text_muted, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_MUTED_TEXT_CANDIDATES, + ) + accent = ensure_contrast( + base.accent, + background, + min_ratio=_PROMPT_MIN_CONTRAST, + candidates=_PROMPT_CANDIDATES, + ) + accent_dim = ensure_contrast( + base.accent_dim, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_BORDER_CANDIDATES, + ) + border = ensure_contrast( + base.border, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_BORDER_CANDIDATES, + ) + border_dim = ensure_contrast( + base.border_dim, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_BORDER_CANDIDATES, + ) + input_border = ensure_contrast( + base.input_border, + background, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_BORDER_CANDIDATES, + ) + toolbar_fg = ensure_contrast( + base.toolbar_fg, + surface_bg, + min_ratio=_SECONDARY_TEXT_MIN_CONTRAST, + candidates=_MUTED_TEXT_CANDIDATES, + ) + prompt_color = ensure_contrast( + _style_color(base.prompt_char) or base.accent, + background, + min_ratio=_PROMPT_MIN_CONTRAST, + candidates=_PROMPT_CANDIDATES, + ) + prompt_paused_color = ensure_contrast( + _style_color(base.prompt_paused) or base.warning, + background, + min_ratio=_PROMPT_MIN_CONTRAST, + candidates=_PROMPT_CANDIDATES, + ) + focus_border = ensure_contrast( + base.input_focus_border, + background, + min_ratio=_PROMPT_MIN_CONTRAST, + candidates=_BORDER_CANDIDATES, + ) + panel_title_color = ensure_contrast( + _style_color(base.panel_title) or accent, + background, + min_ratio=_PROMPT_MIN_CONTRAST, + candidates=_PROMPT_CANDIDATES, + ) + + return ResolvedTheme( + name=base.name, + accent=accent, + accent_dim=accent_dim, + success=base.success, + warning=base.warning, + error=base.error, + info=base.info, + text=text, + text_dim=text_dim, + text_muted=text_muted, + border=border, + border_dim=border_dim, + panel_title=_with_style_color(base.panel_title, panel_title_color), + recording=base.recording, + executing=base.executing, + code_bg=base.code_bg, + prompt_char=_with_style_color(base.prompt_char, prompt_color), + input_text=input_text, + input_bg=background, + input_placeholder=input_placeholder, + input_border=input_border, + input_focus_border=focus_border, + input_selection_bg=base.input_selection_bg, + input_selection_fg=base.input_selection_fg, + input_disabled_text=input_disabled_text, + toolbar_bg=surface_bg, + toolbar_fg=toolbar_fg, + prompt_paused=_with_style_color(base.prompt_paused, prompt_paused_color), + auto_suggest=auto_suggest, + terminal_bg=background, + base_name=base.name, + ) + + +def detect_theme( + env: Mapping[str, str] | None = None, + *, + is_tty: bool | None = None, +) -> ResolvedTheme: + """Return a contrast-safe theme based on terminal environment.""" + tty = sys.stdout.isatty() if is_tty is None else is_tty + has_override = bool(_explicit_theme_name(env) or _background_from_env(env)) + if not tty and not has_override: + return resolve_theme(_DARK, env=env, terminal_bg=_DARK.input_bg) + return resolve_theme(_select_base_theme(env), env=env) diff --git a/src/leapflow/config.py b/src/leapflow/config.py index 81d1e26..a9d0563 100644 --- a/src/leapflow/config.py +++ b/src/leapflow/config.py @@ -12,6 +12,7 @@ import logging import os +import re import shutil import sys from dataclasses import dataclass, field @@ -25,6 +26,19 @@ logger = logging.getLogger(__name__) +DEFAULT_LLM_CONTEXT_LENGTH = 256_000 +_PROFILE_NAME_RE = re.compile(r"^[A-Za-z0-9_-]+$") + + +def _validate_profile_name(profile: str) -> str: + """Return a safe profile name or raise before any path construction.""" + normalized = (profile or "default").strip() + if not _PROFILE_NAME_RE.fullmatch(normalized): + raise ValueError( + "Invalid LEAPFLOW_PROFILE; use only letters, numbers, underscores, or dashes" + ) + return normalized + ALL_SIGNAL_CHANNELS = frozenset({ "click", "app_switch", "clipboard", "clipboard_content", "keyboard", "scroll", "drag", @@ -35,25 +49,41 @@ def _expand_path(value: str) -> Path: return Path(value.replace("~", str(Path.home()))).expanduser() -# Standard subdirectories created under the data directory on first run. -_STANDARD_SUBDIRS = [ - "var", +# Global subdirectories (shared across profiles). +_GLOBAL_SUBDIRS = [ + "logs", +] + +# Per-profile subdirectories — created under profiles//. +_PROFILE_SUBDIRS = [ + "db", + "memory", + "memory/global", + "skills", "cache", "cache/frames", "cache/video", - "skills", + "run", + "gateway", + "gateway/manifests", ] -def ensure_data_dir(data_dir: Path) -> None: +def ensure_data_dir(data_dir: Path, *, profile: str = "default") -> None: """Create the LeapFlow data directory and standard subdirectories. - Idempotent — safe to call on every startup. + Idempotent — safe to call on every startup. Creates both global + directories and profile-specific directories under ``profiles//``. """ + profile = _validate_profile_name(profile) data_dir.mkdir(parents=True, exist_ok=True) - for sub in _STANDARD_SUBDIRS: + for sub in _GLOBAL_SUBDIRS: (data_dir / sub).mkdir(parents=True, exist_ok=True) + profile_dir = data_dir / "profiles" / profile + for sub in _PROFILE_SUBDIRS: + (profile_dir / sub).mkdir(parents=True, exist_ok=True) + def ensure_default_env(data_dir: Path) -> bool: """Create a default ``.env`` in *data_dir* if one does not already exist. @@ -72,12 +102,14 @@ def ensure_default_env(data_dir: Path) -> bool: if sys.stderr.isatty(): sys.stderr.write( f"\033[2m→ Created default config: {env_path}\033[0m\n" - f"\033[2m Edit LEAPFLOW_LLM_API_KEY to configure your LLM provider.\033[0m\n" + f"\033[2m Edit LEAPFLOW_LLM_API_KEY in this file to configure your LLM provider.\033[0m\n" + f"\033[2m Optional project override: ./.env in your working directory.\033[0m\n" ) else: sys.stderr.write( f"→ Created default config: {env_path}\n" - f" Edit LEAPFLOW_LLM_API_KEY to configure your LLM provider.\n" + f" Edit LEAPFLOW_LLM_API_KEY in this file to configure your LLM provider.\n" + f" Optional project override: ./.env in your working directory.\n" ) sys.stderr.flush() return True @@ -104,11 +136,12 @@ class Settings: memory_prefetch_timeout_s: float = 2.0 # max wait for prefetch in PREPARING memory_prefetch_limit: int = 5 # max entries injected per turn - # ── Data Root ── + # ── Data Root & Profile ── data_dir: Path = Path("~/.leapflow") + profile: str = "default" # Audit - audit_log_path: Path = Path("~/.leapflow/audit.jsonl") + audit_log_path: Path = Path("~/.leapflow/profiles/default/audit.jsonl") # Imitation Learning pattern_library_path: str = "" # Custom patterns.yaml path (empty = use default) @@ -132,13 +165,13 @@ class Settings: confirm_default_level: str = "confirm" # Default confirmation level # Skill Documents (SKILL.md) - skills_dir: Path = Path("~/.leapflow/skills/") + skills_dir: Path = Path("~/.leapflow/profiles/default/skills/") skill_view_max_chars: int = 5000 # Max chars returned by skill_view tool skill_min_quality: float = 0.5 # SkillIndex quality threshold # Visual Track visual_track_enabled: bool = False - visual_frame_cache_dir: Path = Path("~/.leapflow/cache/frames") + visual_frame_cache_dir: Path = Path("~/.leapflow/profiles/default/cache/frames") visual_sample_strategy: str = "keyframe" # keyframe | periodic | all vlm_model: str = "" # 空则复用 llm_model vlm_api_key: str = "" # 空则复用 llm_api_key @@ -181,7 +214,7 @@ class Settings: # ── Perceptual Field ── perceptual_field_enabled: bool = False - perceptual_field_config: str = "~/.leapflow/perceptual_fields.yaml" + perceptual_field_config: str = "~/.leapflow/profiles/default/perceptual_fields.yaml" # ── Context Learning Attention ── attention_foreground_gate: bool = True @@ -256,7 +289,7 @@ class Settings: video_resolution_scale: float = 0.75 video_codec: str = "h264" video_max_segment_s: int = 600 - video_cache_dir: Path = Path("~/.leapflow/cache/video") + video_cache_dir: Path = Path("~/.leapflow/profiles/default/cache/video") video_cache_max_age_days: int = 7 # 视频缓存最大保留天数 video_cache_max_size_gb: float = 5.0 # 视频缓存最大占用空间(GB) video_l2_enabled: bool = True @@ -321,7 +354,7 @@ class Settings: llm_aux_model: str = "" # Auxiliary model for cheap operations (empty = reuse primary) llm_aux_api_key: str = "" # Aux API key (empty = reuse primary) llm_aux_base_url: str = "" # Aux base URL (empty = reuse primary) - llm_context_length: int = 128_000 # Primary provider's context window + llm_context_length: int = DEFAULT_LLM_CONTEXT_LENGTH # Primary provider's runtime context budget llm_credential_cooldown_s: float = 60.0 # Per-key rate-limit cooldown # ── Stream & Tool Robustness ── @@ -382,10 +415,20 @@ class Settings: scheduler_grace_seconds: float = 120.0 scheduler_default_tier: str = "auto" # auto | local | cloud + @property + def profile_dir(self) -> Path: + """Root directory for the active profile.""" + return self.data_dir / "profiles" / self.profile + @property def has_llm_credentials(self) -> bool: return bool(self.llm_api_key.strip()) + @property + def has_vlm_credentials(self) -> bool: + """Return True when visual perception can call a VLM provider.""" + return bool((self.vlm_api_key or self.llm_api_key).strip()) + def _deep_merge(base: Dict[str, Any], overlay: Dict[str, Any]) -> Dict[str, Any]: """Recursively merge overlay into base. Overlay values take precedence.""" @@ -419,7 +462,7 @@ def _load_yaml_overlay(data_dir: Path) -> Dict[str, str]: if not isinstance(parsed, dict): raise ValueError("config.yaml root must be a mapping") except Exception as exc: - backup = yaml_path.with_suffix(f".yaml.corrupt.bak") + backup = yaml_path.with_suffix(".yaml.corrupt.bak") logger.warning("config.yaml parse error (%s), backing up to %s", exc, backup.name) try: shutil.copy2(yaml_path, backup) @@ -475,7 +518,8 @@ def load_config(*, env_file: str | Path | None = None) -> Settings: _data_dir = _expand_path(_data_dir_raw) # Bootstrap: create directory structure + default .env on first run. - ensure_data_dir(_data_dir) + _profile = _validate_profile_name(os.getenv("LEAPFLOW_PROFILE", "default")) + ensure_data_dir(_data_dir, profile=_profile) ensure_default_env(_data_dir) if env_file is None: @@ -514,9 +558,11 @@ def _build_settings_from_env() -> Settings: max_retries = int(os.getenv("LEAPFLOW_LLM_MAX_RETRIES", "3")) data_dir = _expand_path(os.getenv("LEAPFLOW_DATA_DIR", "~/.leapflow").strip()) + profile = _validate_profile_name(os.getenv("LEAPFLOW_PROFILE", "default")) + _profile_dir = data_dir / "profiles" / profile mock_host = os.getenv("LEAPFLOW_MOCK_HOST", "0").strip() in ("1", "true", "True", "yes") - duckdb = os.getenv("LEAPFLOW_DUCKDB_PATH", "~/.leapflow/memory.duckdb").strip() + duckdb = os.getenv("LEAPFLOW_DUCKDB_PATH", str(_profile_dir / "db" / "leap.duckdb")).strip() log_level = os.getenv("LEAPFLOW_LOG_LEVEL", "INFO").strip() # Memory Providers @@ -529,11 +575,11 @@ def _build_settings_from_env() -> Settings: memory_prefetch_limit = int(os.getenv("LEAPFLOW_MEMORY_PREFETCH_LIMIT", "5")) # Audit - audit_log_path = os.getenv("LEAPFLOW_AUDIT_LOG_PATH", "~/.leapflow/audit.jsonl").strip() + audit_log_path = os.getenv("LEAPFLOW_AUDIT_LOG_PATH", str(_profile_dir / "audit.jsonl")).strip() # Visual Track - visual_track_enabled = os.getenv("LEAPFLOW_VISUAL_TRACK_ENABLED", "1").strip() in ("1", "true", "True", "yes") - visual_frame_cache_dir = os.getenv("LEAPFLOW_VISUAL_FRAME_CACHE_DIR", "~/.leapflow/cache/frames").strip() + visual_track_enabled = os.getenv("LEAPFLOW_VISUAL_TRACK_ENABLED", "0").strip() in ("1", "true", "True", "yes") + visual_frame_cache_dir = os.getenv("LEAPFLOW_VISUAL_FRAME_CACHE_DIR", str(_profile_dir / "cache" / "frames")).strip() visual_sample_strategy = os.getenv("LEAPFLOW_VISUAL_SAMPLE_STRATEGY", "keyframe").strip() vlm_model = os.getenv("LEAPFLOW_VLM_MODEL", "").strip() or model vlm_api_key = os.getenv("LEAPFLOW_VLM_API_KEY", "").strip() @@ -542,7 +588,7 @@ def _build_settings_from_env() -> Settings: privacy_sensitive_apps = tuple(b.strip() for b in privacy_sensitive_apps_raw.split(",") if b.strip()) if privacy_sensitive_apps_raw else () # Skill Documents - skills_dir = os.getenv("LEAPFLOW_SKILLS_DIR", "~/.leapflow/skills/").strip() + skills_dir = os.getenv("LEAPFLOW_SKILLS_DIR", str(_profile_dir / "skills")).strip() skill_view_max_chars = int(os.getenv("LEAPFLOW_SKILL_VIEW_MAX_CHARS", "5000")) skill_min_quality = float(os.getenv("LEAPFLOW_SKILL_MIN_QUALITY", "0.5")) @@ -590,7 +636,7 @@ def _build_settings_from_env() -> Settings: # Perceptual Field perceptual_field_enabled = _bool("LEAPFLOW_PERCEPTUAL_FIELD_ENABLED", "false") perceptual_field_config = os.getenv( - "LEAPFLOW_PERCEPTUAL_FIELD_CONFIG", "~/.leapflow/perceptual_fields.yaml" + "LEAPFLOW_PERCEPTUAL_FIELD_CONFIG", str(_profile_dir / "perceptual_fields.yaml") ).strip() # Context Learning Attention @@ -668,7 +714,7 @@ def _build_settings_from_env() -> Settings: video_resolution_scale = float(os.getenv("LEAPFLOW_VIDEO_RESOLUTION_SCALE", "0.75")) video_codec = os.getenv("LEAPFLOW_VIDEO_CODEC", "h264").strip() video_max_segment_s = int(os.getenv("LEAPFLOW_VIDEO_MAX_SEGMENT_S", "600")) - video_cache_dir = os.getenv("LEAPFLOW_VIDEO_CACHE_DIR", "~/.leapflow/cache/video").strip() + video_cache_dir = os.getenv("LEAPFLOW_VIDEO_CACHE_DIR", str(_profile_dir / "cache" / "video")).strip() video_cache_max_age_days = int(os.getenv("LEAPFLOW_VIDEO_CACHE_MAX_AGE_DAYS", "7")) video_cache_max_size_gb = float(os.getenv("LEAPFLOW_VIDEO_CACHE_MAX_SIZE_GB", "5.0")) video_l2_enabled = _bool("LEAPFLOW_VIDEO_L2_ENABLED", "true") @@ -731,7 +777,7 @@ def _build_settings_from_env() -> Settings: llm_aux_model = os.getenv("LEAPFLOW_LLM_AUX_MODEL", "").strip() llm_aux_api_key = os.getenv("LEAPFLOW_LLM_AUX_API_KEY", "").strip() llm_aux_base_url = os.getenv("LEAPFLOW_LLM_AUX_BASE_URL", "").strip() - llm_context_length = int(os.getenv("LEAPFLOW_LLM_CONTEXT_LENGTH", "128000")) + llm_context_length = int(os.getenv("LEAPFLOW_LLM_CONTEXT_LENGTH", str(DEFAULT_LLM_CONTEXT_LENGTH))) llm_credential_cooldown_s = float(os.getenv("LEAPFLOW_LLM_CREDENTIAL_COOLDOWN_S", "60.0")) # Stream & Tool Robustness @@ -824,6 +870,7 @@ def _build_settings_from_env() -> Settings: memory_prefetch_timeout_s=memory_prefetch_timeout_s, memory_prefetch_limit=memory_prefetch_limit, data_dir=data_dir, + profile=profile, audit_log_path=_expand_path(audit_log_path), skills_dir=_expand_path(skills_dir), skill_view_max_chars=skill_view_max_chars, diff --git a/src/leapflow/daemon/__init__.py b/src/leapflow/daemon/__init__.py new file mode 100644 index 0000000..f2be332 --- /dev/null +++ b/src/leapflow/daemon/__init__.py @@ -0,0 +1,12 @@ +"""LeapFlow daemon (leapd) — centralized process for DuckDB + runtime. + +The daemon architecture follows the "single process owns all mutable state" +principle. In single-window mode, the daemon runs in-process (zero IPC +overhead). In multi-window mode, a detached leapd process is spawned and +CLI instances connect as thin clients over Unix socket. + +Submodules: + +- ``protocol``: JSON-RPC message types and ``LeapService`` interface +- ``lifecycle``: PID/lock file management, health checks, process spawning +""" diff --git a/src/leapflow/daemon/client.py b/src/leapflow/daemon/client.py new file mode 100644 index 0000000..c3ce69b --- /dev/null +++ b/src/leapflow/daemon/client.py @@ -0,0 +1,222 @@ +"""Thin client for connecting LeapFlow CLI processes to leapd.""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +from collections.abc import AsyncIterator, Callable +from pathlib import Path +from typing import Any + +from leapflow.daemon.lifecycle import ( + DaemonInfo, + DaemonLock, + cleanup_stale, + spawn_daemon, + wait_ready, +) +from leapflow.daemon.protocol import RpcRequest +from leapflow.engine import StreamEvent + +logger = logging.getLogger(__name__) + +StatusCallback = Callable[[str], None] + + +class DaemonUnavailableError(RuntimeError): + """Raised when a usable leapd daemon cannot be reached.""" + + +class DaemonClient: + """Small JSON-RPC client that opens one Unix socket per request.""" + + def __init__(self, sock_path: Path, *, timeout_s: float = 30.0) -> None: + self._sock_path = sock_path + self._timeout_s = timeout_s + + @property + def sock_path(self) -> Path: + """Return the Unix socket path used by this client.""" + return self._sock_path + + async def request(self, method: str, params: dict[str, Any] | None = None) -> Any: + """Send one non-streaming JSON-RPC request and return its result.""" + request = RpcRequest(method=method, params=params or {}) + reader, writer = await self._open() + try: + await _send(writer, request.to_json()) + while True: + payload = await self._read_payload(reader) + if payload.get("id") != request.id: + continue + if "error" in payload: + raise DaemonUnavailableError(_format_rpc_error(payload["error"])) + return payload.get("result") + finally: + await _close_writer(writer) + + async def engine_chat( + self, + message: str, + *, + enable_thinking: bool = False, + ) -> AsyncIterator[StreamEvent]: + """Stream chat events from the daemon-owned AgentEngine.""" + request = RpcRequest( + method="engine.chat", + params={"message": message, "enable_thinking": enable_thinking}, + ) + reader, writer = await self._open() + try: + await _send(writer, request.to_json()) + while True: + payload = await self._read_payload(reader) + method = payload.get("method") + params = dict(payload.get("params") or {}) + if method == "stream.chunk" and params.get("id") == request.id: + if params.get("done"): + continue + yield _event_from_params(params) + continue + if payload.get("id") == request.id: + if "error" in payload: + raise DaemonUnavailableError(_format_rpc_error(payload["error"])) + break + finally: + await _close_writer(writer) + + async def session_resume(self, session_id: str) -> dict[str, Any]: + """Ask the daemon to load an existing conversation session.""" + result = await self.request("session.resume", {"session_id": session_id}) + return dict(result or {}) + + async def status(self) -> dict[str, Any]: + """Return daemon status.""" + result = await self.request("daemon.status") + return dict(result or {}) + + async def shutdown(self) -> None: + """Request graceful daemon shutdown.""" + await self.request("daemon.shutdown") + + async def _open(self) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + try: + return await asyncio.wait_for( + asyncio.open_unix_connection(str(self._sock_path)), + timeout=self._timeout_s, + ) + except (TimeoutError, OSError) as exc: + raise DaemonUnavailableError( + f"Cannot connect to leapd at {self._sock_path}: {exc}" + ) from exc + + async def _read_payload(self, reader: asyncio.StreamReader) -> dict[str, Any]: + try: + raw = await asyncio.wait_for(reader.readline(), timeout=self._timeout_s) + except TimeoutError as exc: + raise DaemonUnavailableError("Timed out waiting for leapd response") from exc + if not raw: + raise DaemonUnavailableError("leapd closed the connection unexpectedly") + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DaemonUnavailableError("Received invalid JSON from leapd") from exc + if not isinstance(payload, dict): + raise DaemonUnavailableError("Received invalid JSON-RPC payload from leapd") + return payload + + +async def ensure_daemon_client( + settings: Any, + *, + mock_host: bool = False, + status_callback: StatusCallback | None = None, +) -> DaemonClient: + """Return a client connected to a healthy daemon, starting one if needed.""" + run_dir = settings.profile_dir / "run" + sock_path = run_dir / "leapd.sock" + info = DaemonInfo.discover(run_dir) + if info.is_healthy: + _emit(status_callback, f"Connected to leapd (pid={info.pid}).") + return DaemonClient(sock_path) + + if info.pid is not None and not info.is_running: + cleanup_stale(run_dir) + elif info.is_running and not info.is_healthy: + raise DaemonUnavailableError( + f"leapd is running but unhealthy (pid={info.pid}). " + "Run 'leap daemon stop' and retry." + ) + + lock = DaemonLock(run_dir / "leapd.lock") + if lock.acquire(): + try: + _emit(status_callback, "Starting leapd daemon...") + spawn_daemon(settings, mock_host=mock_host) + finally: + lock.release() + else: + _emit(status_callback, "Waiting for leapd daemon...") + + ready = wait_ready(run_dir, timeout_s=_daemon_start_timeout()) + if not ready.is_healthy: + raise DaemonUnavailableError( + "leapd did not become ready. Run 'leap daemon status' for details." + ) + _emit(status_callback, f"Connected to leapd (pid={ready.pid}).") + return DaemonClient(sock_path) + + +def _event_from_params(params: dict[str, Any]) -> StreamEvent: + event_type = str(params.get("event_type") or "chunk") + if event_type not in { + "chunk", + "final", + "tool_start", + "tool_complete", + "thinking", + "status", + "error", + }: + event_type = "chunk" + metadata = params.get("metadata") + return StreamEvent( + type=event_type, # type: ignore[arg-type] + content=str(params.get("content") or ""), + metadata=dict(metadata) if isinstance(metadata, dict) else None, + ) + + +def _format_rpc_error(error: object) -> str: + if isinstance(error, dict): + message = str(error.get("message") or "Daemon request failed") + data = error.get("data") + return f"{message}: {data}" if data else message + return str(error) + + +def _emit(callback: StatusCallback | None, message: str) -> None: + if callback is not None: + callback(message) + + +def _daemon_start_timeout() -> float: + raw = os.getenv("LEAPFLOW_DAEMON_START_TIMEOUT", "30").strip() + try: + return max(1.0, float(raw)) + except ValueError: + return 30.0 + + +async def _send(writer: asyncio.StreamWriter, text: str) -> None: + writer.write(text.encode("utf-8") + b"\n") + await writer.drain() + + +async def _close_writer(writer: asyncio.StreamWriter) -> None: + writer.close() + try: + await writer.wait_closed() + except (BrokenPipeError, ConnectionError, OSError): + logger.debug("daemon client: socket closed with transport error", exc_info=True) diff --git a/src/leapflow/daemon/lifecycle.py b/src/leapflow/daemon/lifecycle.py new file mode 100644 index 0000000..8100e9a --- /dev/null +++ b/src/leapflow/daemon/lifecycle.py @@ -0,0 +1,269 @@ +"""Daemon lifecycle management — PID files, lock files, health checks. + +Handles the leapd daemon lifecycle: + +1. **Discovery** — check if a daemon is running for a given profile +2. **Lock acquisition** — ``fcntl.flock`` on ``leapd.lock`` for leader election +3. **PID file** — ``leapd.pid`` tracks the daemon process +4. **Health check** — probe ``leapd.sock`` for liveness +5. **Stale cleanup** — detect and remove orphaned PID/socket files + +Usage:: + + info = DaemonInfo.discover(run_dir) + if info.is_healthy: + # connect to existing daemon + else: + # acquire lock and spawn new daemon +""" +from __future__ import annotations + +import fcntl +import json +import logging +import os +import signal +import socket +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class DaemonInfo: + """Snapshot of daemon state for a profile.""" + pid: Optional[int] + sock_path: Optional[Path] + start_time: Optional[float] + is_running: bool + is_healthy: bool + + @classmethod + def discover(cls, run_dir: Path) -> DaemonInfo: + """Probe the run directory to determine daemon state.""" + pid = _read_pid(run_dir / "leapd.pid") + sock_path = run_dir / "leapd.sock" + meta = _read_meta(run_dir / "leapd.json") + start_time = meta.get("start_time") if meta else None + + is_running = pid is not None and _process_alive(pid) + is_healthy = is_running and sock_path.exists() and _sock_healthy(sock_path) + + return cls( + pid=pid, + sock_path=sock_path if sock_path.exists() else None, + start_time=start_time, + is_running=is_running, + is_healthy=is_healthy, + ) + + @property + def uptime_s(self) -> Optional[float]: + if self.start_time is None: + return None + return time.time() - self.start_time + + def format_status(self) -> str: + """Human-readable status string.""" + if self.is_healthy: + uptime = self.uptime_s + up_str = _format_duration(uptime) if uptime else "unknown" + return f"leapd running (pid={self.pid}, uptime={up_str})" + if self.is_running: + return f"leapd running but unhealthy (pid={self.pid})" + if self.pid is not None: + return f"leapd stale (pid={self.pid} not running)" + return "leapd not running" + + +class DaemonLock: + """Advisory file lock for daemon leader election. + + Usage:: + + lock = DaemonLock(run_dir / "leapd.lock") + if lock.acquire(): + # we are the leader — spawn daemon + ... + lock.release() + """ + + def __init__(self, lock_path: Path) -> None: + self._path = lock_path + self._fd: Optional[int] = None + + def acquire(self) -> bool: + """Try to acquire the lock (non-blocking). Returns True if acquired.""" + self._path.parent.mkdir(parents=True, exist_ok=True) + try: + self._fd = os.open(str(self._path), os.O_CREAT | os.O_RDWR) + fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except (OSError, IOError): + if self._fd is not None: + os.close(self._fd) + self._fd = None + return False + + def release(self) -> None: + """Release the lock.""" + if self._fd is not None: + try: + fcntl.flock(self._fd, fcntl.LOCK_UN) + os.close(self._fd) + except OSError: + pass + self._fd = None + + def __enter__(self) -> DaemonLock: + return self + + def __exit__(self, *_: object) -> None: + self.release() + + +def write_pid_file(run_dir: Path, pid: Optional[int] = None) -> None: + """Write the daemon PID file and metadata.""" + run_dir.mkdir(parents=True, exist_ok=True) + actual_pid = pid or os.getpid() + (run_dir / "leapd.pid").write_text(str(actual_pid)) + + meta = { + "pid": actual_pid, + "start_time": time.time(), + "version": "1", + } + (run_dir / "leapd.json").write_text(json.dumps(meta)) + logger.info("daemon: wrote pid=%d to %s", actual_pid, run_dir / "leapd.pid") + + +def cleanup_run_dir(run_dir: Path) -> None: + """Remove daemon runtime files (PID, socket, metadata).""" + for name in ("leapd.pid", "leapd.json", "leapd.sock"): + path = run_dir / name + path.unlink(missing_ok=True) + logger.info("daemon: cleaned up %s", run_dir) + + +def cleanup_stale(run_dir: Path) -> bool: + """Detect and clean up stale daemon files. + + Returns True if stale files were cleaned. + """ + pid = _read_pid(run_dir / "leapd.pid") + if pid is None: + return False + + if _process_alive(pid): + return False + + logger.info("daemon: stale pid=%d detected, cleaning up", pid) + cleanup_run_dir(run_dir) + return True + + +def send_signal(run_dir: Path, sig: int = signal.SIGTERM) -> bool: + """Send a signal to the running daemon. Returns True if sent.""" + pid = _read_pid(run_dir / "leapd.pid") + if pid is None or not _process_alive(pid): + return False + try: + os.kill(pid, sig) + return True + except OSError: + return False + + +def spawn_daemon(settings: object, *, mock_host: bool = False) -> subprocess.Popen[bytes]: + """Spawn a detached leapd process for the active environment.""" + run_dir = getattr(settings, "profile_dir") / "run" + run_dir.mkdir(parents=True, exist_ok=True) + log_path = run_dir / "leapd.log" + command = [sys.executable, "-m", "leapflow"] + if mock_host: + command.append("--mock-host") + command.extend(["daemon", "serve", "--internal"]) + log_file = open(log_path, "ab") + try: + proc = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + close_fds=True, + ) + finally: + log_file.close() + logger.info("daemon: spawned pid=%s log=%s", proc.pid, log_path) + return proc + + +def wait_ready(run_dir: Path, *, timeout_s: float = 30.0, interval_s: float = 0.1) -> DaemonInfo: + """Wait until the daemon socket becomes healthy or timeout expires.""" + deadline = time.time() + max(0.1, timeout_s) + last = DaemonInfo.discover(run_dir) + while time.time() < deadline: + last = DaemonInfo.discover(run_dir) + if last.is_healthy: + return last + time.sleep(max(0.01, interval_s)) + return last + + +# ── Internal helpers ── + +def _read_pid(path: Path) -> Optional[int]: + """Read PID from file, returning None if missing/invalid.""" + try: + return int(path.read_text().strip()) + except (OSError, ValueError): + return None + + +def _read_meta(path: Path) -> Optional[dict]: + """Read daemon metadata JSON.""" + try: + return json.loads(path.read_text()) + except (OSError, json.JSONDecodeError, ValueError): + return None + + +def _process_alive(pid: int) -> bool: + """Check if a process with the given PID exists.""" + try: + os.kill(pid, 0) + return True + except OSError: + return False + + +def _sock_healthy(sock_path: Path) -> bool: + """Quick health check by connecting to the Unix socket.""" + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.settimeout(1.0) + s.connect(str(sock_path)) + s.close() + return True + except (OSError, socket.timeout): + return False + + +def _format_duration(seconds: Optional[float]) -> str: + """Format seconds into human-readable duration.""" + if seconds is None: + return "unknown" + s = int(seconds) + if s < 60: + return f"{s}s" + if s < 3600: + return f"{s // 60}m{s % 60}s" + h = s // 3600 + m = (s % 3600) // 60 + return f"{h}h{m}m" diff --git a/src/leapflow/daemon/protocol.py b/src/leapflow/daemon/protocol.py new file mode 100644 index 0000000..e9f42e5 --- /dev/null +++ b/src/leapflow/daemon/protocol.py @@ -0,0 +1,236 @@ +"""JSON-RPC 2.0 protocol types and LeapService interface for leapd. + +The protocol layer defines: + +1. **Message types** — JSON-RPC request, response, notification, and error +2. **LeapService** — the abstract service interface that both in-process + and daemon modes implement +3. **Method registry** — ``domain.action`` naming convention + +Wire format: newline-delimited JSON over Unix socket. +""" +from __future__ import annotations + +import json +import uuid +from dataclasses import asdict, dataclass, field +from enum import IntEnum +from typing import Any, AsyncIterator, Dict, List, Literal, Optional, Protocol, runtime_checkable + + +class ErrorCode(IntEnum): + """Standard JSON-RPC 2.0 error codes plus application-specific ones.""" + PARSE_ERROR = -32700 + INVALID_REQUEST = -32600 + METHOD_NOT_FOUND = -32601 + INVALID_PARAMS = -32602 + INTERNAL_ERROR = -32603 + + # Application-specific (reserved range: -32000 to -32099) + DATABASE_LOCKED = -32001 + SESSION_NOT_FOUND = -32002 + SKILL_NOT_FOUND = -32003 + CANCELLED = -32004 + + +@dataclass(frozen=True) +class RpcError: + """JSON-RPC error object.""" + code: int + message: str + data: Optional[Any] = None + + def to_dict(self) -> Dict[str, Any]: + d: Dict[str, Any] = {"code": self.code, "message": self.message} + if self.data is not None: + d["data"] = self.data + return d + + +@dataclass(frozen=True) +class RpcRequest: + """JSON-RPC 2.0 request.""" + method: str + params: Dict[str, Any] = field(default_factory=dict) + id: str = field(default_factory=lambda: uuid.uuid4().hex[:12]) + jsonrpc: str = "2.0" + + def to_json(self) -> str: + return json.dumps(asdict(self), ensure_ascii=False) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> RpcRequest: + return cls( + method=data["method"], + params=data.get("params", {}), + id=data.get("id", uuid.uuid4().hex[:12]), + ) + + +@dataclass(frozen=True) +class RpcResponse: + """JSON-RPC 2.0 response (success or error).""" + id: str + result: Optional[Any] = None + error: Optional[RpcError] = None + jsonrpc: str = "2.0" + + def to_json(self) -> str: + d: Dict[str, Any] = {"jsonrpc": self.jsonrpc, "id": self.id} + if self.error is not None: + d["error"] = self.error.to_dict() + else: + d["result"] = self.result + return json.dumps(d, ensure_ascii=False) + + @classmethod + def success(cls, request_id: str, result: Any = None) -> RpcResponse: + return cls(id=request_id, result=result) + + @classmethod + def fail(cls, request_id: str, code: int, message: str, data: Any = None) -> RpcResponse: + return cls(id=request_id, error=RpcError(code=code, message=message, data=data)) + + +@dataclass(frozen=True) +class RpcNotification: + """JSON-RPC 2.0 notification (no id, no response expected).""" + method: str + params: Dict[str, Any] = field(default_factory=dict) + jsonrpc: str = "2.0" + + def to_json(self) -> str: + return json.dumps(asdict(self), ensure_ascii=False) + + +@dataclass(frozen=True) +class StreamChunk: + """A single streaming chunk for daemon engine events.""" + request_id: str + content: str + done: bool = False + event_type: Literal[ + "chunk", "final", "tool_start", "tool_complete", "thinking", "status", "error" + ] = "chunk" + metadata: Optional[Dict[str, Any]] = None + + def to_notification(self) -> RpcNotification: + params: Dict[str, Any] = { + "id": self.request_id, + "content": self.content, + "done": self.done, + "event_type": self.event_type, + } + if self.metadata: + params["metadata"] = self.metadata + return RpcNotification(method="stream.chunk", params=params) + + +# ══════════════════════════════════════════════════════════════════════ +# LeapService — the contract between client and daemon +# ══════════════════════════════════════════════════════════════════════ + +@runtime_checkable +class LeapService(Protocol): + """Service interface for leapd operations. + + Both in-process mode and daemon mode implement this protocol. + Client code is agnostic to whether it talks to a local object + or a remote daemon over Unix socket. + """ + + async def signal_record(self, signal_data: Dict[str, Any]) -> Dict[str, Any]: + """Record a signal (observation, action, event).""" + ... + + async def memory_search(self, query: str, *, limit: int = 10) -> List[Dict[str, Any]]: + """Search memory across all providers.""" + ... + + async def memory_insert(self, content: str, kind: str = "fact", **kwargs: Any) -> str: + """Insert a memory entry. Returns entry_id.""" + ... + + async def session_create(self, **kwargs: Any) -> Dict[str, Any]: + """Create a new conversation session.""" + ... + + async def session_resume(self, session_id: str) -> Dict[str, Any]: + """Resume an existing session.""" + ... + + async def engine_chat(self, message: str, **kwargs: Any) -> AsyncIterator[StreamChunk]: + """Chat with the engine (streaming). Yields StreamChunks.""" + ... + + async def engine_cancel(self) -> bool: + """Cancel the currently running engine task.""" + ... + + async def skill_execute(self, skill_name: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Execute a skill by name.""" + ... + + async def scheduler_arm(self, task_config: Dict[str, Any]) -> str: + """Arm a scheduled task. Returns task_id.""" + ... + + async def status(self) -> Dict[str, Any]: + """Return daemon status (uptime, connections, db path, etc.).""" + ... + + async def shutdown(self) -> None: + """Graceful shutdown.""" + ... + + # ── Gateway ────────────────────────────────────────────────────── + + async def gateway_connect( + self, + platform: str, + credentials: Dict[str, str], + options: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Connect a platform via the gateway.""" + ... + + async def gateway_disconnect(self, platform: str) -> Dict[str, Any]: + """Disconnect a platform.""" + ... + + async def gateway_status(self) -> List[Dict[str, Any]]: + """Return status of all gateway platforms.""" + ... + + async def gateway_send( + self, + platform: str, + chat_id: str, + text: str, + thread_id: str = "", + ) -> Dict[str, Any]: + """Send a message to a connected platform conversation.""" + ... + + +# ══════════════════════════════════════════════════════════════════════ +# Method registry — maps JSON-RPC method names to LeapService methods +# ══════════════════════════════════════════════════════════════════════ + +METHOD_REGISTRY: Dict[str, str] = { + "signal.record": "signal_record", + "memory.search": "memory_search", + "memory.insert": "memory_insert", + "session.create": "session_create", + "session.resume": "session_resume", + "engine.chat": "engine_chat", + "engine.cancel": "engine_cancel", + "skill.execute": "skill_execute", + "scheduler.arm": "scheduler_arm", + "daemon.status": "status", + "daemon.shutdown": "shutdown", + "gateway.connect": "gateway_connect", + "gateway.disconnect": "gateway_disconnect", + "gateway.status": "gateway_status", + "gateway.send": "gateway_send", +} diff --git a/src/leapflow/daemon/server.py b/src/leapflow/daemon/server.py new file mode 100644 index 0000000..663daa4 --- /dev/null +++ b/src/leapflow/daemon/server.py @@ -0,0 +1,212 @@ +"""Unix socket JSON-RPC server for leapd.""" +from __future__ import annotations + +import asyncio +import json +import logging +import signal +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from leapflow.daemon.lifecycle import cleanup_run_dir, write_pid_file +from leapflow.daemon.protocol import ErrorCode, METHOD_REGISTRY, RpcRequest, RpcResponse, StreamChunk + +logger = logging.getLogger(__name__) + + +class UnixRpcServer: + """Newline-delimited JSON-RPC server bound to one Unix socket.""" + + def __init__(self, service: Any, *, sock_path: Path, run_dir: Path) -> None: + self._service = service + self._sock_path = sock_path + self._run_dir = run_dir + self._server: asyncio.AbstractServer | None = None + self._active_connections = 0 + if hasattr(service, "set_client_count_provider"): + service.set_client_count_provider(lambda: self._active_connections) + + @property + def active_connections(self) -> int: + """Return the current number of connected clients.""" + return self._active_connections + + async def serve_forever(self) -> None: + """Start listening and serve until cancelled.""" + self._run_dir.mkdir(parents=True, exist_ok=True) + self._sock_path.unlink(missing_ok=True) + self._server = await asyncio.start_unix_server( + self._handle_client, + path=str(self._sock_path), + ) + write_pid_file(self._run_dir) + try: + async with self._server: + await self._server.serve_forever() + except asyncio.CancelledError: + raise + finally: + self._sock_path.unlink(missing_ok=True) + + async def stop(self) -> None: + """Stop accepting clients and close the listening socket.""" + if self._server is not None: + self._server.close() + await self._server.wait_closed() + self._sock_path.unlink(missing_ok=True) + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + self._active_connections += 1 + try: + while not reader.at_eof(): + raw = await reader.readline() + if not raw: + break + await self._handle_line(raw, writer) + finally: + self._active_connections -= 1 + await _close_writer(writer) + + async def _handle_line(self, raw: bytes, writer: asyncio.StreamWriter) -> None: + try: + payload = json.loads(raw.decode("utf-8")) + request = RpcRequest.from_dict(payload) + except (UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + response = RpcResponse.fail( + "", + ErrorCode.PARSE_ERROR, + "Invalid JSON-RPC request", + data=str(exc), + ) + await _write_json(writer, response.to_json()) + return + + try: + await self._dispatch(request, writer) + except NotImplementedError as exc: + response = RpcResponse.fail( + request.id, + ErrorCode.METHOD_NOT_FOUND, + str(exc), + ) + await _write_json(writer, response.to_json()) + except Exception as exc: + logger.exception("daemon: request failed method=%s", request.method) + response = RpcResponse.fail( + request.id, + ErrorCode.INTERNAL_ERROR, + "Daemon request failed", + data=str(exc), + ) + await _write_json(writer, response.to_json()) + + async def _dispatch(self, request: RpcRequest, writer: asyncio.StreamWriter) -> None: + attr = METHOD_REGISTRY.get(request.method) + if attr is None: + response = RpcResponse.fail( + request.id, + ErrorCode.METHOD_NOT_FOUND, + f"Unknown method: {request.method}", + ) + await _write_json(writer, response.to_json()) + return + + method = getattr(self._service, attr) + params = dict(request.params or {}) + if request.method == "engine.chat": + await self._dispatch_stream(request, method, params, writer) + return + + result = method(**params) + if hasattr(result, "__await__"): + result = await result + response = RpcResponse.success(request.id, result) + await _write_json(writer, response.to_json()) + + async def _dispatch_stream( + self, + request: RpcRequest, + method: Callable[..., Any], + params: dict[str, Any], + writer: asyncio.StreamWriter, + ) -> None: + try: + async for chunk in method(**params): + notification = StreamChunk( + request_id=request.id, + content=chunk.content, + done=chunk.done, + event_type=chunk.event_type, + metadata=chunk.metadata, + ).to_notification() + await _write_json(writer, notification.to_json()) + except Exception as exc: + logger.exception("daemon: stream failed method=%s", request.method) + response = RpcResponse.fail( + request.id, + ErrorCode.INTERNAL_ERROR, + "Daemon stream failed", + data=str(exc), + ) + await _write_json(writer, response.to_json()) + return + + done = StreamChunk(request_id=request.id, content="", done=True).to_notification() + await _write_json(writer, done.to_json()) + response = RpcResponse.success(request.id, {"ok": True}) + await _write_json(writer, response.to_json()) + + +async def serve_daemon(settings: Any, *, mock_host: bool = False) -> int: + """Run a daemon server for the provided settings until signalled.""" + from leapflow.daemon.service import RuntimeLeapService + + run_dir = settings.profile_dir / "run" + sock_path = run_dir / "leapd.sock" + service = RuntimeLeapService(settings, mock_host=mock_host) + await service.start() + server = UnixRpcServer(service, sock_path=sock_path, run_dir=run_dir) + + loop = asyncio.get_running_loop() + stop_event = asyncio.Event() + + def _request_stop() -> None: + stop_event.set() + + for sig in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(sig, _request_stop) + except NotImplementedError: + logger.debug("daemon: signal handlers unsupported on this event loop") + + task = asyncio.create_task(server.serve_forever()) + try: + await stop_event.wait() + finally: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + await server.stop() + await service.shutdown() + cleanup_run_dir(run_dir) + return 0 + + +async def _write_json(writer: asyncio.StreamWriter, text: str) -> None: + writer.write(text.encode("utf-8") + b"\n") + await writer.drain() + + +async def _close_writer(writer: asyncio.StreamWriter) -> None: + writer.close() + try: + await writer.wait_closed() + except (BrokenPipeError, ConnectionError, OSError): + logger.debug("daemon: client connection closed with transport error", exc_info=True) diff --git a/src/leapflow/daemon/service.py b/src/leapflow/daemon/service.py new file mode 100644 index 0000000..6764b72 --- /dev/null +++ b/src/leapflow/daemon/service.py @@ -0,0 +1,234 @@ +"""Runtime-backed LeapService implementation for leapd.""" +from __future__ import annotations + +import asyncio +import logging +import os +import sys +import time +from collections.abc import AsyncIterator, Callable +from typing import Any + +from leapflow.daemon.protocol import StreamChunk +from leapflow.engine import StreamEvent +from leapflow.memory.protocol import MemoryEntry, MemoryQuery + +logger = logging.getLogger(__name__) + + +class RuntimeLeapService: + """LeapService implementation backed by a single initialized Context.""" + + def __init__(self, settings: Any, *, mock_host: bool = False) -> None: + self._settings = settings + self._mock_host = mock_host + self._ctx: Any | None = None + self._engine_lock = asyncio.Lock() + self._started_at = time.time() + self._client_count: Callable[[], int] = lambda: 0 + + def set_client_count_provider(self, provider: Callable[[], int]) -> None: + """Set a lightweight callback used by status reporting.""" + self._client_count = provider + + async def start(self) -> None: + """Initialize the daemon-owned runtime once.""" + if self._ctx is not None: + return + from leapflow.cli.context import Context + + ctx = Context(self._settings, self._mock_host) + await ctx.initialize() + self._ctx = ctx + + @property + def context(self) -> Any: + """Return the initialized Context or raise a clear lifecycle error.""" + if self._ctx is None: + raise RuntimeError("leapd runtime is not initialized") + return self._ctx + + async def signal_record(self, signal_data: dict[str, Any]) -> dict[str, Any]: + ctx = self.context + event_type = str(signal_data.get("type") or "daemon.signal") + payload = dict(signal_data.get("payload") or {}) + await ctx.event_bus.handle_event(event_type, payload) + return {"ok": True} + + async def memory_search(self, query: str, *, limit: int = 10) -> list[dict[str, Any]]: + ctx = self.context + memory_query = MemoryQuery(keywords=query.split()[:8], limit=limit) + results = await ctx.memory.search(memory_query) + return [self._memory_entry_to_dict(item) for item in results] + + async def memory_insert(self, content: str, kind: str = "fact", **kwargs: Any) -> str: + ctx = self.context + metadata = dict(kwargs.get("metadata") or {}) + entry_id = ctx.lt.ingest(kind, content, metadata=metadata) + return str(entry_id) + + async def session_create(self, **kwargs: Any) -> dict[str, Any]: + ctx = self.context + session_id = getattr(ctx.session, "session_id", "") if ctx.session else "" + return {"session_id": str(session_id), "created": bool(session_id), **kwargs} + + async def session_resume(self, session_id: str) -> dict[str, Any]: + ctx = self.context + engine = getattr(ctx, "engine", None) + found = bool(engine and engine.load_session(session_id)) + current = getattr(engine, "_current_session_id", "") if engine else "" + return {"found": found, "session_id": str(current or session_id)} + + async def engine_chat(self, message: str, **kwargs: Any) -> AsyncIterator[StreamChunk]: + async with self._engine_lock: + ctx = self.context + if ctx.reload_runtime_config_if_changed(): + self._settings = ctx.settings + yield StreamChunk( + request_id="", + content="Configuration reloaded — LLM settings updated in leapd.", + event_type="status", + metadata={ + "llm_model": getattr(ctx.settings, "llm_model", ""), + "llm_context_length": getattr(ctx.settings, "llm_context_length", 0), + "context_used": getattr(getattr(ctx, "engine", None), "context_token_count", 0), + }, + ) + engine = getattr(ctx, "engine", None) + if engine is None: + raise RuntimeError("leapd engine is not initialized") + + enable_thinking = bool(kwargs.get("enable_thinking", False)) + async for event in engine.run_stream(message, enable_thinking=enable_thinking): + stream_event = self._normalize_event(event) + yield self._chunk_from_event(stream_event) + + async def engine_cancel(self) -> bool: + ctx = self.context + engine = getattr(ctx, "engine", None) + if engine is not None and hasattr(engine, "cancel"): + result = engine.cancel() + if hasattr(result, "__await__"): + await result + return True + return False + + async def skill_execute(self, skill_name: str, params: dict[str, Any]) -> dict[str, Any]: + raise NotImplementedError("skill.execute is not available in this daemon phase") + + async def scheduler_arm(self, task_config: dict[str, Any]) -> str: + raise NotImplementedError("scheduler.arm is not available in this daemon phase") + + async def status(self) -> dict[str, Any]: + ctx = self._ctx + settings = getattr(ctx, "settings", self._settings) if ctx is not None else self._settings + engine = getattr(ctx, "engine", None) if ctx is not None else None + db_holder = getattr(ctx, "_db_holder", None) if ctx is not None else None + config_path = os.path.join(str(getattr(settings, "data_dir", "")), ".env") + project_env_path = os.path.join(os.getcwd(), ".env") + return { + "pid": os.getpid(), + "profile": getattr(settings, "profile", "default"), + "profile_dir": str(getattr(settings, "profile_dir", "")), + "config_path": config_path, + "project_env_path": project_env_path, + "db_path": str(getattr(db_holder, "db_path", settings.duckdb_path)), + "volatile": bool(getattr(ctx, "storage_volatile", False)) if ctx is not None else False, + "uptime_s": max(0.0, time.time() - self._started_at), + "active_clients": max(0, self._client_count()), + "model": getattr(settings, "llm_model", ""), + "llm_context_length": getattr(settings, "llm_context_length", 0), + "context_used": getattr(engine, "context_token_count", 0) if engine is not None else 0, + "session_id": str(getattr(engine, "_current_session_id", "") or ""), + "runtime_source": self._runtime_source(), + "runtime_executable": sys.executable, + "runtime_version": self._runtime_version(), + } + + @staticmethod + def _runtime_source() -> str: + import leapflow + + return str(getattr(leapflow, "__file__", "")) + + @staticmethod + def _runtime_version() -> str: + try: + from leapflow.version import __version__ + except ImportError: + return "unknown" + return str(__version__) + + async def shutdown(self) -> None: + if self._ctx is None: + return + ctx = self._ctx + self._ctx = None + self._checkpoint_open_connection(ctx) + await ctx.cleanup() + + async def gateway_connect( + self, + platform: str, + credentials: dict[str, str], + options: dict[str, Any] | None = None, + ) -> dict[str, Any]: + raise NotImplementedError("gateway.connect is not available in this daemon phase") + + async def gateway_disconnect(self, platform: str) -> dict[str, Any]: + raise NotImplementedError("gateway.disconnect is not available in this daemon phase") + + async def gateway_status(self) -> list[dict[str, Any]]: + raise NotImplementedError("gateway.status is not available in this daemon phase") + + async def gateway_send( + self, + platform: str, + chat_id: str, + text: str, + thread_id: str = "", + ) -> dict[str, Any]: + raise NotImplementedError("gateway.send is not available in this daemon phase") + + def _memory_entry_to_dict(self, entry: MemoryEntry) -> dict[str, Any]: + return { + "entry_id": entry.entry_id, + "kind": entry.kind.value, + "domain": entry.domain.value, + "content": entry.content, + "timestamp": entry.timestamp, + "score": entry.score, + "metadata": dict(entry.metadata), + } + + def _checkpoint_open_connection(self, ctx: Any) -> None: + holder = getattr(ctx, "_db_holder", None) + conn = getattr(holder, "_conn", None) + if conn is None: + return + try: + conn.execute("CHECKPOINT") + except Exception: + logger.debug("daemon: DuckDB checkpoint skipped", exc_info=True) + + def _normalize_event(self, event: object) -> StreamEvent: + if isinstance(event, StreamEvent): + return event + return StreamEvent(type="chunk", content=str(event), metadata=None) + + def _chunk_from_event(self, event: StreamEvent) -> StreamChunk: + ctx = self.context + engine = getattr(ctx, "engine", None) + metadata = dict(event.metadata or {}) + session_id = getattr(engine, "_current_session_id", "") if engine else "" + if session_id: + metadata.setdefault("session_id", str(session_id)) + if engine is not None: + metadata.setdefault("context_used", getattr(engine, "context_token_count", 0)) + return StreamChunk( + request_id="", + content=event.content, + done=False, + event_type=event.type, + metadata=metadata, + ) diff --git a/src/leapflow/engine/engine.py b/src/leapflow/engine/engine.py index 0e57d3c..fc20f6d 100644 --- a/src/leapflow/engine/engine.py +++ b/src/leapflow/engine/engine.py @@ -60,6 +60,45 @@ logger = logging.getLogger(__name__) +def _estimate_text_tokens(text: str) -> int: + """Approximate token count for status display when provider usage is absent.""" + if not text: + return 0 + cjk_count = sum( + 1 for ch in text if "\u4e00" <= ch <= "\u9fff" or "\u3000" <= ch <= "\u303f" + ) + latin_chars = len(text) - cjk_count + return max(1, cjk_count + latin_chars // 4) + + +def _estimate_message_tokens(message: Dict[str, Any]) -> int: + """Approximate chat-message token cost, including small role overhead.""" + content = message.get("content", "") + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, dict): + if item.get("type") == "text": + parts.append(str(item.get("text", ""))) + elif "text" in item: + parts.append(str(item.get("text", ""))) + else: + parts.append(str(item)) + else: + parts.append(str(item)) + content = "\n".join(parts) + elif not isinstance(content, str): + content = str(content) + return 6 + _estimate_text_tokens(content) + + +def _estimate_prompt_tokens(messages: List[Dict[str, Any]]) -> int: + """Approximate prompt token count for the exact message batch sent to the LLM.""" + if not messages: + return 0 + return max(1, sum(_estimate_message_tokens(msg) for msg in messages) + 3) + + def _log_progress(msg: str) -> None: """Print a persistent progress line to stderr (visible to user during `leap run`).""" if sys.stderr.isatty(): @@ -334,6 +373,35 @@ def set_sanitizer(self, sanitizer: MessageSanitizer | None) -> None: """Configure output message sanitizer.""" self._sanitizer = sanitizer + def reconfigure_runtime( + self, + *, + settings: Settings, + llm: LLMProvider, + vlm: Optional[Any], + classifier: IntentClassifier, + ) -> None: + """Refresh runtime LLM configuration without resetting session state.""" + self._settings = settings + self._llm = llm + self._vlm = vlm + self._classifier = classifier + self._skill_merger = SkillMerger( + registry=self._registry, + llm=llm, + execution=self._execution, + ) + if settings.has_llm_credentials: + self._graph_planner = GraphPlanner(self._llm, self._registry) + self._scheduler = TaskScheduler( + self._registry, + self._rpc, + graph_planner=self._graph_planner, + ) + else: + self._graph_planner = None + self._scheduler = None + def set_tool_result_budget(self, budget: int) -> None: """Override per-tool result truncation budget (e.g. linked to model context).""" self._tool_result_budget = max(1, budget) @@ -592,6 +660,10 @@ async def run(self, user_text: str, *, enable_thinking: bool = False) -> str: # 4. Everything else → unified tool loop (LLM decides tools vs direct response) logger.debug("route.unified user_text_len=%d", len(user_text)) + if not self._settings.has_llm_credentials: + msg = self._error_classifier.friendly_message(ErrorCategory.AUTH_PERMANENT) + self._wm.remember_chat(build_assistant_message(msg)) + return msg return await self._unified_tool_loop(user_text, enable_thinking=enable_thinking) async def run_stream( @@ -638,6 +710,11 @@ async def run_stream( # 4. Everything else → unified tool loop (streaming) logger.debug("route.unified user_text_len=%d", len(user_text)) + if not self._settings.has_llm_credentials: + msg = self._error_classifier.friendly_message(ErrorCategory.AUTH_PERMANENT) + self._wm.remember_chat(build_assistant_message(msg)) + yield StreamEvent(type="final", content=msg) + return async for chunk in self._unified_tool_loop_stream(user_text, enable_thinking=enable_thinking): yield chunk @@ -1080,6 +1157,7 @@ async def _unified_tool_loop( ] content = "" + fatal_error: Optional[str] = None recovery = TurnRecoveryState() use_native_tools = self._settings.native_tool_calling_enabled result_budget = self._effective_tool_result_budget() @@ -1110,6 +1188,7 @@ async def _unified_tool_loop( compressed = self._compressor.preflight_check(compressed) if self._cache_strategy: compressed = self._cache_strategy.optimize(compressed) + self._last_context_tokens = _estimate_prompt_tokens(compressed) _show_progress("thinking", f"round {budget.used}") try: @@ -1124,7 +1203,9 @@ async def _unified_tool_loop( provider=getattr(self._llm, 'active_provider_name', ''), model=resp.model or '', ) - self._last_context_tokens = usage.get("prompt_tokens", self._last_context_tokens) + provider_prompt = usage.get("prompt_tokens", 0) + if provider_prompt > 0: + self._last_context_tokens = provider_prompt if self._model_capabilities and resp.model and usage: self._model_capabilities.update_from_usage(resp.model, usage) except Exception as exc: @@ -1146,6 +1227,8 @@ async def _unified_tool_loop( tools_kwarg = {} use_native_tools = False continue + fatal_error = self._error_classifier.friendly_message(classified, str(exc)) + logger.error("unified_loop: unrecoverable %s: %s", category_str, exc) break _clear_indicator() @@ -1254,7 +1337,7 @@ async def _unified_tool_loop( logger.info("turn_usage: %s", self._usage_tracker.format_log_line()) - return content if content else "I've reached my processing limit." + return content if content else (fatal_error or "I've reached my processing limit.") async def _post_turn_review( self, messages: List[Dict[str, Any]], final_content: str @@ -1430,6 +1513,7 @@ async def _unified_tool_loop_stream( ] content = "" + fatal_error: Optional[str] = None turn_recovery = TurnRecoveryState() use_native_tools = self._settings.native_tool_calling_enabled result_budget = self._effective_tool_result_budget() @@ -1460,6 +1544,7 @@ async def _unified_tool_loop_stream( compressed = self._compressor.preflight_check(compressed) if self._cache_strategy: compressed = self._cache_strategy.optimize(compressed) + self._last_context_tokens = _estimate_prompt_tokens(compressed) yield StreamEvent(type="thinking", content=f"round {budget.used}") @@ -1478,7 +1563,9 @@ async def _unified_tool_loop_stream( provider=getattr(self._llm, 'active_provider_name', ''), model=resp.model or '', ) - self._last_context_tokens = usage.get("prompt_tokens", self._last_context_tokens) + provider_prompt = usage.get("prompt_tokens", 0) + if provider_prompt > 0: + self._last_context_tokens = provider_prompt except Exception as exc: _clear_indicator() classified = self._error_classifier.classify(exc) @@ -1591,7 +1678,9 @@ async def _unified_tool_loop_stream( classified, rec, turn_recovery, messages, budget, ) == "continue": continue - yield StreamEvent(type="error", content=str(exc)) + fatal_error = self._error_classifier.friendly_message(classified, str(exc)) + logger.error("unified_loop_stream: unrecoverable %s: %s", classified.value, exc) + yield StreamEvent(type="error", content=fatal_error) break content = "".join(content_parts).strip() @@ -1609,7 +1698,9 @@ async def _unified_tool_loop_stream( provider=getattr(self._llm, 'active_provider_name', ''), model=resp.model or '', ) - self._last_context_tokens = usage.get("prompt_tokens", self._last_context_tokens) + provider_prompt = usage.get("prompt_tokens", 0) + if provider_prompt > 0: + self._last_context_tokens = provider_prompt except Exception as exc: _clear_indicator() classified = self._error_classifier.classify(exc) @@ -1619,7 +1710,9 @@ async def _unified_tool_loop_stream( classified, rec, turn_recovery, messages, budget, ) == "continue": continue - yield StreamEvent(type="error", content=str(exc)) + fatal_error = self._error_classifier.friendly_message(classified, str(exc)) + logger.error("unified_loop_stream: unrecoverable %s: %s", classified.value, exc) + yield StreamEvent(type="error", content=fatal_error) break _clear_indicator() content = (resp.content or "").strip() @@ -1693,7 +1786,7 @@ async def _unified_tool_loop_stream( logger.info("turn_usage: %s", self._usage_tracker.format_log_line()) - yield StreamEvent(type="final", content=content if content else "I've reached my processing limit.") + yield StreamEvent(type="final", content=content if content else (fatal_error or "I've reached my processing limit.")) # ── Unified Loop Helpers ─────────────────────────────────────────────── @@ -1830,31 +1923,43 @@ async def _execute_general_tool( """ from leapflow.skills.tool_executor import ToolCall as TC from leapflow.security.redact import redact_sensitive_text - from leapflow.security.threat_patterns import is_untrusted_source, wrap_untrusted_result name = tool_call.get("name", "") args = tool_call.get("arguments", {}) result: Dict[str, Any] - # Route through ToolBridge when available (single source of truth) - if self._tool_bridge is not None: - prefixed = f"gp_{name}" - result = await self._tool_bridge.dispatch(TC(name=prefixed, params=args)) - if not (isinstance(result, dict) and "unknown_tool" in str(result.get("error", ""))): - return self._post_process_tool_result(name, result) - result = await self._tool_bridge.dispatch(TC(name=name, params=args)) - if not (isinstance(result, dict) and "unknown_tool" in str(result.get("error", ""))): - return self._post_process_tool_result(name, result) - - # Fallback: direct handler dispatch - handler = handlers.get(name) - if handler is None: - return {"ok": False, "error": f"Unknown tool: {name}"} - timeout = self._tool_timeouts.get(name, self._default_tool_timeout_s) t0 = time.perf_counter() + try: + # Route through ToolBridge when available (single source of truth) + if self._tool_bridge is not None: + prefixed = f"gp_{name}" + result = await asyncio.wait_for( + self._tool_bridge.dispatch(TC(name=prefixed, params=args)), + timeout=timeout, + ) + if not (isinstance(result, dict) and "unknown_tool" in str(result.get("error", ""))): + duration = (time.perf_counter() - t0) * 1000 + is_ok = not (isinstance(result, dict) and not result.get("ok", True)) + self._usage_tracker.record_tool_call(name, is_ok, duration) + return self._post_process_tool_result(name, result) + result = await asyncio.wait_for( + self._tool_bridge.dispatch(TC(name=name, params=args)), + timeout=timeout, + ) + if not (isinstance(result, dict) and "unknown_tool" in str(result.get("error", ""))): + duration = (time.perf_counter() - t0) * 1000 + is_ok = not (isinstance(result, dict) and not result.get("ok", True)) + self._usage_tracker.record_tool_call(name, is_ok, duration) + return self._post_process_tool_result(name, result) + + # Fallback: direct handler dispatch + handler = handlers.get(name) + if handler is None: + return {"ok": False, "error": f"Unknown tool: {name}"} + result = await asyncio.wait_for(handler(args), timeout=timeout) except asyncio.TimeoutError: duration = (time.perf_counter() - t0) * 1000 @@ -2002,7 +2107,11 @@ def _persist_message( logger.debug("session.persist_message failed", exc_info=True) async def _prefetch_and_freeze_memory(self, user_text: str) -> str: - """Prefetch memory context and freeze snapshot for session duration.""" + """Prefetch memory context and freeze snapshot for session duration. + + Combines narrative memory (always-on MEMORY.md) with signal-based + prefetch results into a unified context block. + """ if self._memory_context_snapshot is not None: return self._memory_context_snapshot @@ -2010,6 +2119,19 @@ async def _prefetch_and_freeze_memory(self, user_text: str) -> str: self._memory_context_snapshot = "" return "" + parts: list[str] = [] + + # Layer 1: Narrative memory (MEMORY.md — always loaded, no timeout) + narrative = self._memory_manager.get_provider("narrative") + if narrative is not None and hasattr(narrative, "context_block"): + try: + block = narrative.context_block() + if block: + parts.append(block) + except Exception: + logger.debug("narrative.context_block failed", exc_info=True) + + # Layer 2: Signal-based prefetch (DuckDB — timeout-bounded) try: entries = await asyncio.wait_for( self._memory_manager.prefetch( @@ -2018,19 +2140,18 @@ async def _prefetch_and_freeze_memory(self, user_text: str) -> str: timeout=self._settings.memory_prefetch_timeout_s, ) if entries: - context = "## Recent Context\n" + "\n".join( + parts.append("## Recent Context\n" + "\n".join( f"- [{e.kind.value}] {e.content[:100]}" for e in entries - ) - self._memory_context_snapshot = context - return context + )) except asyncio.TimeoutError: logger.debug( "memory.prefetch timed out (%.1fs)", self._settings.memory_prefetch_timeout_s, ) except Exception: logger.debug("memory.prefetch failed", exc_info=True) - self._memory_context_snapshot = "" - return "" + + self._memory_context_snapshot = "\n\n".join(parts) + return self._memory_context_snapshot async def _sync_turn_safe(self, messages: List[Dict[str, Any]]) -> None: """Non-blocking wrapper for MemoryManager.sync_turn.""" diff --git a/src/leapflow/engine/error_classifier.py b/src/leapflow/engine/error_classifier.py index 4d6873d..f748965 100644 --- a/src/leapflow/engine/error_classifier.py +++ b/src/leapflow/engine/error_classifier.py @@ -106,6 +106,51 @@ def build_recovery_map( RECOVERY_MAP: Dict[ErrorCategory, RecoveryStrategy] = build_recovery_map() +# User-facing, actionable messages per error category. Surfaced by the agent +# loop so a failed turn explains *why* it failed and how to fix it, instead of +# a generic "I've reached my processing limit." fallback. +_FRIENDLY_MESSAGES: Dict[ErrorCategory, str] = { + ErrorCategory.AUTH_ERROR: ( + "LLM authentication failed — the API key is missing or invalid. " + "Set a valid LEAPFLOW_LLM_API_KEY in ~/.leapflow/.env " + "or in a project-specific ./.env file, then retry. " + "Also verify LEAPFLOW_LLM_BASE_URL / LEAPFLOW_LLM_MODEL." + ), + ErrorCategory.AUTH_PERMANENT: ( + "LLM authentication failed — the API key is missing or invalid. " + "Set a valid LEAPFLOW_LLM_API_KEY in ~/.leapflow/.env " + "or in a project-specific ./.env file, then retry. " + "Also verify LEAPFLOW_LLM_BASE_URL / LEAPFLOW_LLM_MODEL." + ), + ErrorCategory.BILLING: ( + "LLM request rejected due to billing/quota limits \u2014 " + "check your provider account balance or quota." + ), + ErrorCategory.MODEL_NOT_FOUND: ( + "The configured LLM model was not found \u2014 check LEAPFLOW_LLM_MODEL." + ), + ErrorCategory.RATE_LIMITED: ( + "LLM rate limit reached \u2014 please wait a moment and try again." + ), + ErrorCategory.OVERLOADED: ( + "The LLM provider is overloaded \u2014 please retry shortly." + ), + ErrorCategory.CONTEXT_OVERFLOW: ( + "The conversation exceeded the model's context window \u2014 " + "start a new session or shorten the input." + ), + ErrorCategory.PAYLOAD_TOO_LARGE: ( + "The request payload was too large for the provider." + ), + ErrorCategory.CONTENT_BLOCKED: ( + "The request was blocked by the provider's content policy." + ), + ErrorCategory.SSL_ERROR: ( + "TLS/SSL error connecting to the LLM provider \u2014 " + "check your network, proxy, or certificates." + ), +} + _AUTH_KEYWORDS = ("api_key", "api key", "unauthorized", "forbidden", "401", "403") _BILLING_KEYWORDS = ("insufficient_quota", "billing", "payment", "quota exceeded", "402") _RATE_LIMIT_KEYWORDS = ("rate", "429", "too many", "throttl") @@ -143,7 +188,6 @@ def classify(self, exc: Exception) -> ErrorCategory: def classify_detailed(self, exc: Exception) -> ClassifiedError: """Classify with full context for advanced recovery logic.""" - msg = str(exc).lower() status = self._extract_status_code(exc) category = self.classify(exc) recovery = self.get_recovery(category) @@ -176,6 +220,21 @@ def classify_tool_error(self, observation: Dict[str, Any]) -> ErrorCategory: def get_recovery(self, category: ErrorCategory) -> RecoveryStrategy: return self._map.get(category, RecoveryStrategy()) + @staticmethod + def friendly_message(category: ErrorCategory, detail: str = "") -> str: + """Return a clear, actionable user-facing message for an error category. + + Lets the agent loop surface *why* a turn failed (with remediation + guidance) instead of a generic fallback message. + """ + base = _FRIENDLY_MESSAGES.get(category) + if base: + return base + detail = (detail or "").strip() + if detail: + return f"LLM request failed ({category.value}): {detail[:200]}" + return f"LLM request failed ({category.value})." + @staticmethod def _extract_status_code(exc: Exception) -> Optional[int]: """Extract HTTP status code from common exception types.""" diff --git a/src/leapflow/gateway/__init__.py b/src/leapflow/gateway/__init__.py new file mode 100644 index 0000000..b4f6015 --- /dev/null +++ b/src/leapflow/gateway/__init__.py @@ -0,0 +1,67 @@ +"""Gateway module — platform adapter management and message routing. + +Public API: + +- Protocol types: ``MessageSource``, ``InboundMessage``, ``OutboundContent``, etc. +- ``PlatformAdapter`` Protocol + ``PlatformAdapterMixin`` for graceful degradation +- ``GatewayServer`` for adapter lifecycle and message routing +- ``ManifestLoader`` for declarative platform discovery +- ``CredentialVault`` for at-rest credential encryption +- ``SessionKey`` / ``build_session_key`` for structured session routing +- ``GatewayRouter`` for per-session LLM processing of inbound messages +""" +from leapflow.gateway.config_store import GatewayConfig, GatewayConfigStore +from leapflow.gateway.credential_vault import CredentialVault +from leapflow.gateway.events import ( + GatewayMessageReceived, + GatewaySessionCreated, + GatewaySessionEnded, +) +from leapflow.gateway.manifest import ManifestLoader, PlatformManifest +from leapflow.gateway.mixin import PlatformAdapterMixin +from leapflow.gateway.protocol import ( + InboundMessage, + MediaAttachment, + MessageSource, + OutboundContent, + PlatformAdapter, + PlatformStatus, + SendResult, + SendTarget, +) +from leapflow.gateway.router import GatewayRouter +from leapflow.gateway.server import GatewayServer +from leapflow.gateway.session_router import SessionKey, build_session_key +from leapflow.gateway.validators import register_validator, validate_credentials + +__all__ = [ + # Protocol types + "MessageSource", + "MediaAttachment", + "InboundMessage", + "SendTarget", + "OutboundContent", + "SendResult", + "PlatformStatus", + # Adapter contract + "PlatformAdapter", + "PlatformAdapterMixin", + # Events + "GatewayMessageReceived", + "GatewaySessionCreated", + "GatewaySessionEnded", + # Core components + "GatewayRouter", + "GatewayServer", + "ManifestLoader", + "PlatformManifest", + "CredentialVault", + "GatewayConfig", + "GatewayConfigStore", + # Session routing + "SessionKey", + "build_session_key", + # Validators + "register_validator", + "validate_credentials", +] diff --git a/src/leapflow/gateway/config_store.py b/src/leapflow/gateway/config_store.py new file mode 100644 index 0000000..83d254a --- /dev/null +++ b/src/leapflow/gateway/config_store.py @@ -0,0 +1,215 @@ +"""Gateway configuration persistence (``gateway.yaml``). + +Reads and writes platform configurations with encrypted credentials. +Thread-safe via atomic write (write to temp file then ``os.replace``). +""" +from __future__ import annotations + +import logging +import os +import tempfile +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from leapflow.gateway.credential_vault import CredentialVault, _ensure_file_permissions +from leapflow.gateway.manifest import PlatformManifest + +logger = logging.getLogger(__name__) + +_CONFIG_VERSION = 1 + + +# ═══════════════════════════════════════════════════════════════ +# Config domain types +# ═══════════════════════════════════════════════════════════════ + +@dataclass +class PlatformConfig: + """Runtime configuration for a single platform.""" + + enabled: bool = True + credentials: Dict[str, str] = field(default_factory=dict) + options: Dict[str, Any] = field(default_factory=dict) + configured_at: str = "" + configured_by: str = "conversation" + + +@dataclass +class GatewayConfig: + """Full gateway configuration read from / written to disk.""" + + version: int = _CONFIG_VERSION + platforms: Dict[str, PlatformConfig] = field(default_factory=dict) + auto_connect: List[str] = field(default_factory=list) + + +# ═══════════════════════════════════════════════════════════════ +# Config store +# ═══════════════════════════════════════════════════════════════ + +class GatewayConfigStore: + """Reads and writes ``gateway.yaml`` with credential encryption.""" + + def __init__(self, profile_dir: Path, vault: CredentialVault) -> None: + self._config_path = profile_dir / "gateway.yaml" + self._vault = vault + self._manifests: Dict[str, PlatformManifest] = {} + + def set_manifests(self, manifests: Dict[str, PlatformManifest]) -> None: + """Provide manifest lookup for encryption / decryption decisions.""" + self._manifests = manifests + + # ── Read ───────────────────────────────────────────────── + + def load(self) -> GatewayConfig: + """Load gateway config from disk. Returns empty config if missing.""" + if not self._config_path.exists(): + return GatewayConfig() + try: + import yaml + + raw = yaml.safe_load( + self._config_path.read_text(encoding="utf-8"), + ) or {} + except Exception: + logger.warning( + "Failed to parse gateway.yaml, using defaults", exc_info=True, + ) + return GatewayConfig() + + config = GatewayConfig(version=raw.get("version", _CONFIG_VERSION)) + for pid, pdata in raw.get("platforms", {}).items(): + config.platforms[pid] = PlatformConfig( + enabled=pdata.get("enabled", True), + credentials=pdata.get("credentials", {}), + options=pdata.get("options", {}), + configured_at=pdata.get("configured_at", ""), + configured_by=pdata.get("configured_by", "manual"), + ) + config.auto_connect = raw.get("auto_connect", []) + return config + + # ── Write (atomic) ─────────────────────────────────────── + + def save(self, config: GatewayConfig) -> None: + """Atomically write gateway config to disk.""" + import yaml + + raw: Dict[str, Any] = { + "version": config.version, + "platforms": {}, + "auto_connect": config.auto_connect, + } + for pid, pc in config.platforms.items(): + raw["platforms"][pid] = { + "enabled": pc.enabled, + "credentials": pc.credentials, + "options": pc.options, + "configured_at": pc.configured_at, + "configured_by": pc.configured_by, + } + + self._config_path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + dir=str(self._config_path.parent), + suffix=".tmp", + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + yaml.safe_dump( + raw, fh, default_flow_style=False, allow_unicode=True, + ) + os.replace(tmp_path, str(self._config_path)) + _ensure_file_permissions(self._config_path) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + # ── Platform-level convenience ─────────────────────────── + + def save_platform( + self, + platform_id: str, + credentials: Dict[str, str], + options: Dict[str, Any], + manifest: PlatformManifest, + ) -> None: + """Save or update a single platform's configuration.""" + config = self.load() + + secret_keys = frozenset( + c.key for c in manifest.credentials if c.secret + ) + encrypted = self._vault.encrypt_credentials(credentials, secret_keys) + + config.platforms[platform_id] = PlatformConfig( + enabled=True, + credentials=encrypted, + options=options, + configured_at=datetime.now(timezone.utc).isoformat(), + configured_by="conversation", + ) + if platform_id not in config.auto_connect: + config.auto_connect.append(platform_id) + + self.save(config) + + def load_platform_credentials( + self, + platform_id: str, + manifest: PlatformManifest, + ) -> Optional[Dict[str, str]]: + """Load and decrypt credentials for a platform. + + Environment variables ``LEAPFLOW__`` (uppercased) + take precedence over file-stored values, enabling container and + CI/CD deployments without touching ``gateway.yaml``. + + Returns ``None`` if the platform is not configured (neither file + nor env vars provide credentials). + """ + config = self.load() + pc = config.platforms.get(platform_id) + + if pc is None or not pc.credentials: + creds = self._load_from_env(platform_id, manifest) + return creds if creds else None + + secret_keys = frozenset( + c.key for c in manifest.credentials if c.secret + ) + result = self._vault.decrypt_credentials(pc.credentials, secret_keys) + + env_overrides = self._load_from_env(platform_id, manifest) + if env_overrides: + result.update(env_overrides) + + return result + + @staticmethod + def _load_from_env( + platform_id: str, + manifest: PlatformManifest, + ) -> Dict[str, str]: + """Check for ``LEAPFLOW__`` environment overrides.""" + prefix = f"LEAPFLOW_{platform_id.upper()}_" + overrides: Dict[str, str] = {} + for cred in manifest.credentials: + env_key = prefix + cred.key.upper() + env_val = os.environ.get(env_key) + if env_val: + overrides[cred.key] = env_val + return overrides + + def remove_platform(self, platform_id: str) -> None: + """Remove a platform from configuration.""" + config = self.load() + config.platforms.pop(platform_id, None) + if platform_id in config.auto_connect: + config.auto_connect.remove(platform_id) + self.save(config) diff --git a/src/leapflow/gateway/credential_vault.py b/src/leapflow/gateway/credential_vault.py new file mode 100644 index 0000000..89704d8 --- /dev/null +++ b/src/leapflow/gateway/credential_vault.py @@ -0,0 +1,146 @@ +"""Credential encryption for gateway platform credentials. + +Defence layers implemented here: + +1. Fernet (AES-128-CBC + HMAC-SHA256) encryption at rest for secret fields +2. File permissions ``0600`` on ``gateway.yaml`` and key file +3. Graceful degradation to base64 if ``cryptography`` is not installed + +Additional layers (file-read denial, tool-result sanitisation, log +redaction) are implemented at their respective trust boundaries. +""" +from __future__ import annotations + +import base64 +import logging +import os +import stat +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_FERNET_PREFIX = "enc:fernet:" +_B64_PREFIX = "enc:b64:" +_ENC_PREFIXES = (_FERNET_PREFIX, _B64_PREFIX) + + +def _ensure_file_permissions(path: Path) -> None: + """Set file permissions to 0600 (owner read/write only).""" + if path.exists() and os.name != "nt": + path.chmod(stat.S_IRUSR | stat.S_IWUSR) + + +class CredentialVault: + """Encrypts and decrypts platform credentials at rest. + + Only fields declared ``secret: true`` in the platform manifest are + encrypted. Non-secret fields (e.g. ``app_id``) remain plaintext so + users can inspect ``gateway.yaml`` without tools. + """ + + def __init__(self, profile_dir: Path) -> None: + self._key_path = profile_dir / ".credential_key" + self._fernet: Optional[Any] = None + self._fallback_mode = False + + # ── Key management ─────────────────────────────────────── + + def _ensure_key(self) -> None: + """Load or generate the encryption key (lazy).""" + if self._fernet is not None or self._fallback_mode: + return + try: + from cryptography.fernet import Fernet + except ImportError: + logger.warning( + "cryptography package not installed; credentials will use " + "base64 encoding only. Install 'cryptography' for AES encryption.", + ) + self._fallback_mode = True + return + + if self._key_path.exists(): + key = self._key_path.read_bytes().strip() + else: + key = Fernet.generate_key() + self._key_path.parent.mkdir(parents=True, exist_ok=True) + self._key_path.write_bytes(key) + _ensure_file_permissions(self._key_path) + + self._fernet = Fernet(key) + + # ── Single-value operations ────────────────────────────── + + def encrypt_value(self, plaintext: str) -> str: + """Encrypt a single credential value. Returns prefixed ciphertext.""" + self._ensure_key() + if self._fernet is not None: + token = self._fernet.encrypt(plaintext.encode("utf-8")) + return _FERNET_PREFIX + token.decode("ascii") + return _B64_PREFIX + base64.b64encode( + plaintext.encode("utf-8"), + ).decode("ascii") + + def decrypt_value(self, stored: str) -> str: + """Decrypt a single stored credential value. + + If decryption fails (corrupted key file or tampered ciphertext), + a clear error is logged with recovery guidance. + """ + if stored.startswith(_FERNET_PREFIX): + self._ensure_key() + if self._fernet is None: + raise RuntimeError( + "Fernet-encrypted value present but cryptography not installed", + ) + token = stored[len(_FERNET_PREFIX):].encode("ascii") + try: + return self._fernet.decrypt(token).decode("utf-8") + except Exception as exc: + logger.error( + "Credential decryption failed (key file may be corrupted " + "or replaced). Recovery: delete %s and reconfigure the " + "platform via 'gateway_connect'. Error: %s", + self._key_path, + type(exc).__name__, + ) + raise + if stored.startswith(_B64_PREFIX): + encoded = stored[len(_B64_PREFIX):] + return base64.b64decode(encoded).decode("utf-8") + return stored + + # ── Batch operations (dict-level) ──────────────────────── + + def encrypt_credentials( + self, + credentials: Dict[str, str], + secret_keys: frozenset, + ) -> Dict[str, str]: + """Encrypt secret fields in a credentials dict. Idempotent.""" + result: Dict[str, str] = {} + for key, value in credentials.items(): + if ( + key in secret_keys + and value + and not value.startswith(_ENC_PREFIXES) + ): + result[key] = self.encrypt_value(value) + else: + result[key] = value + return result + + def decrypt_credentials( + self, + stored: Dict[str, str], + secret_keys: frozenset, + ) -> Dict[str, str]: + """Decrypt secret fields from stored credentials.""" + result: Dict[str, str] = {} + for key, value in stored.items(): + if key in secret_keys and value.startswith(_ENC_PREFIXES): + result[key] = self.decrypt_value(value) + else: + result[key] = value + return result diff --git a/src/leapflow/gateway/events.py b/src/leapflow/gateway/events.py new file mode 100644 index 0000000..8949097 --- /dev/null +++ b/src/leapflow/gateway/events.py @@ -0,0 +1,47 @@ +"""Gateway event types for EventBus integration. + +Gateway publishes events; MemoryManager, Copilot, and AgentEngine can +subscribe independently. All types are frozen dataclasses — safe to +pass across asyncio tasks. + +The current ``EventBus`` is perception-specific (``handle_event``). +These types define the contract for a future generalised pub/sub layer. +For Phase 1, ``GatewayServer`` uses an optional callback to notify +interested subscribers. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Tuple + +from leapflow.gateway.protocol import MessageSource + + +@dataclass(frozen=True) +class GatewayMessageReceived: + """Inbound message from any external platform.""" + + source: MessageSource + session_key: str + text: str + media_urls: Tuple[str, ...] = () + timestamp: float = field(default_factory=time.time) + + +@dataclass(frozen=True) +class GatewaySessionCreated: + """New session established through gateway.""" + + session_key: str + source: MessageSource + timestamp: float = field(default_factory=time.time) + + +@dataclass(frozen=True) +class GatewaySessionEnded: + """Session ended (reset / timeout / explicit close).""" + + session_key: str + reason: str + timestamp: float = field(default_factory=time.time) diff --git a/src/leapflow/gateway/manifest.py b/src/leapflow/gateway/manifest.py new file mode 100644 index 0000000..1aaa92f --- /dev/null +++ b/src/leapflow/gateway/manifest.py @@ -0,0 +1,195 @@ +"""Platform manifest loading and discovery. + +A manifest is a YAML file that declares: + +- What credentials a platform requires (for conversational config) +- Setup instructions for the user (multilingual) +- Adapter module / class to instantiate +- Validation method to call + +Manifests are discovered from a prioritised list of directories +(built-in → user profile → pip packages) so that new platforms can +be added without touching core code. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +# ═══════════════════════════════════════════════════════════════ +# Manifest domain types (all frozen) +# ═══════════════════════════════════════════════════════════════ + +@dataclass(frozen=True) +class CredentialField: + """A single credential field declared by a platform.""" + + key: str + label: str + required: bool = True + secret: bool = False + help_zh: str = "" + help_en: str = "" + + +@dataclass(frozen=True) +class OptionField: + """An optional configuration field with a default value.""" + + key: str + label: str + field_type: str = "string" + choices: tuple = () + default: Any = None + required: bool = False + advanced: bool = False + depends_on: Dict[str, Any] = field(default_factory=dict) + help_zh: str = "" + help_en: str = "" + + +@dataclass(frozen=True) +class SetupGuide: + """User-facing setup instructions (bilingual).""" + + summary_zh: str = "" + summary_en: str = "" + steps_zh: tuple = () + steps_en: tuple = () + console_url: str = "" + + +@dataclass(frozen=True) +class AdapterSpec: + """How to dynamically instantiate the adapter.""" + + module: str + class_name: str + dependencies: tuple = () + + +@dataclass(frozen=True) +class PlatformManifest: + """Complete declaration of a platform's integration requirements.""" + + platform_id: str + display_name: str + description: str = "" + category: str = "im" + credentials: tuple = () + options: tuple = () + setup_guide: SetupGuide = field(default_factory=SetupGuide) + validation_method: str = "" + validation_timeout_s: float = 10.0 + adapter: Optional[AdapterSpec] = None + source_path: str = "" + + +# ═══════════════════════════════════════════════════════════════ +# Manifest loader +# ═══════════════════════════════════════════════════════════════ + +class ManifestLoader: + """Discovers and loads platform manifests from YAML files. + + Search order (later wins on ID collision): + 1. Built-in manifests shipped with leapflow + 2. Extra directories (user profile, pip packages) + """ + + def __init__(self, extra_dirs: Optional[List[Path]] = None) -> None: + self._search_paths: List[Path] = [ + Path(__file__).parent / "manifests", + ] + if extra_dirs: + self._search_paths.extend(extra_dirs) + + def discover(self) -> Dict[str, PlatformManifest]: + """Scan all search paths and return discovered manifests.""" + manifests: Dict[str, PlatformManifest] = {} + for search_path in self._search_paths: + if not search_path.is_dir(): + continue + for yaml_file in sorted(search_path.glob("*.yaml")): + try: + manifest = self._parse(yaml_file) + manifests[manifest.platform_id] = manifest + except Exception: + logger.warning( + "Failed to parse manifest: %s", yaml_file, exc_info=True, + ) + return manifests + + # ── Internal ───────────────────────────────────────────── + + def _parse(self, path: Path) -> PlatformManifest: + """Parse a single YAML file into a ``PlatformManifest``.""" + import yaml # lazy — not needed if no manifests exist + + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + + credentials = tuple( + CredentialField( + key=c["key"], + label=c["label"], + required=c.get("required", True), + secret=c.get("secret", False), + help_zh=c.get("help_zh", ""), + help_en=c.get("help_en", ""), + ) + for c in raw.get("credentials", []) + ) + + options = tuple( + OptionField( + key=o["key"], + label=o["label"], + field_type=o.get("type", "string"), + choices=tuple(o.get("choices", [])), + default=o.get("default"), + required=o.get("required", False), + advanced=o.get("advanced", False), + depends_on=o.get("depends_on", {}), + help_zh=o.get("help_zh", ""), + help_en=o.get("help_en", ""), + ) + for o in raw.get("options", []) + ) + + guide_raw = raw.get("setup_guide", {}) + setup_guide = SetupGuide( + summary_zh=guide_raw.get("summary_zh", ""), + summary_en=guide_raw.get("summary_en", ""), + steps_zh=tuple(guide_raw.get("steps_zh", [])), + steps_en=tuple(guide_raw.get("steps_en", [])), + console_url=guide_raw.get("console_url", ""), + ) + + adapter_raw = raw.get("adapter") + adapter = None + if adapter_raw: + adapter = AdapterSpec( + module=adapter_raw["module"], + class_name=adapter_raw["class"], + dependencies=tuple(adapter_raw.get("dependencies", [])), + ) + + validation_raw = raw.get("validation", {}) + return PlatformManifest( + platform_id=raw["platform_id"], + display_name=raw.get("display_name", raw["platform_id"]), + description=raw.get("description", ""), + category=raw.get("category", "im"), + credentials=credentials, + options=options, + setup_guide=setup_guide, + validation_method=validation_raw.get("method", ""), + validation_timeout_s=float(validation_raw.get("timeout_s", 10)), + adapter=adapter, + source_path=str(path), + ) diff --git a/src/leapflow/gateway/manifests/api_server.yaml b/src/leapflow/gateway/manifests/api_server.yaml new file mode 100644 index 0000000..a4d75e9 --- /dev/null +++ b/src/leapflow/gateway/manifests/api_server.yaml @@ -0,0 +1,43 @@ +platform_id: api_server +display_name: "API Server (OpenAI Compatible)" +description: HTTP API server exposing OpenAI-compatible chat completions endpoint +category: api + +credentials: + - key: api_key + label: API Key + required: true + secret: true + help_zh: "用于认证 API 请求的密钥（至少 16 字符，自行设定）" + help_en: "API key for authenticating requests (at least 16 chars, you define it)" + +options: + - key: host + label: 绑定地址 + type: string + default: "127.0.0.1" + help_zh: "HTTP 服务绑定地址（默认仅本机访问）" + help_en: "HTTP server bind address (default: localhost only)" + - key: port + label: 端口 + type: string + default: "8080" + help_zh: "HTTP 服务端口" + help_en: "HTTP server port" + +setup_guide: + summary_zh: | + API Server 提供 OpenAI 兼容的 HTTP API，可供外部应用调用。 + 你需要设定一个 API Key 用于认证（至少 16 字符）。 + summary_en: | + API Server exposes an OpenAI-compatible HTTP API for external applications. + You need to set an API key for authentication (at least 16 characters). + +validation: + method: "" + timeout_s: 5 + +adapter: + module: leapflow.gateway.adapters.api_server + class: APIServerAdapter + dependencies: [aiohttp] diff --git a/src/leapflow/gateway/manifests/dingtalk.yaml b/src/leapflow/gateway/manifests/dingtalk.yaml new file mode 100644 index 0000000..c0ad180 --- /dev/null +++ b/src/leapflow/gateway/manifests/dingtalk.yaml @@ -0,0 +1,49 @@ +platform_id: dingtalk +display_name: "钉钉 (DingTalk)" +description: "钉钉企业协作平台" +category: im + +credentials: + - key: app_key + label: AppKey + required: true + secret: false + help_zh: "钉钉开放平台 → 应用开发 → 企业内部应用 → 凭证与基础信息 → AppKey" + help_en: "DingTalk Open Platform → App Dev → Internal App → Credentials → AppKey" + - key: app_secret + label: AppSecret + required: true + secret: true + help_zh: "同页面 → AppSecret" + help_en: "Same page → AppSecret" + +options: + - key: robot_code + label: 机器人编码 + type: string + required: false + help_zh: "如使用自定义机器人，请提供 robot_code（可选）" + help_en: "Provide robot_code if using a custom bot (optional)" + +setup_guide: + summary_zh: | + 接入钉钉需要在钉钉开放平台创建企业内部应用并开启机器人能力。 + 完成后请提供 AppKey 和 AppSecret。 + summary_en: | + To connect DingTalk, create an internal app on open-dev.dingtalk.com + with bot capability. Then provide your AppKey and AppSecret. + steps_zh: + - "打开 https://open-dev.dingtalk.com → 应用开发 → 企业内部应用 → 创建应用" + - "在「应用能力」中添加「机器人」" + - "在「凭证与基础信息」复制 AppKey 和 AppSecret" + - "在「权限管理」中添加消息相关权限" + console_url: "https://open-dev.dingtalk.com" + +validation: + method: dingtalk_token_check + timeout_s: 10 + +adapter: + module: leapflow.gateway.adapters.dingtalk + class: DingTalkAdapter + dependencies: [dingtalk-stream] diff --git a/src/leapflow/gateway/manifests/feishu.yaml b/src/leapflow/gateway/manifests/feishu.yaml new file mode 100644 index 0000000..56c9a99 --- /dev/null +++ b/src/leapflow/gateway/manifests/feishu.yaml @@ -0,0 +1,50 @@ +platform_id: feishu +display_name: "飞书 (Feishu/Lark)" +description: "飞书企业即时通讯平台" +category: im + +credentials: + - key: app_id + label: App ID + required: true + secret: false + help_zh: "登录 open.feishu.cn → 创建企业自建应用 → 凭证与基础信息 → 复制 App ID" + help_en: "Login open.feishu.cn → Create App → Credentials → Copy App ID" + - key: app_secret + label: App Secret + required: true + secret: true + help_zh: "同页面 → 复制 App Secret" + help_en: "Same page → Copy App Secret" + +options: + - key: connection_mode + label: 连接模式 + type: choice + choices: [websocket, webhook] + default: websocket + help_zh: "WebSocket 无需公网 IP（推荐），Webhook 需要公网回调地址" + help_en: "WebSocket needs no public IP (recommended), Webhook requires public callback URL" + +setup_guide: + summary_zh: | + 接入飞书需要在飞书开放平台创建一个企业自建应用并开启机器人能力。 + 完成后请提供 App ID 和 App Secret。 + summary_en: | + To connect Feishu, create an enterprise app on open.feishu.cn + with bot capability enabled. Then provide your App ID and App Secret. + steps_zh: + - "打开 https://open.feishu.cn/app → 点击「创建企业自建应用」" + - "在「应用能力」中启用「机器人」" + - "在「凭证与基础信息」页面复制 App ID 和 App Secret" + - "在「权限管理」中添加：im:message、im:message.group_at_msg" + console_url: "https://open.feishu.cn/app" + +validation: + method: feishu_token_check + timeout_s: 10 + +adapter: + module: leapflow.gateway.adapters.feishu + class: FeishuAdapter + dependencies: [lark-oapi] diff --git a/src/leapflow/gateway/manifests/telegram.yaml b/src/leapflow/gateway/manifests/telegram.yaml new file mode 100644 index 0000000..76c0bd9 --- /dev/null +++ b/src/leapflow/gateway/manifests/telegram.yaml @@ -0,0 +1,52 @@ +platform_id: telegram +display_name: Telegram +description: Telegram Messenger +category: social + +credentials: + - key: bot_token + label: Bot Token + required: true + secret: true + help_zh: "在 Telegram 中搜索 @BotFather → 发送 /newbot → 按提示创建 → 复制 token" + help_en: "Search @BotFather in Telegram → /newbot → Follow prompts → Copy token" + +options: + - key: transport + label: 传输模式 + type: choice + choices: [polling, webhook] + default: polling + help_zh: "polling 无需公网 IP（推荐），webhook 需要公网 HTTPS 地址" + help_en: "polling needs no public IP (recommended), webhook requires public HTTPS URL" + - key: webhook_url + label: Webhook URL + type: string + required: false + advanced: true + depends_on: + transport: webhook + help_zh: "Webhook 模式下的公网回调地址" + help_en: "Public callback URL for webhook mode" + +setup_guide: + summary_zh: | + 接入 Telegram 只需一个 Bot Token。 + 在 Telegram 中搜索 @BotFather，发送 /newbot 创建机器人并复制 token。 + summary_en: | + To connect Telegram, you only need a Bot Token. + Search @BotFather in Telegram, send /newbot, and copy the token. + steps_zh: + - "在 Telegram 中搜索 @BotFather 并开始对话" + - "发送 /newbot，按提示输入机器人名称和用户名" + - "复制返回的 Bot Token（格式如 123456:ABC-DEF...）" + console_url: "https://t.me/BotFather" + +validation: + method: telegram_getme + timeout_s: 10 + +adapter: + module: leapflow.gateway.adapters.telegram + class: TelegramAdapter + dependencies: [aiohttp] diff --git a/src/leapflow/gateway/manifests/webhook.yaml b/src/leapflow/gateway/manifests/webhook.yaml new file mode 100644 index 0000000..4c0ee78 --- /dev/null +++ b/src/leapflow/gateway/manifests/webhook.yaml @@ -0,0 +1,50 @@ +platform_id: webhook +display_name: "Webhook (Generic)" +description: Generic HTTP webhook receiver for custom integrations +category: webhook + +credentials: + - key: webhook_secret + label: Webhook Secret + required: false + secret: true + help_zh: "HMAC 签名密钥（推荐设置，用于验证请求来源）" + help_en: "HMAC signing secret (recommended, used to verify request origin)" + +options: + - key: host + label: 绑定地址 + type: string + default: "127.0.0.1" + help_zh: "HTTP 服务绑定地址" + help_en: "HTTP server bind address" + - key: port + label: 端口 + type: string + default: "9090" + help_zh: "HTTP 服务端口" + help_en: "HTTP server port" + - key: path + label: 路径 + type: string + default: "/webhook" + help_zh: "Webhook 接收路径" + help_en: "Webhook receive path" + +setup_guide: + summary_zh: | + 通用 Webhook 接收器，接受 HTTP POST 请求并触发 Agent 处理。 + 可选设置 HMAC 签名密钥以验证请求来源。 + summary_en: | + Generic webhook receiver that accepts HTTP POST requests + and triggers agent processing. Optionally set an HMAC secret + to verify request origin. + +validation: + method: "" + timeout_s: 5 + +adapter: + module: leapflow.gateway.adapters.webhook + class: WebhookAdapter + dependencies: [aiohttp] diff --git a/src/leapflow/gateway/mixin.py b/src/leapflow/gateway/mixin.py new file mode 100644 index 0000000..5fbfa14 --- /dev/null +++ b/src/leapflow/gateway/mixin.py @@ -0,0 +1,94 @@ +"""Default implementations for optional ``PlatformAdapter`` capabilities. + +Adapters mix this in to get graceful degradation for methods they do not +natively support. The ``Protocol`` defines the contract; this mixin +provides sensible defaults. Adapters override only what they can do +natively. + +Separated from the Protocol to avoid the Hermes 5.6K-line base-class +problem — each adapter composes behaviour freely. +""" +from __future__ import annotations + +from typing import Sequence + +from leapflow.gateway.protocol import OutboundContent, SendResult, SendTarget + + +class PlatformAdapterMixin: + """Optional mixin — graceful degradation for unsupported methods. + + Usage:: + + class FeishuAdapter(PlatformAdapterMixin): + supports_async_delivery = True + max_message_length = 8000 + ... + + # Override natively-supported methods; inherit degradation for the rest. + """ + + supports_async_delivery: bool = True + splits_long_messages: bool = False + max_message_length: int = 4000 + + # ── Message editing ────────────────────────────────────── + + async def edit_message( + self, + target: SendTarget, + message_id: str, + content: OutboundContent, + *, + finalize: bool = False, + ) -> SendResult: + return SendResult(ok=False, error="edit not supported") + + # ── Rich media (degrade to text) ───────────────────────── + + async def send_image( + self, + target: SendTarget, + image_url: str, + caption: str = "", + **kw: object, + ) -> SendResult: + text = f"[Image] {image_url}" + if caption: + text = f"{caption}\n{text}" + return await self.send(target, OutboundContent(text=text)) # type: ignore[attr-defined] + + async def send_document( + self, + target: SendTarget, + file_name: str, + caption: str = "", + **kw: object, + ) -> SendResult: + """Degrade to a friendly notice — never leaks host file paths.""" + text = f"[Document: {file_name}]" + if caption: + text = f"{caption}\n{text}" + return await self.send(target, OutboundContent(text=text)) # type: ignore[attr-defined] + + # ── Interactive UX (degrade to numbered list) ───────────── + + async def send_clarify( + self, + target: SendTarget, + question: str, + choices: Sequence[str], + **kw: object, + ) -> SendResult: + lines = [question, ""] + for i, choice in enumerate(choices, 1): + lines.append(f"{i}. {choice}") + return await self.send(target, OutboundContent(text="\n".join(lines))) # type: ignore[attr-defined] + + # ── Typing indicators (no-op) ──────────────────────────── + + async def send_typing(self, target: SendTarget, **kw: object) -> None: + pass + + async def stop_typing(self, target: SendTarget) -> None: + pass diff --git a/src/leapflow/gateway/protocol.py b/src/leapflow/gateway/protocol.py new file mode 100644 index 0000000..c77a6c3 --- /dev/null +++ b/src/leapflow/gateway/protocol.py @@ -0,0 +1,175 @@ +"""Gateway protocol types and adapter interface. + +Defines the contract between platform adapters and the gateway server. +All domain types are immutable (frozen dataclass) for safety at trust +boundaries. ``PlatformAdapter`` uses ``typing.Protocol`` — no base-class +inheritance — so adapters satisfy the contract via structural subtyping. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Optional, + Protocol, + Tuple, + runtime_checkable, +) + + +# ═══════════════════════════════════════════════════════════════ +# Inbound types (platform → gateway) +# ═══════════════════════════════════════════════════════════════ + +@dataclass(frozen=True) +class MessageSource: + """Deterministic identity for message origin. + + Platform-specific normalisation (WhatsApp JID, Feishu union_id, …) + stays in the adapter layer — this type is platform-agnostic. + """ + + platform: str + chat_id: str + chat_type: str = "dm" + user_id: str = "" + user_name: str = "" + thread_id: str = "" + scope_id: str = "" + profile: str = "default" + + +@dataclass(frozen=True) +class MediaAttachment: + """A media item attached to a message.""" + + url: str + media_type: str = "" + filename: str = "" + size_bytes: int = 0 + + +@dataclass(frozen=True) +class InboundMessage: + """Platform-normalised inbound message.""" + + source: MessageSource + text: str + message_id: str + media: Tuple[MediaAttachment, ...] = () + reply_to_id: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + timestamp: float = field(default_factory=time.time) + + +# ═══════════════════════════════════════════════════════════════ +# Outbound types (gateway → platform) +# ═══════════════════════════════════════════════════════════════ + +@dataclass(frozen=True) +class SendTarget: + """Where to send a message.""" + + platform: str + chat_id: str + thread_id: str = "" + reply_to_id: str = "" + + +@dataclass(frozen=True) +class OutboundContent: + """Content to send to a platform. + + ``metadata`` serves as an extensibility escape-hatch: platform-specific + data, WebUI control hints (cards, forms, buttons) — anything that the + core doesn't need to understand. + """ + + text: str + media: Tuple[MediaAttachment, ...] = () + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class SendResult: + """Result of a send operation.""" + + ok: bool + message_id: str = "" + error: str = "" + + +# ═══════════════════════════════════════════════════════════════ +# Platform adapter contract +# ═══════════════════════════════════════════════════════════════ + +MessageHandler = Callable[[InboundMessage], Awaitable[None]] + + +@runtime_checkable +class PlatformAdapter(Protocol): + """Contract for a platform connection. + + Each adapter manages its own connection lifecycle. + The gateway sets ``on_message`` before calling ``connect()``. + + Capability flags are declared as class-level attributes. Callers + read them via ``getattr()`` to determine platform-specific behaviour + without ``isinstance`` checks. + """ + + @property + def platform_id(self) -> str: ... + + # ── Capability flags (class-level declarations) ────────── + supports_async_delivery: bool + splits_long_messages: bool + max_message_length: int + + # ── Message handler (set by server before connect) ─────── + @property + def on_message(self) -> Optional[MessageHandler]: ... + + @on_message.setter + def on_message(self, handler: MessageHandler) -> None: ... + + # ── Lifecycle ──────────────────────────────────────────── + async def connect(self, *, is_reconnect: bool = False) -> None: + """Establish connection. Raises on failure. + + When *is_reconnect* is ``True``, preserve server-side message + queues (e.g. Telegram offset, Feishu subscription). + """ + ... + + async def disconnect(self) -> None: + """Cleanly disconnect from the platform.""" + ... + + # ── Messaging ──────────────────────────────────────────── + async def send( + self, + target: SendTarget, + content: OutboundContent, + ) -> SendResult: + """Send a message to the platform.""" + ... + + +# ═══════════════════════════════════════════════════════════════ +# Runtime status +# ═══════════════════════════════════════════════════════════════ + +@dataclass(frozen=True) +class PlatformStatus: + """Runtime status of a known platform.""" + + platform_id: str + connected: bool + connected_since: float = 0.0 + error: str = "" + metadata: Dict[str, Any] = field(default_factory=dict) diff --git a/src/leapflow/gateway/router.py b/src/leapflow/gateway/router.py new file mode 100644 index 0000000..8798910 --- /dev/null +++ b/src/leapflow/gateway/router.py @@ -0,0 +1,211 @@ +"""Gateway message router — per-session LLM processing for inbound platform messages. + +Sits between ``GatewayServer`` (message ingress) and the LLM/tool layer +(response generation). Each external session gets independent message +history so concurrent conversations on different platforms don't collide +with each other or the interactive CLI. + +Tool support +~~~~~~~~~~~~ +The router can optionally execute a **restricted** set of safe tools +(read-only: memory_search, time_get, skills_list, etc.) during inbound +message processing. Dangerous tools (shell_run, file_write, delegate_task) +are excluded by default — configurable via ``allowed_tools``. + +Module boundary +~~~~~~~~~~~~~~~ +Depends on ``leapflow.llm`` (LLM provider interface) and optionally on +tool handlers, but **not** on ``AgentEngine`` or ``cli/``. ``Context`` +is the sole point that wires all dependencies. +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any, Callable, Coroutine, Dict, List, Optional, Sequence + +from leapflow.gateway.protocol import ( + InboundMessage, + MessageSource, + OutboundContent, + SendTarget, +) + +logger = logging.getLogger(__name__) + +SendFn = Callable[[MessageSource, str], Coroutine[Any, Any, None]] + +SAFE_TOOLS: frozenset[str] = frozenset({ + "memory_search", "memory_add", + "time_get", "env_info", + "skills_list", "skill_view", + "text_search", + "gateway_connect", "gateway_send", +}) + + +class GatewayRouter: + """Routes inbound gateway messages through LLM with per-session history. + + Parameters + ---------- + llm + Any object implementing ``achat(messages, *, stream=False, **kw)``. + system_prompt + System message prepended to every new session. + send_fn + ``async (source, reply_text) -> None`` — called to deliver the + LLM response back to the originating conversation. + tool_definitions + OpenAI-format tool schemas. Filtered to ``allowed_tools`` on init. + tool_handlers + ``name → async handler`` mapping. Filtered to ``allowed_tools``. + allowed_tools + Frozenset of tool names permitted for gateway sessions. + Defaults to ``SAFE_TOOLS`` (read-only, no shell/file-write). + max_history + Maximum messages retained per session before tail-trimming. + max_tool_rounds + Maximum number of LLM→tool→LLM rounds before forcing a text reply. + """ + + def __init__( + self, + *, + llm: Any, + system_prompt: str = "", + send_fn: SendFn, + tool_definitions: Sequence[Dict[str, Any]] = (), + tool_handlers: Optional[Dict[str, Any]] = None, + allowed_tools: frozenset[str] = SAFE_TOOLS, + max_history: int = 50, + max_tool_rounds: int = 3, + ) -> None: + self._llm = llm + self._system_prompt = system_prompt + self._send_fn = send_fn + self._max_history = max_history + self._max_tool_rounds = max_tool_rounds + self._sessions: Dict[str, List[Dict[str, Any]]] = {} + self._locks: Dict[str, asyncio.Lock] = {} + + self._tool_defs = [ + td for td in tool_definitions + if td.get("function", {}).get("name", "") in allowed_tools + ] + all_handlers = tool_handlers or {} + self._tool_handlers = { + k: v for k, v in all_handlers.items() if k in allowed_tools + } + + async def handle_message( + self, + message: InboundMessage, + session_key: str, + ) -> None: + """Process an inbound message: add to history → LLM call → reply. + + Serialises per-session to prevent interleaving within one chat. + """ + lock = self._locks.setdefault(session_key, asyncio.Lock()) + async with lock: + await self._process(message, session_key) + + async def _process( + self, + message: InboundMessage, + session_key: str, + ) -> None: + history = self._sessions.get(session_key) + if history is None: + history = [] + if self._system_prompt: + history.append({"role": "system", "content": self._system_prompt}) + self._sessions[session_key] = history + + history.append({"role": "user", "content": message.text}) + self._trim_history(history) + + reply = await self._llm_with_tools(history) + if not reply: + return + + history.append({"role": "assistant", "content": reply}) + try: + await self._send_fn(message.source, reply) + except Exception: + logger.error( + "Failed to send reply for session %s", + session_key, + exc_info=True, + ) + + async def _llm_with_tools( + self, + history: List[Dict[str, Any]], + ) -> str: + """Call LLM with optional tool loop (bounded rounds).""" + llm_kwargs: Dict[str, Any] = {} + if self._tool_defs: + llm_kwargs["tools"] = self._tool_defs + + for _round in range(self._max_tool_rounds + 1): + try: + resp = await self._llm.achat(history, stream=False, **llm_kwargs) + except Exception: + logger.error("LLM call failed in gateway router", exc_info=True) + return "" + + tool_calls = getattr(resp, "tool_calls", None) + if not tool_calls: + return (resp.content or "").strip() + + for tc in tool_calls: + handler = self._tool_handlers.get(tc.name) + if handler is None: + history.append({ + "role": "tool", + "tool_call_id": getattr(tc, "id", ""), + "content": f"Unknown tool: {tc.name}", + }) + continue + try: + tool_args = getattr(tc, "arguments", {}) + if isinstance(tool_args, str): + import json + tool_args = json.loads(tool_args) + result = await asyncio.wait_for(handler(tool_args), timeout=30) + result_text = str(result)[:2000] + except asyncio.TimeoutError: + result_text = f"Tool '{tc.name}' timed out" + except Exception as exc: + result_text = f"Tool error: {type(exc).__name__}" + + history.append({ + "role": "tool", + "tool_call_id": getattr(tc, "id", ""), + "content": result_text, + }) + + logger.warning("Gateway router: max tool rounds exceeded") + return "" + + def _trim_history(self, history: List[Dict[str, Any]]) -> None: + """Keep history under budget by discarding oldest non-system messages.""" + if len(history) <= self._max_history: + return + has_system = history and history[0].get("role") == "system" + keep = self._max_history - (1 if has_system else 0) + if has_system: + history[1:] = history[-keep:] + else: + history[:] = history[-keep:] + + def clear_session(self, session_key: str) -> None: + """Remove all state for a session (called on disconnect/timeout).""" + self._sessions.pop(session_key, None) + self._locks.pop(session_key, None) + + @property + def active_sessions(self) -> int: + return len(self._sessions) diff --git a/src/leapflow/gateway/server.py b/src/leapflow/gateway/server.py new file mode 100644 index 0000000..ecb7ec1 --- /dev/null +++ b/src/leapflow/gateway/server.py @@ -0,0 +1,393 @@ +"""Gateway server — manages platform adapters and routes messages. + +Intentionally thin (< 250 lines): session / transcript persistence, +agent execution, memory, and skills are all delegated to downstream +subscribers. The gateway only owns: + +- Adapter lifecycle (connect / disconnect / reconnect) +- Message normalisation (inbound → SessionKey → handler callback) +- Event notification (optional callback for loose coupling) + +Designed for two deployment modes: + +1. **In-process** — called directly from ``Context`` (single-window CLI) +2. **Daemon-backed** — called from ``leapd`` with ``LeapService`` RPC +""" +from __future__ import annotations + +import asyncio +import importlib +import logging +import time +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from leapflow.gateway.config_store import GatewayConfigStore +from leapflow.gateway.credential_vault import CredentialVault +from leapflow.gateway.events import ( + GatewayMessageReceived, + GatewaySessionCreated, + GatewaySessionEnded, +) +from leapflow.gateway.manifest import ManifestLoader, PlatformManifest +from leapflow.gateway.protocol import ( + InboundMessage, + MessageSource, + OutboundContent, + PlatformAdapter, + PlatformStatus, + SendResult, + SendTarget, +) +from leapflow.gateway.session_router import SessionKey, build_session_key +from leapflow.gateway.validators import validate_credentials + +logger = logging.getLogger(__name__) + +EventCallback = Callable[..., Any] + + +class GatewayServer: + """Manages platform adapter lifecycle and message routing.""" + + def __init__( + self, + profile_dir: Path, + *, + extra_manifest_dirs: Optional[List[Path]] = None, + on_event: Optional[EventCallback] = None, + ) -> None: + self._profile_dir = profile_dir + self._vault = CredentialVault(profile_dir) + self._config_store = GatewayConfigStore(profile_dir, self._vault) + self._manifest_loader = ManifestLoader(extra_dirs=extra_manifest_dirs) + self._manifests: Dict[str, PlatformManifest] = {} + self._adapters: Dict[str, PlatformAdapter] = {} + self._connected_since: Dict[str, float] = {} + self._known_sessions: set[SessionKey] = set() + self._message_handler: Optional[Callable[..., Any]] = None + self._on_event = on_event + self._started = False + + # ── Manifest discovery ─────────────────────────────────── + + def discover_manifests(self) -> Dict[str, PlatformManifest]: + """Load all available platform manifests.""" + self._manifests = self._manifest_loader.discover() + self._config_store.set_manifests(self._manifests) + return self._manifests + + @property + def manifests(self) -> Dict[str, PlatformManifest]: + if not self._manifests: + self.discover_manifests() + return self._manifests + + def set_message_handler(self, handler: Callable[..., Any]) -> None: + """Set the callback for inbound messages (typically engine dispatch).""" + self._message_handler = handler + + # ── Platform connection ────────────────────────────────── + + async def connect_platform( + self, + platform_id: str, + credentials: Dict[str, str], + options: Optional[Dict[str, Any]] = None, + *, + is_reconnect: bool = False, + ) -> Dict[str, Any]: + """Validate, persist, and connect a platform adapter. + + Called by the ``gateway_connect`` tool during conversation, or + by ``start()`` for auto-reconnect. When *is_reconnect* is + ``True``, adapters preserve server-side message queues. + The return value **never** contains credential values. + """ + manifest = self._manifests.get(platform_id) + if manifest is None: + return {"ok": False, "error": f"Unknown platform: {platform_id}"} + + for cred in manifest.credentials: + if cred.required and not credentials.get(cred.key): + return {"ok": False, "error": f"Missing required field: {cred.label}"} + + validation_info = "" + if manifest.validation_method: + ok, msg = await validate_credentials( + manifest.validation_method, + credentials, + timeout_s=manifest.validation_timeout_s, + ) + if not ok: + return {"ok": False, "error": msg} + validation_info = msg + + self._config_store.save_platform( + platform_id, credentials, options or {}, manifest, + ) + + if manifest.adapter: + try: + adapter = self._instantiate_adapter( + manifest, credentials, options or {}, + ) + adapter.on_message = self._on_inbound_message + await adapter.connect(is_reconnect=is_reconnect) + self._adapters[platform_id] = adapter + self._connected_since[platform_id] = time.time() + except Exception as exc: + from leapflow.security.redact import redact_sensitive_text + + safe_err = redact_sensitive_text(str(exc), force=True) + logger.error("Failed to connect %s: %s", platform_id, safe_err) + return {"ok": False, "error": f"Connection failed: {safe_err}"} + + display = manifest.display_name + return { + "ok": True, + "status": "connected", + "platform": display, + "info": validation_info, + } + + async def disconnect_platform(self, platform_id: str) -> Dict[str, Any]: + """Disconnect a platform adapter and clean up its sessions.""" + adapter = self._adapters.pop(platform_id, None) + self._connected_since.pop(platform_id, None) + + affected = {k for k in self._known_sessions if k.platform == platform_id} + self._known_sessions -= affected + for key in affected: + await self._emit_event(GatewaySessionEnded( + session_key=str(key), + reason="platform_disconnect", + )) + + if adapter: + try: + await adapter.disconnect() + except Exception: + logger.warning( + "Error disconnecting %s", platform_id, exc_info=True, + ) + return {"ok": True, "status": "disconnected"} + + def platform_status(self) -> List[PlatformStatus]: + """Return status of all known platforms.""" + result: List[PlatformStatus] = [] + config = self._config_store.load() + for pid in self._manifests: + if pid in self._adapters: + result.append(PlatformStatus( + platform_id=pid, + connected=True, + connected_since=self._connected_since.get(pid, 0), + )) + elif pid in config.platforms and config.platforms[pid].enabled: + result.append(PlatformStatus( + platform_id=pid, + connected=False, + error="configured but not connected", + )) + else: + result.append(PlatformStatus( + platform_id=pid, + connected=False, + )) + return result + + # ── Lifecycle ──────────────────────────────────────────── + + async def start(self) -> int: + """Auto-connect all platforms in ``auto_connect`` list. + + Returns the number of successfully connected platforms. + """ + self.discover_manifests() + config = self._config_store.load() + connected = 0 + for pid in config.auto_connect: + manifest = self._manifests.get(pid) + pc = config.platforms.get(pid) + if not (manifest and pc and pc.enabled): + continue + try: + creds = self._config_store.load_platform_credentials( + pid, manifest, + ) + if not creds: + continue + result = await self.connect_platform( + pid, creds, pc.options, + is_reconnect=True, + ) + if result.get("ok"): + connected += 1 + except Exception as exc: + logger.warning( + "gateway.auto_connect_failed platform=%s error=%s", + pid, + exc, + exc_info=True, + ) + self._started = True + return connected + + async def stop(self) -> None: + """Disconnect all adapters.""" + for pid in list(self._adapters): + await self.disconnect_platform(pid) + self._started = False + + # ── Outbound messaging ────────────────────────────────── + + async def send_message( + self, + platform_id: str, + chat_id: str, + text: str, + *, + thread_id: str = "", + reply_to_id: str = "", + ) -> Dict[str, Any]: + """Send a message to any connected platform conversation. + + Public API for the ``gateway_send`` tool and any component that + needs proactive outbound messaging. Unlike ``send_reply``, + this does not require a prior inbound ``MessageSource``. + """ + adapter = self._adapters.get(platform_id) + if adapter is None: + return {"ok": False, "error": f"Platform '{platform_id}' is not connected"} + target = SendTarget( + platform=platform_id, + chat_id=chat_id, + thread_id=thread_id, + reply_to_id=reply_to_id, + ) + content = OutboundContent(text=text) + try: + result = await adapter.send(target, content) + return { + "ok": result.ok, + "message_id": result.message_id, + **({"error": result.error} if result.error else {}), + } + except Exception as exc: + from leapflow.security.redact import redact_sensitive_text + safe_err = redact_sensitive_text(str(exc), force=True) + return {"ok": False, "error": f"Send failed: {safe_err}"} + + async def send_reply( + self, + source: MessageSource, + text: str, + ) -> Optional[SendResult]: + """Send a reply back to the originating conversation. + + Convenience wrapper around ``send_message`` for ``GatewayRouter``. + """ + adapter = self._adapters.get(source.platform) + if adapter is None: + logger.warning( + "No adapter for %s, cannot send reply", source.platform, + ) + return None + target = SendTarget( + platform=source.platform, + chat_id=source.chat_id, + thread_id=source.thread_id, + ) + content = OutboundContent(text=text) + return await adapter.send(target, content) + + def remove_platform_config(self, platform_id: str) -> None: + """Remove a platform's saved configuration from disk. + + Public API — avoids callers needing direct ``_config_store`` access. + """ + self._config_store.remove_platform(platform_id) + + # ── Message routing ────────────────────────────────────── + + async def _on_inbound_message(self, message: InboundMessage) -> None: + """Route an inbound message via ``SessionKey``.""" + session_key = build_session_key(message.source) + is_new_session = session_key not in self._known_sessions + + if is_new_session: + self._known_sessions.add(session_key) + await self._emit_event(GatewaySessionCreated( + session_key=str(session_key), + source=message.source, + )) + + await self._emit_event(GatewayMessageReceived( + source=message.source, + session_key=str(session_key), + text=message.text, + media_urls=tuple(m.url for m in message.media), + )) + + if self._message_handler is None: + logger.warning( + "No message handler set, dropping message from %s", + message.source.platform, + ) + return + + try: + result = self._message_handler(message, str(session_key)) + if asyncio.iscoroutine(result): + await result + except Exception: + logger.error( + "Error handling message from %s", + message.source.platform, + exc_info=True, + ) + + async def _emit_event(self, event: object) -> None: + """Dispatch an event to the optional callback (non-blocking).""" + if self._on_event is None: + return + try: + result = self._on_event(event) + if asyncio.iscoroutine(result): + await result + except Exception: + logger.debug("Event callback error", exc_info=True) + + # ── Internal ───────────────────────────────────────────── + + @staticmethod + def _instantiate_adapter( + manifest: PlatformManifest, + credentials: Dict[str, str], + options: Dict[str, Any], + ) -> PlatformAdapter: + """Dynamically import and instantiate a platform adapter. + + Adapter modules are loaded on-demand — platforms whose SDK is not + installed never affect startup (deferred loading). If the import + fails due to missing dependencies, the error message includes a + ``pip install`` hint derived from the manifest. + """ + assert manifest.adapter is not None + try: + module = importlib.import_module(manifest.adapter.module) + except ImportError as exc: + deps = manifest.adapter.dependencies + missing_name = getattr(exc, "name", "") or "" + module_name = manifest.adapter.module + module_missing = missing_name == module_name or module_name.startswith(f"{missing_name}.") + if deps and module_missing: + hint = f"pip install {' '.join(deps)}" + raise ImportError( + f"Adapter module '{module_name}' not found. " + f"Install dependencies: {hint}", + ) from None + raise + cls = getattr(module, manifest.adapter.class_name) + return cls(**credentials, **options) diff --git a/src/leapflow/gateway/session_router.py b/src/leapflow/gateway/session_router.py new file mode 100644 index 0000000..4be4514 --- /dev/null +++ b/src/leapflow/gateway/session_router.py @@ -0,0 +1,91 @@ +"""Structured session routing for gateway messages. + +``SessionKey`` is an immutable domain type that replaces simple string +concatenation. Code logic accesses fields by name (``key.platform``, +``key.chat_id``); ``__str__()`` produces a colon-separated form **only** +for serialisation (DuckDB storage, dict keys, logs). + +``build_session_key()`` is the single source of truth for routing — +platform-specific normalisation (WhatsApp JID, Feishu union_id, …) +stays in the adapter layer. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from leapflow.gateway.protocol import MessageSource + +_SINGLE_CHAR_UNSAFE = frozenset("/\\:") +_SUBSTRING_UNSAFE = ("..",) + + +@dataclass(frozen=True) +class SessionKey: + """Structured session identifier. + + ``frozen=True`` makes it hashable — usable as a dict key directly via + ``__hash__``. ``__str__()`` is reserved for serialisation only. + """ + + profile: str + platform: str + chat_type: str + chat_id: str + thread_id: str = "" + user_id: str = "" + + def __post_init__(self) -> None: + for field_name in ("profile", "platform", "chat_type", "chat_id", "thread_id", "user_id"): + value = getattr(self, field_name) + normalized = "" if value is None else str(value) + object.__setattr__(self, field_name, normalized) + + for field_name in ("profile", "platform", "chat_type", "chat_id", "thread_id", "user_id"): + field_val = getattr(self, field_name) + if not field_val: + continue + if any(c in field_val for c in _SINGLE_CHAR_UNSAFE): + raise ValueError( + f"Unsafe characters in session key field {field_name}: {field_val!r}", + ) + if any(sub in field_val for sub in _SUBSTRING_UNSAFE): + raise ValueError( + f"Unsafe substring in session key field {field_name}: {field_val!r}", + ) + + def __str__(self) -> str: + parts = [self.profile, self.platform, self.chat_type, self.chat_id] + if self.thread_id: + parts.append(self.thread_id) + if self.user_id: + parts.append(self.user_id) + return ":".join(parts) + + +def build_session_key( + source: MessageSource, + *, + per_user_in_group: bool = True, +) -> SessionKey: + """Build a deterministic ``SessionKey`` from a message source. + + Group chats are isolated per-user by default (each user gets their + own session). DMs are keyed by ``chat_id`` (1:1 with user). + Threads append ``thread_id`` to the parent key. + """ + user_id = "" + if ( + source.chat_type in ("group", "channel") + and per_user_in_group + and source.user_id + ): + user_id = source.user_id + + return SessionKey( + profile=source.profile, + platform=source.platform, + chat_type=source.chat_type, + chat_id=source.chat_id, + thread_id=source.thread_id, + user_id=user_id, + ) diff --git a/src/leapflow/gateway/validators.py b/src/leapflow/gateway/validators.py new file mode 100644 index 0000000..ae70b9e --- /dev/null +++ b/src/leapflow/gateway/validators.py @@ -0,0 +1,153 @@ +"""Platform credential validation functions. + +Each validator is a simple async function: +``(credentials: Dict[str, str]) → (ok: bool, error_or_info: str)``. + +Validators are registered by name so YAML manifests can reference them +declaratively. New platforms add a validator + register call — zero +core changes. +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Awaitable, Callable, Dict, Optional, Tuple + +logger = logging.getLogger(__name__) + +ValidatorFn = Callable[[Dict[str, str]], Awaitable[Tuple[bool, str]]] + +_registry: Dict[str, ValidatorFn] = {} + + +# ═══════════════════════════════════════════════════════════════ +# Registry API +# ═══════════════════════════════════════════════════════════════ + +def register_validator(name: str, fn: ValidatorFn) -> None: + """Register a credential validator by name.""" + _registry[name] = fn + + +def get_validator(name: str) -> Optional[ValidatorFn]: + """Retrieve a registered validator (or ``None``).""" + return _registry.get(name) + + +async def validate_credentials( + method_name: str, + credentials: Dict[str, str], + *, + timeout_s: float = 10.0, +) -> Tuple[bool, str]: + """Run the named validator with a timeout. + + Returns ``(True, "")`` if no validator is registered (safe default). + Errors are redacted before returning. + """ + fn = _registry.get(method_name) + if fn is None: + return True, "" + + try: + ok, msg = await asyncio.wait_for(fn(credentials), timeout=timeout_s) + return ok, msg + except asyncio.TimeoutError: + return False, f"Validation timed out after {timeout_s}s" + except Exception as exc: + from leapflow.security.redact import redact_sensitive_text + + safe_error = redact_sensitive_text(str(exc), force=True) + return False, f"Validation error: {safe_error}" + + +# ═══════════════════════════════════════════════════════════════ +# Built-in validators (for bundled manifests) +# ═══════════════════════════════════════════════════════════════ + +def _make_client_timeout() -> "aiohttp.ClientTimeout": + """Create a strict per-request timeout for credential validation. + + Imported lazily to avoid top-level ``aiohttp`` dependency. + """ + import aiohttp + + return aiohttp.ClientTimeout(total=8, connect=5) + + +async def _feishu_token_check(credentials: Dict[str, str]) -> Tuple[bool, str]: + """Validate Feishu credentials by fetching ``tenant_access_token``.""" + import aiohttp + + app_id = credentials.get("app_id", "") + app_secret = credentials.get("app_secret", "") + if not app_id or not app_secret: + return False, "Missing app_id or app_secret" + + url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal" + async with aiohttp.ClientSession( + timeout=_make_client_timeout(), + trace_configs=[], + ) as session: + async with session.post( + url, + json={"app_id": app_id, "app_secret": app_secret}, + ) as resp: + data = await resp.json() + if data.get("code") == 0 and data.get("tenant_access_token"): + return True, "" + return False, data.get("msg", "Unknown error from Feishu API") + + +async def _dingtalk_token_check(credentials: Dict[str, str]) -> Tuple[bool, str]: + """Validate DingTalk credentials by fetching ``access_token``.""" + import aiohttp + + app_key = credentials.get("app_key", "") + app_secret = credentials.get("app_secret", "") + if not app_key or not app_secret: + return False, "Missing app_key or app_secret" + + url = "https://oapi.dingtalk.com/gettoken" + params = {"appkey": app_key, "appsecret": app_secret} + async with aiohttp.ClientSession( + timeout=_make_client_timeout(), + trace_configs=[], + ) as session: + async with session.get(url, params=params) as resp: + data = await resp.json() + if data.get("errcode") == 0 and data.get("access_token"): + return True, "" + return False, data.get("errmsg", "Unknown error from DingTalk API") + + +async def _telegram_getme(credentials: Dict[str, str]) -> Tuple[bool, str]: + """Validate Telegram bot token via ``getMe`` API. + + The token is embedded in the URL path (Telegram API convention). + ``trace_configs=[]`` disables aiohttp request tracing to prevent + the token-containing URL from appearing in debug logs. + """ + import aiohttp + + token = credentials.get("bot_token", "") + if not token: + return False, "Missing bot_token" + + url = f"https://api.telegram.org/bot{token}/getMe" + async with aiohttp.ClientSession( + timeout=_make_client_timeout(), + trace_configs=[], + ) as session: + async with session.get(url) as resp: + data = await resp.json() + if data.get("ok"): + bot_name = data.get("result", {}).get("username", "unknown") + return True, f"Bot: @{bot_name}" + return False, data.get("description", "Invalid bot token") + + +# ── Auto-register built-in validators ──────────────────────── +register_validator("feishu_token_check", _feishu_token_check) +register_validator("dingtalk_token_check", _dingtalk_token_check) +register_validator("telegram_getme", _telegram_getme) diff --git a/src/leapflow/llm/model_capabilities.py b/src/leapflow/llm/model_capabilities.py index 3000add..1c50ae6 100644 --- a/src/leapflow/llm/model_capabilities.py +++ b/src/leapflow/llm/model_capabilities.py @@ -13,9 +13,11 @@ import logging import re -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, Dict, List, Optional, Protocol, runtime_checkable +from leapflow.config import DEFAULT_LLM_CONTEXT_LENGTH + logger = logging.getLogger(__name__) _IMAGE_TOKEN_ESTIMATE = 1600 @@ -25,7 +27,7 @@ class ModelCapabilities: """Capability descriptor for a specific model.""" - context_length: int = 128_000 + context_length: int = DEFAULT_LLM_CONTEXT_LENGTH max_output_tokens: int = 16_384 supports_tools: bool = True supports_vision: bool = False diff --git a/src/leapflow/llm/provider_chain.py b/src/leapflow/llm/provider_chain.py index 3c8c3e1..4a5e71e 100644 --- a/src/leapflow/llm/provider_chain.py +++ b/src/leapflow/llm/provider_chain.py @@ -14,19 +14,18 @@ """ from __future__ import annotations -import asyncio import json import logging -import os import time -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, AsyncIterator, Dict, FrozenSet, List, Optional, Protocol, runtime_checkable -from leapflow.llm.base import ChunkCallback, LLMChatResponse, LLMProvider, ToolCallInfo +from leapflow.config import DEFAULT_LLM_CONTEXT_LENGTH +from leapflow.llm.base import ChunkCallback, LLMChatResponse, LLMProvider logger = logging.getLogger(__name__) -_DEFAULT_CONTEXT_LENGTH = 128_000 +_DEFAULT_CONTEXT_LENGTH = DEFAULT_LLM_CONTEXT_LENGTH @dataclass(frozen=True) diff --git a/src/leapflow/memory/__init__.py b/src/leapflow/memory/__init__.py index 7b8e6b1..e130140 100644 --- a/src/leapflow/memory/__init__.py +++ b/src/leapflow/memory/__init__.py @@ -9,6 +9,7 @@ from leapflow.memory.providers import ( WorkingMemoryProvider, EpisodicMemoryProvider, SemanticMemoryProvider, EvolutionMemoryProvider, + NarrativeProvider, ) from leapflow.memory.providers.episodic import MemoryFragment from leapflow.memory.providers.semantic import MemoryHit @@ -34,5 +35,6 @@ def decay_weight( "MemoryToolSchema", "SignalDomain", "MemoryManager", "WorkingMemoryProvider", "EpisodicMemoryProvider", "SemanticMemoryProvider", "EvolutionMemoryProvider", + "NarrativeProvider", "MemoryFragment", "MemoryHit", "SkillEpisode", "decay_weight", ] diff --git a/src/leapflow/memory/providers/__init__.py b/src/leapflow/memory/providers/__init__.py index 52d1d9b..b08661b 100644 --- a/src/leapflow/memory/providers/__init__.py +++ b/src/leapflow/memory/providers/__init__.py @@ -4,10 +4,12 @@ from leapflow.memory.providers.episodic import EpisodicMemoryProvider from leapflow.memory.providers.semantic import SemanticMemoryProvider from leapflow.memory.providers.evolution import EvolutionMemoryProvider +from leapflow.memory.providers.narrative import NarrativeProvider __all__ = [ "WorkingMemoryProvider", "EpisodicMemoryProvider", "SemanticMemoryProvider", "EvolutionMemoryProvider", + "NarrativeProvider", ] diff --git a/src/leapflow/memory/providers/narrative.py b/src/leapflow/memory/providers/narrative.py new file mode 100644 index 0000000..1ce13ce --- /dev/null +++ b/src/leapflow/memory/providers/narrative.py @@ -0,0 +1,397 @@ +"""Narrative memory provider — pure-text Markdown for LLM-readable knowledge. + +Implements the narrative layer of the dual memory architecture: + +- **Narrative (this)**: Low-frequency, stable, LLM-readable text (MEMORY.md files) +- **Signal (SemanticMemoryProvider)**: High-frequency DuckDB signal/analytical data + +Storage layout (inside ``profiles//memory/``):: + + global/ + MEMORY.md # Global index (always loaded) + topics/.md # Detail files (lazy-loaded on search) + workspaces// + MEMORY.md # Project-specific index + topics/.md + +Token budget control: + +1. **Index always loaded** — ``MEMORY.md`` from global + current workspace +2. **Topics lazy-loaded** — ``topics/.md`` fetched on keyword match +3. **Conditional** — workspace scoping by cwd git root hash + +Access pattern: client-direct read/write (advisory file lock). No daemon needed. +""" +from __future__ import annotations + +import fcntl +import hashlib +import logging +import os +import re +import uuid +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from leapflow.memory.protocol import ( + MemoryEntry, + MemoryKind, + MemoryQuery, + MemoryToolSchema, +) + +logger = logging.getLogger(__name__) + +_NARRATIVE_KINDS = frozenset({ + MemoryKind.USER_PREFERENCE, + MemoryKind.FACT, + MemoryKind.SESSION_SUMMARY, +}) + +_KIND_SECTION = { + MemoryKind.USER_PREFERENCE: "User Preferences", + MemoryKind.FACT: "Facts", + MemoryKind.SESSION_SUMMARY: "Session Summaries", +} + +_MEMORY_HEADER = "# Memory\n\n" + + +def _slug(text: str) -> str: + """Convert text to a filesystem-safe slug.""" + s = re.sub(r"[^\w\s-]", "", text.lower().strip()) + return re.sub(r"[\s_]+", "-", s)[:60] or "untitled" + + +def _workspace_hash(workspace_path: str) -> str: + """Deterministic short hash for workspace directory isolation.""" + return hashlib.sha256(workspace_path.encode()).hexdigest()[:12] + + +def _atomic_write(path: Path, content: str) -> None: + """Write text atomically under a dedicated advisory lock.""" + path.parent.mkdir(parents=True, exist_ok=True) + lock_path = path.with_name(f"{path.name}.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + with open(lock_path, "a+", encoding="utf-8") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX) + _write_text_unlocked(path, content) + + +def _write_text_unlocked(path: Path, content: str) -> None: + """Replace a file atomically. Caller must hold the file lock.""" + tmp = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") + try: + with open(tmp, "w", encoding="utf-8") as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + tmp.replace(path) + except OSError: + tmp.unlink(missing_ok=True) + raise + + +def _locked_update(path: Path, update: Callable[[str], str]) -> str: + """Run read-modify-write under one lock and return the final content.""" + path.parent.mkdir(parents=True, exist_ok=True) + lock_path = path.with_name(f"{path.name}.lock") + with open(lock_path, "a+", encoding="utf-8") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX) + current = _read_text(path) or _MEMORY_HEADER + updated = update(current) + if updated != current: + _write_text_unlocked(path, updated) + return updated + + +def _read_text(path: Path) -> str: + """Read file content, returning empty string if missing.""" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return "" + + +class NarrativeProvider: + """Pure-text Markdown memory for LLM-readable knowledge. + + Accepts ``USER_PREFERENCE``, ``FACT``, and ``SESSION_SUMMARY`` + memory kinds. Entries are appended as bullet points to the + appropriate section in ``MEMORY.md``. + """ + + def __init__( + self, + memory_dir: Path, + *, + workspace_path: Optional[str] = None, + ) -> None: + self._memory_dir = memory_dir + self._workspace_path = workspace_path + self._workspace_hash = _workspace_hash(workspace_path) if workspace_path else None + + @property + def name(self) -> str: + return "narrative" + + @property + def _global_dir(self) -> Path: + return self._memory_dir / "global" + + @property + def _workspace_dir(self) -> Optional[Path]: + if self._workspace_hash is None: + return None + return self._memory_dir / "workspaces" / self._workspace_hash + + async def initialize(self, **kwargs: Any) -> None: + self._global_dir.mkdir(parents=True, exist_ok=True) + index = self._global_dir / "MEMORY.md" + if not index.exists(): + _atomic_write(index, _MEMORY_HEADER) + (self._global_dir / "topics").mkdir(exist_ok=True) + + if self._workspace_dir is not None: + self._workspace_dir.mkdir(parents=True, exist_ok=True) + ws_index = self._workspace_dir / "MEMORY.md" + if not ws_index.exists(): + _atomic_write(ws_index, _MEMORY_HEADER) + (self._workspace_dir / "topics").mkdir(exist_ok=True) + + async def shutdown(self) -> None: + pass + + def accepts(self, entry: MemoryEntry) -> bool: + return entry.kind in _NARRATIVE_KINDS + + async def insert(self, entry: MemoryEntry) -> str: + """Append entry as a bullet point to the appropriate MEMORY.md section.""" + section_name = _KIND_SECTION.get(entry.kind, "Notes") + scope = entry.metadata.get("scope", "global") + target_dir = ( + self._workspace_dir if scope == "workspace" and self._workspace_dir + else self._global_dir + ) + index_path = target_dir / "MEMORY.md" + + section_header = f"## {section_name}" + bullet = f"- {entry.content.strip()}" + + def _append_if_missing(content: str) -> str: + if section_header in content: + if bullet in content: + return content + parts = content.split(section_header, 1) + after = parts[1] + next_section = after.find("\n## ") + if next_section == -1: + return parts[0] + section_header + after.rstrip() + "\n" + bullet + "\n" + before_next = after[:next_section].rstrip() + rest = after[next_section:] + return parts[0] + section_header + before_next + "\n" + bullet + "\n" + rest + return content.rstrip() + "\n\n" + section_header + "\n" + bullet + "\n" + + _locked_update(index_path, _append_if_missing) + logger.debug("narrative: inserted %s into %s/%s", entry.entry_id, scope, section_name) + + # Long entries also get a topic file for detail + if len(entry.content) > 200: + slug = _slug(entry.content[:50]) + topic_path = target_dir / "topics" / f"{slug}.md" + if not topic_path.exists(): + topic_content = f"# {entry.content[:80]}\n\n{entry.content}\n" + _atomic_write(topic_path, topic_content) + + return entry.entry_id + + async def search(self, query: MemoryQuery) -> List[MemoryEntry]: + """Keyword search across MEMORY.md files and topic files.""" + if not query.keywords: + return [] + + results: List[MemoryEntry] = [] + seen_content: set = set() + + # Search index files first (higher priority) + for scope, directory in self._iter_scopes(): + index_path = directory / "MEMORY.md" + index_text = _read_text(index_path) + if index_text: + self._search_text( + index_text, query.keywords, results, seen_content, + source=f"{scope}/MEMORY.md", + ) + + # Search topic files (lazy load) + topics_dir = directory / "topics" + if topics_dir.exists(): + for topic_file in sorted(topics_dir.glob("*.md")): + topic_text = _read_text(topic_file) + if topic_text: + self._search_text( + topic_text, query.keywords, results, seen_content, + source=f"{scope}/topics/{topic_file.name}", + ) + + results.sort(key=lambda e: e.score, reverse=True) + return results[:query.limit] + + async def delete(self, entry_id: str) -> bool: + return False + + def context_block(self) -> str: + """Return text for system prompt injection. + + Loads the always-on index files (global + workspace MEMORY.md). + This is called at session start to provide persistent context. + """ + parts: List[str] = [] + + global_index = _read_text(self._global_dir / "MEMORY.md") + if global_index.strip() and global_index.strip() != _MEMORY_HEADER.strip(): + parts.append(global_index.strip()) + + if self._workspace_dir is not None: + ws_index = _read_text(self._workspace_dir / "MEMORY.md") + if ws_index.strip() and ws_index.strip() != _MEMORY_HEADER.strip(): + parts.append(f"## Project Memory\n{ws_index.strip()}") + + return "\n\n".join(parts) + + # ── Lifecycle hooks (no-op for narrative) ── + + def on_turn_start(self, turn: int, user_message: str) -> None: + pass + + def on_promoted(self, entry: MemoryEntry, source_provider: str) -> None: + pass + + def on_inserted(self, entry: MemoryEntry) -> None: + pass + + def on_accessed(self, entry: MemoryEntry) -> None: + pass + + # ── Tool interface ── + + def get_tool_schemas(self) -> List[MemoryToolSchema]: + return [ + MemoryToolSchema( + name="narrative_read", + description="Read persistent narrative memory (user preferences, facts, project notes).", + parameters={ + "type": "object", + "properties": { + "scope": { + "type": "string", + "enum": ["global", "workspace"], + "description": "Memory scope to read", + }, + }, + }, + provider_name="narrative", + ), + MemoryToolSchema( + name="narrative_write", + description="Write to persistent narrative memory (user preferences, facts, lessons learned).", + parameters={ + "type": "object", + "properties": { + "content": {"type": "string", "description": "What to remember"}, + "kind": { + "type": "string", + "enum": ["user_pref", "fact", "session_summary"], + }, + "scope": { + "type": "string", + "enum": ["global", "workspace"], + "default": "global", + }, + }, + "required": ["content"], + }, + provider_name="narrative", + ), + ] + + def handle_tool_call(self, tool_name: str, args: Dict[str, Any]) -> str: + import asyncio + import json + + if tool_name == "narrative_read": + scope = args.get("scope", "global") + directory = ( + self._workspace_dir if scope == "workspace" and self._workspace_dir + else self._global_dir + ) + text = _read_text(directory / "MEMORY.md") + return json.dumps({"content": text or "(empty)"}, ensure_ascii=False) + + if tool_name == "narrative_write": + content = args.get("content", "") + kind_str = args.get("kind", "fact") + scope = args.get("scope", "global") + try: + kind = MemoryKind(kind_str) + except ValueError: + kind = MemoryKind.FACT + + entry = MemoryEntry( + kind=kind, + content=content[:2000], + metadata={"scope": scope}, + ) + try: + loop = asyncio.get_running_loop() + loop.create_task(self.insert(entry)) + except RuntimeError: + asyncio.run(self.insert(entry)) + return json.dumps({"success": True, "id": entry.entry_id}) + + return json.dumps({"error": f"Unknown tool: {tool_name}"}) + + # ── Internal ── + + def _iter_scopes(self): + """Yield (scope_name, directory) pairs for search iteration.""" + yield "global", self._global_dir + if self._workspace_dir is not None and self._workspace_dir.exists(): + yield "workspace", self._workspace_dir + + def _search_text( + self, + text: str, + keywords: List[str], + results: List[MemoryEntry], + seen: set, + *, + source: str, + ) -> None: + """Extract matching lines/paragraphs from text content.""" + text_lower = text.lower() + matched_keywords = [kw for kw in keywords if kw.lower() in text_lower] + if not matched_keywords: + return + + score = len(matched_keywords) / max(len(keywords), 1) + + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + # Remove markdown bullet prefix + clean = stripped.lstrip("- ").strip() + if not clean or clean in seen: + continue + + line_lower = clean.lower() + if any(kw.lower() in line_lower for kw in matched_keywords): + seen.add(clean) + results.append(MemoryEntry( + entry_id=uuid.uuid4().hex[:12], + kind=MemoryKind.FACT, + content=clean, + score=score, + metadata={"source": source}, + )) diff --git a/src/leapflow/memory/providers/semantic.py b/src/leapflow/memory/providers/semantic.py index 284d61f..42841eb 100644 --- a/src/leapflow/memory/providers/semantic.py +++ b/src/leapflow/memory/providers/semantic.py @@ -17,7 +17,7 @@ import uuid from pathlib import Path from dataclasses import dataclass -from typing import Any, Dict, Iterable, List, Optional, Sequence +from typing import Any, Dict, Iterable, List, Optional, Sequence, Union import duckdb @@ -27,6 +27,7 @@ MemoryQuery, SignalDomain, ) +from leapflow.storage.connection import ConnectionHolder logger = logging.getLogger(__name__) @@ -95,8 +96,9 @@ class SemanticMemoryProvider: Accepts all MemoryKinds — acts as the universal long-term store. """ - def __init__(self, db_path: Path) -> None: - self._db_path = db_path + def __init__(self, source: Union[ConnectionHolder, Path]) -> None: + self._owns_holder = isinstance(source, Path) + self._source = source self._con: Optional[duckdb.DuckDBPyConnection] = None # ── Protocol properties ─────────────────────────────────────────── @@ -109,15 +111,18 @@ def name(self) -> str: async def initialize(self, **kwargs: Any) -> None: """Open DB connection, ensure schema exists, migrate if needed.""" - self._db_path.parent.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(str(self._db_path)) + if self._owns_holder: + from leapflow.storage.duckdb_connect import connect as safe_connect + self._con = safe_connect(self._source) # type: ignore[arg-type] + else: + self._con = self._source.connection # type: ignore[union-attr] self._init_schema() async def shutdown(self) -> None: - """Close DB connection.""" - if self._con: + """Close DB connection only if we own it.""" + if self._owns_holder and self._con: self._con.close() - self._con = None + self._con = None def accepts(self, entry: MemoryEntry) -> bool: # Semantic tier accepts everything — it's the final store @@ -125,11 +130,13 @@ def accepts(self, entry: MemoryEntry) -> bool: async def insert(self, entry: MemoryEntry) -> str: """Persist an entry to DuckDB. Returns entry_id.""" + from leapflow.storage.write_buffer import execute_with_retry con = self._connection() now = time.time() meta_json = json.dumps(entry.metadata, ensure_ascii=False) path = entry.metadata.get("path") - con.execute( + execute_with_retry( + con, """ INSERT INTO leap_memory (id, kind, domain, content, path, metadata, created_at, accessed_at, access_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -229,19 +236,27 @@ async def search(self, query: MemoryQuery) -> List[MemoryEntry]: async def delete(self, entry_id: str) -> bool: """Remove an entry by ID.""" + from leapflow.storage.write_buffer import execute_with_retry con = self._connection() - result = con.execute( - "DELETE FROM leap_memory WHERE id = ? RETURNING id", [entry_id] - ).fetchall() - return len(result) > 0 + count = con.execute( + "SELECT COUNT(*) FROM leap_memory WHERE id = ?", [entry_id] + ).fetchone()[0] + if count > 0: + execute_with_retry( + con, "DELETE FROM leap_memory WHERE id = ?", [entry_id] + ) + return True + return False # ── Additional public methods ───────────────────────────────────── def touch(self, entry_id: str) -> None: """Increment access count and update accessed_at timestamp.""" + from leapflow.storage.write_buffer import execute_with_retry con = self._connection() now = time.time() - con.execute( + execute_with_retry( + con, "UPDATE leap_memory SET accessed_at = ?, access_count = access_count + 1 WHERE id = ?", [now, entry_id], ) @@ -254,21 +269,31 @@ def prune( protected_kinds: Sequence[str] = ("skill_episode", "prediction", "skill", "prediction_experience"), ) -> int: """Remove old, low-value rows. Protected kinds are exempt.""" + from leapflow.storage.write_buffer import execute_with_retry con = self._connection() cutoff = time.time() - max_age_days * 86400 placeholders = ",".join(["?"] * len(protected_kinds)) - result = con.execute( + count_row = con.execute( f""" - DELETE FROM leap_memory + SELECT COUNT(*) FROM leap_memory WHERE created_at < ? AND access_count <= 1 AND kind NOT IN ({placeholders}) - RETURNING id """, [cutoff] + list(protected_kinds), - ).fetchall() - deleted = len(result) - if deleted: + ).fetchone() + deleted = count_row[0] if count_row else 0 + if deleted > 0: + execute_with_retry( + con, + f""" + DELETE FROM leap_memory + WHERE created_at < ? + AND access_count <= 1 + AND kind NOT IN ({placeholders}) + """, + [cutoff] + list(protected_kinds), + ) logger.info("semantic.prune deleted=%d cutoff_days=%.0f", deleted, max_age_days) return deleted @@ -290,12 +315,14 @@ def insert_raw( memory_id: Optional[str] = None, ) -> str: """Insert a row using the legacy (kind, content) signature.""" + from leapflow.storage.write_buffer import execute_with_retry con = self._ensure_connection() mid = memory_id or str(uuid.uuid4()) now = time.time() meta_json = json.dumps(metadata or {}, ensure_ascii=False) domain = SignalDomain.FILESYSTEM.value if "file" in kind else SignalDomain.SYSTEM.value - con.execute( + execute_with_retry( + con, """ INSERT INTO leap_memory (id, kind, domain, content, path, metadata, created_at, accessed_at, access_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -318,13 +345,15 @@ def upsert_raw( Useful for singleton state blobs (e.g. Markov state) that must be overwritten on each session rather than duplicated. """ + from leapflow.storage.write_buffer import execute_with_retry con = self._ensure_connection() mid = memory_id or str(uuid.uuid4()) now = time.time() meta_json = json.dumps(metadata or {}, ensure_ascii=False) domain = SignalDomain.FILESYSTEM.value if "file" in kind else SignalDomain.SYSTEM.value - con.execute("DELETE FROM leap_memory WHERE id = ?", [mid]) - con.execute( + execute_with_retry(con, "DELETE FROM leap_memory WHERE id = ?", [mid]) + execute_with_retry( + con, """ INSERT INTO leap_memory (id, kind, domain, content, path, metadata, created_at, accessed_at, access_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -497,9 +526,11 @@ def count_by_kind(self, kind: str) -> int: def update_metadata(self, memory_id: str, metadata: Dict[str, Any]) -> None: """Update the metadata JSON for a given row.""" + from leapflow.storage.write_buffer import execute_with_retry con = self._ensure_connection() meta_json = json.dumps(metadata, ensure_ascii=False) - con.execute( + execute_with_retry( + con, "UPDATE leap_memory SET metadata = ? WHERE id = ?", [meta_json, memory_id], ) @@ -537,9 +568,11 @@ def on_promoted(self, entry: MemoryEntry, source_provider: str) -> None: access_count=entry.access_count, ) try: + from leapflow.storage.write_buffer import execute_with_retry con = self._ensure_connection() meta_json = json.dumps(entry.metadata, ensure_ascii=False) - con.execute( + execute_with_retry( + con, """ INSERT INTO leap_memory (id, kind, domain, content, path, metadata, created_at, accessed_at, access_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -614,8 +647,11 @@ def _connection(self) -> duckdb.DuckDBPyConnection: def _ensure_connection(self) -> duckdb.DuckDBPyConnection: """Ensure DB is connected, auto-initializing if needed (for legacy callers).""" if self._con is None: - self._db_path.parent.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(str(self._db_path)) + if self._owns_holder: + from leapflow.storage.duckdb_connect import connect as safe_connect + self._con = safe_connect(self._source) # type: ignore[arg-type] + else: + self._con = self._source.connection # type: ignore[union-attr] self._init_schema() return self._con diff --git a/src/leapflow/perception/config.py b/src/leapflow/perception/config.py index 87782c8..48da74a 100644 --- a/src/leapflow/perception/config.py +++ b/src/leapflow/perception/config.py @@ -53,7 +53,7 @@ class PerceptionConfig: """Top-level perception module configuration.""" enabled: bool = False - frame_cache_dir: Path = field(default_factory=lambda: Path("~/.leapflow/cache/frames")) + frame_cache_dir: Path = field(default_factory=lambda: Path("~/.leapflow/profiles/default/cache/frames")) privacy_sensitive_apps: FrozenSet[str] = field(default_factory=frozenset) # Sampling diff --git a/src/leapflow/scheduler/store.py b/src/leapflow/scheduler/store.py index 1578929..68948da 100644 --- a/src/leapflow/scheduler/store.py +++ b/src/leapflow/scheduler/store.py @@ -11,11 +11,11 @@ import logging import time from pathlib import Path -from typing import List, Optional - -import duckdb +from typing import List, Optional, Union from leapflow.scheduler.types import ArmedTask +from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder +from leapflow.storage.write_buffer import execute_with_retry logger = logging.getLogger(__name__) @@ -26,18 +26,23 @@ class TaskStore: Provides atomic CRUD and query operations. At-most-once guarantee: advance_next_due() atomically updates before execution to prevent double-fire on crash recovery. + + Accepts ``ConnectionHolder`` (shared) or legacy ``Path``. """ - def __init__(self, db_path: str | Path) -> None: + def __init__(self, source: Union[ConnectionHolder, Path, str]) -> None: """Initialize store, creating table if not exists.""" - self._path = Path(db_path) - self._path.parent.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(str(self._path)) + self._owns_holder = isinstance(source, (str, Path)) + if self._owns_holder: + source = LocalConnectionHolder(Path(source)) + self._holder = source + self._con = self._holder.connection self._ensure_table() def close(self) -> None: - """Close the DuckDB connection.""" - self._con.close() + """Close the DuckDB connection if owned by this store.""" + if self._owns_holder: + self._holder.close() # ------------------------------------------------------------------ # Schema @@ -73,7 +78,8 @@ def _ensure_table(self) -> None: def save(self, task: ArmedTask) -> None: """Insert or update a task.""" - self._con.execute( + execute_with_retry( + self._con, """ INSERT OR REPLACE INTO armed_tasks ( task_id, skill_name, trigger_type, trigger_config, @@ -119,8 +125,9 @@ def load_all(self) -> List[ArmedTask]: def delete(self, task_id: str) -> None: """Remove a task.""" - self._con.execute( - "DELETE FROM armed_tasks WHERE task_id = ?", [task_id] + execute_with_retry( + self._con, + "DELETE FROM armed_tasks WHERE task_id = ?", [task_id], ) # ------------------------------------------------------------------ @@ -145,14 +152,16 @@ def get_due_tasks(self, now: float) -> List[ArmedTask]: def advance_next_due(self, task_id: str, new_due_at: float) -> None: """Atomically advance next_due_at (at-most-once guarantee).""" - self._con.execute( + execute_with_retry( + self._con, "UPDATE armed_tasks SET next_due_at = ? WHERE task_id = ?", [new_due_at, task_id], ) def update_state(self, task_id: str, new_state: str) -> None: """Update task state.""" - self._con.execute( + execute_with_retry( + self._con, "UPDATE armed_tasks SET state = ? WHERE task_id = ?", [new_state, task_id], ) @@ -160,7 +169,8 @@ def update_state(self, task_id: str, new_state: str) -> None: def increment_run_count(self, task_id: str) -> None: """Increment run_count and update last_run_at.""" now = time.time() - self._con.execute( + execute_with_retry( + self._con, """ UPDATE armed_tasks SET run_count = run_count + 1, last_run_at = ? diff --git a/src/leapflow/security/__init__.py b/src/leapflow/security/__init__.py index 82cde57..42f711e 100644 --- a/src/leapflow/security/__init__.py +++ b/src/leapflow/security/__init__.py @@ -1 +1,17 @@ -"""Security module — redaction, threat scanning, and trust boundary enforcement.""" +"""Security module — redaction, threat scanning, approval, and trust boundary enforcement.""" + +from leapflow.security.approval import ( + ApprovalDecision, + ApprovalGate, + ApprovalRequest, + DenyAllGate, + SessionAwareGate, +) + +__all__ = [ + "ApprovalDecision", + "ApprovalGate", + "ApprovalRequest", + "DenyAllGate", + "SessionAwareGate", +] diff --git a/src/leapflow/security/approval.py b/src/leapflow/security/approval.py new file mode 100644 index 0000000..8f56e6d --- /dev/null +++ b/src/leapflow/security/approval.py @@ -0,0 +1,166 @@ +"""Unified approval framework for actions requiring human confirmation. + +Replaces fragmented per-tool gates with a single ``ApprovalGate`` Protocol +and a session-aware wrapper that remembers per-category decisions. + +Design principles: +- **Minimal interruption**: "always for this session" eliminates repeat prompts +- **Fail-closed**: non-interactive environments deny by default +- **Audit trail**: every decision is logged at INFO level +- **Protocol-based**: swappable implementations (TUI, Web UI, headless CI) + +Categories +~~~~~~~~~~ +``shell_dangerous`` + Shell commands matching dangerous regex patterns (sudo, rm -r, etc.) + +``file_write`` + File writes to non-trivial paths (> 500 chars, non-safe extensions) + +``gateway_send`` + Proactive outbound messaging to external platforms (per platform) + +``external_action`` + Reserved for future extensibility (webhooks, API calls, etc.) +""" +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, Protocol, Set, runtime_checkable + +logger = logging.getLogger(__name__) + + +class ApprovalDecision(Enum): + """Result of an approval request.""" + + ALLOW = "allow" + DENY = "deny" + ALLOW_SESSION = "allow_session" + + +@dataclass(frozen=True) +class ApprovalRequest: + """Structured request for human approval. + + ``category`` groups related actions; session memory uses this key. + ``detail`` is the human-readable description shown in the prompt. + ``risk_hint`` is advisory (0.0–1.0); the gate decides policy. + """ + + category: str + detail: str + risk_hint: float = 0.5 + metadata: Dict[str, Any] = field(default_factory=dict) + + +@runtime_checkable +class ApprovalGate(Protocol): + """Protocol for all human-in-the-loop approval decisions. + + Implementations must handle the prompt display and input collection. + Return ``ALLOW_SESSION`` to suppress future prompts for the same + ``category`` during this session. + """ + + async def request_approval( + self, request: ApprovalRequest, + ) -> ApprovalDecision: ... + + +class SessionAwareGate: + """Wraps a base ``ApprovalGate`` with per-category session memory. + + Once a category is approved with ``ALLOW_SESSION``, all subsequent + requests for that category are auto-approved without prompting. + Individual ``ALLOW`` decisions are not remembered. + + Also implements ``check(command) -> bool`` for backward compatibility + with ``CommandApprovalGate`` in ``shell_tools.py``. + """ + + def __init__(self, delegate: ApprovalGate) -> None: + self._delegate = delegate + self._approved_categories: Set[str] = set() + self._decision_log: list[Dict[str, Any]] = [] + + async def check(self, command: str) -> bool: + """``CommandApprovalGate`` compatibility for shell_tools.py.""" + decision = await self.request_approval(ApprovalRequest( + category="shell_dangerous", + detail=command, + risk_hint=0.7, + )) + return decision in (ApprovalDecision.ALLOW, ApprovalDecision.ALLOW_SESSION) + + async def request_approval( + self, request: ApprovalRequest, + ) -> ApprovalDecision: + if request.category in self._approved_categories: + self._log_decision(request, ApprovalDecision.ALLOW, auto=True) + return ApprovalDecision.ALLOW + + decision = await self._delegate.request_approval(request) + + if decision == ApprovalDecision.ALLOW_SESSION: + self._approved_categories.add(request.category) + self._log_decision(request, ApprovalDecision.ALLOW, session=True) + return ApprovalDecision.ALLOW + + self._log_decision(request, decision) + return decision + + def reset(self) -> None: + """Clear all session approvals (e.g. on session restart).""" + self._approved_categories.clear() + + @property + def approved_categories(self) -> frozenset[str]: + return frozenset(self._approved_categories) + + def _log_decision( + self, + request: ApprovalRequest, + decision: ApprovalDecision, + *, + auto: bool = False, + session: bool = False, + ) -> None: + entry = { + "ts": time.time(), + "category": request.category, + "decision": decision.value, + "detail": request.detail[:200], + } + if auto: + entry["reason"] = "session_approved" + elif session: + entry["reason"] = "user_allow_session" + self._decision_log.append(entry) + + level = logging.DEBUG if auto else logging.INFO + logger.log( + level, + "approval.%s category=%s detail=%s%s", + decision.value, + request.category, + request.detail[:80], + " (session-approved)" if auto else "", + ) + + +class DenyAllGate: + """Gate that denies all requests (non-interactive / CI environments).""" + + async def request_approval( + self, request: ApprovalRequest, + ) -> ApprovalDecision: + logger.info( + "approval.deny category=%s detail=%s (non-interactive)", + request.category, + request.detail[:80], + ) + return ApprovalDecision.DENY diff --git a/src/leapflow/security/redact.py b/src/leapflow/security/redact.py index fcce9e8..b7ec067 100644 --- a/src/leapflow/security/redact.py +++ b/src/leapflow/security/redact.py @@ -23,31 +23,46 @@ class _PatternSpec(NamedTuple): def _compile_patterns() -> List[_PatternSpec]: - """Build redaction patterns, ordered by frequency for early exit.""" + """Build redaction patterns, ordered by frequency for early exit. + + The ``gate`` substring is checked first (fast ``in`` test) — the regex + is only compiled/run when the gate matches. This keeps log formatting + near-zero cost for messages that contain no secrets. + """ specs: List[tuple[str, str, str]] = [ - # Known API key prefixes + # ── Known API key prefixes ──────────────────────────── ("sk-", r"\bsk-[A-Za-z0-9_\-]{8,}", "api_key"), ("ghp_", r"\bghp_[A-Za-z0-9]{36,}", "github_pat"), ("gho_", r"\bgho_[A-Za-z0-9]{36,}", "github_oauth"), ("xoxb-", r"\bxoxb-[A-Za-z0-9\-]{30,}", "slack_bot"), ("xoxp-", r"\bxoxp-[A-Za-z0-9\-]{30,}", "slack_user"), ("AKIA", r"\bAKIA[A-Z0-9]{16}", "aws_access_key"), - # JWT tokens + ("hf_", r"\bhf_[A-Za-z0-9]{10,}", "huggingface"), + # ── Platform tokens (gateway) ───────────────────────── + # Telegram bot token: 123456789:AABBccDDeeFF… + ("/bot", r"\d{8,}:[A-Za-z0-9_\-]{30,}", "telegram_bot_token"), + # Feishu app IDs (cli_a…) are non-secret but Feishu tenant + # access tokens (t-…) and app_tickets are secret. + ("t-", r"\bt-[A-Za-z0-9_\-]{20,}", "feishu_access_token"), + # Encrypted credential values in gateway.yaml + ("enc:fernet:", r"enc:fernet:[A-Za-z0-9_=\+/\-]{20,}", "fernet_cipher"), + ("enc:b64:", r"enc:b64:[A-Za-z0-9_=\+/]{8,}", "b64_cipher"), + # ── JWT tokens ──────────────────────────────────────── ("eyJ", r"\beyJ[A-Za-z0-9_\-]{20,}\.[A-Za-z0-9_\-]{20,}\.[A-Za-z0-9_\-]{20,}", "jwt"), - # Auth headers + # ── Auth headers ────────────────────────────────────── ("Bearer", r"(?i)Bearer\s+[A-Za-z0-9_\-\.]{20,}", "auth_header"), ("x-api-key", r"(?i)x-api-key[:\s]+[A-Za-z0-9_\-]{10,}", "api_header"), ("Authorization", r"(?i)Authorization[:\s]+\S{10,}", "auth_header"), - # ENV assignments (KEY=value) - ("API_KEY=", r"(?i)(?:API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)\s*=\s*\S{6,}", "env_assignment"), - # JSON/YAML key-value secrets - ('"apiKey"', r'(?i)"(?:api[_-]?key|secret|token|password|credential)"\s*:\s*"[^"]{6,}"', "json_secret"), - ("password:", r"(?i)password\s*:\s*\S{6,}", "yaml_secret"), - # Database URLs with credentials + # ── ENV / config assignments ────────────────────────── + ("=", r"(?i)(?:API_KEY|APP_SECRET|APP_KEY|BOT_TOKEN|SECRET|TOKEN|PASSWORD|CREDENTIAL)\s*=\s*\S{6,}", "env_assignment"), + # ── JSON/YAML key-value secrets ─────────────────────── + ('"', r'(?i)"(?:api[_-]?key|app[_-]?secret|app[_-]?key|bot[_-]?token|secret|token|password|credential|access[_-]?token|webhook[_-]?secret)"\s*:\s*"[^"]{6,}"', "json_secret"), + ("secret:", r"(?i)(?:app_secret|app_key|bot_token|password|secret)\s*:\s*\S{6,}", "yaml_secret"), + # ── Database URLs ───────────────────────────────────── ("://", r"(?i)(?:postgres|mysql|mongodb|redis)://\S+:\S+@\S+", "db_url"), - # Private keys + # ── Private keys ────────────────────────────────────── ("-----BEGIN", r"-----BEGIN\s+(?:RSA|DSA|EC|OPENSSH)?\s*PRIVATE\s+KEY-----", "private_key"), - # Bare URL tokens + # ── URL-embedded tokens ─────────────────────────────── ("@github.com", r"https?://[A-Za-z0-9_\-]+@github\.com\S*", "url_token"), ] return [ diff --git a/src/leapflow/storage/__init__.py b/src/leapflow/storage/__init__.py index 65f7605..b7bd854 100644 --- a/src/leapflow/storage/__init__.py +++ b/src/leapflow/storage/__init__.py @@ -1,15 +1,32 @@ -"""Persistence layer — trajectory, skill library, session, conversation, and document stores.""" +"""Persistence layer — trajectory, skill library, session, conversation, and document stores. -from leapflow.storage.skill_library import SkillLibraryStore -from leapflow.storage.trajectory_store import TrajectoryStore -from leapflow.storage.session_store import LearningSessionStore +Key infrastructure: +- ``ConnectionHolder`` — shared DuckDB connection protocol +- ``LocalConnectionHolder`` — in-process holder for single-instance mode +- ``connect()`` — lock-aware DuckDB connection factory +- ``DatabaseLockedError`` — clear error for multi-instance lock conflicts +""" + +from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder from leapflow.storage.conversation_store import DuckDBConversationStore +from leapflow.storage.duckdb_connect import DatabaseLockedError, connect, is_lock_error +from leapflow.storage.session_store import LearningSessionStore from leapflow.storage.skill_docs import SkillDocStore +from leapflow.storage.skill_library import SkillLibraryStore +from leapflow.storage.trajectory_store import TrajectoryStore +from leapflow.storage.write_buffer import WriteBuffer, execute_with_retry __all__ = [ + "ConnectionHolder", + "DatabaseLockedError", "DuckDBConversationStore", + "LocalConnectionHolder", + "LearningSessionStore", "SkillDocStore", "SkillLibraryStore", - "LearningSessionStore", "TrajectoryStore", + "WriteBuffer", + "connect", + "execute_with_retry", + "is_lock_error", ] diff --git a/src/leapflow/storage/connection.py b/src/leapflow/storage/connection.py new file mode 100644 index 0000000..139898e --- /dev/null +++ b/src/leapflow/storage/connection.py @@ -0,0 +1,109 @@ +"""ConnectionHolder protocol and implementation for shared DuckDB access. + +All stores receive a ``ConnectionHolder`` instead of a raw ``db_path``. +This enables: + +- **P0a-P0b**: Centralized lock-aware connection with retry +- **P1**: Single ``leap.duckdb`` shared by all stores (6→1 consolidation) +- **P4**: leapd daemon owns the connection; stores are thin wrappers + +The holder creates the connection lazily and shares it. Stores MUST NOT +call ``duckdb.connect()`` themselves. +""" +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from typing import Optional, Protocol, runtime_checkable + +import duckdb + +from leapflow.storage.duckdb_connect import ( + DatabaseLockedError, + connect as _lock_aware_connect, +) + +logger = logging.getLogger(__name__) + + +@runtime_checkable +class ConnectionHolder(Protocol): + """Protocol for obtaining a shared DuckDB connection.""" + + @property + def connection(self) -> duckdb.DuckDBPyConnection: + """Return the managed DuckDB connection.""" + ... + + @property + def db_path(self) -> Path: + """Path to the DuckDB file.""" + ... + + def close(self) -> None: + """Close the managed connection.""" + ... + + +class LocalConnectionHolder: + """In-process holder that lazily opens a single DuckDB connection. + + Thread-safety: DuckDB's embedded connection is single-writer. + Within one process, all stores share this holder and access is + serialized by DuckDB's internal lock. For multi-process, use the + leapd daemon (P4). + """ + + def __init__(self, db_path: Path, *, volatile_on_lock: bool = False) -> None: + self._db_path = db_path + self._conn: Optional[duckdb.DuckDBPyConnection] = None + self._volatile_on_lock = volatile_on_lock + self._volatile_dir: tempfile.TemporaryDirectory[str] | None = None + self._locked_error: DatabaseLockedError | None = None + + @property + def db_path(self) -> Path: + return self._db_path + + @property + def is_volatile(self) -> bool: + return self._volatile_dir is not None + + @property + def locked_error(self) -> DatabaseLockedError | None: + return self._locked_error + + @property + def connection(self) -> duckdb.DuckDBPyConnection: + if self._conn is None: + self._conn = self._connect() + logger.info("duckdb: opened %s", self._db_path.name) + return self._conn + + def _connect(self) -> duckdb.DuckDBPyConnection: + try: + return _lock_aware_connect(self._db_path) + except DatabaseLockedError as exc: + if not self._volatile_on_lock: + raise + self._locked_error = exc + self._volatile_dir = tempfile.TemporaryDirectory(prefix="leapflow-volatile-") + self._db_path = Path(self._volatile_dir.name) / "leap.duckdb" + logger.warning( + "duckdb: primary database locked; using volatile session database at %s", + self._db_path, + ) + return _lock_aware_connect(self._db_path) + + def close(self) -> None: + if self._conn is not None: + try: + self._conn.close() + logger.info("duckdb: closed %s", self._db_path.name) + except Exception: + pass + self._conn = None + if self._volatile_dir is not None: + self._volatile_dir.cleanup() + self._volatile_dir = None diff --git a/src/leapflow/storage/conversation_store.py b/src/leapflow/storage/conversation_store.py index d01dd0b..f21eca4 100644 --- a/src/leapflow/storage/conversation_store.py +++ b/src/leapflow/storage/conversation_store.py @@ -16,12 +16,11 @@ import json import logging -import random import time import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional, Protocol, runtime_checkable +from typing import Any, Dict, List, Optional, Protocol, Union, runtime_checkable logger = logging.getLogger(__name__) @@ -85,11 +84,6 @@ def search_messages(self, query: str, *, limit: int = 10) -> List[ConversationSe def close(self) -> None: ... -_WRITE_RETRIES = 15 -_WRITE_JITTER_MIN_MS = 20 -_WRITE_JITTER_MAX_MS = 150 - - class DuckDBConversationStore: """DuckDB-backed conversation store with FTS and write-retry. @@ -97,12 +91,18 @@ class DuckDBConversationStore: - conversation_sessions: metadata, lineage (parent_session_id), model config - conversation_messages: full transcript with soft-delete (active) and compaction flag - FTS index on messages for keyword search + + Accepts ``ConnectionHolder`` (shared) or legacy ``Path``. """ - def __init__(self, db_path: Path | str) -> None: - import duckdb - self._db_path = str(db_path) - self._conn = duckdb.connect(self._db_path) + def __init__(self, source: "Union[ConnectionHolder, Path, str]") -> None: + from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder + self._owns_holder = isinstance(source, (str, Path)) + if self._owns_holder: + source = LocalConnectionHolder(Path(source)) + self._holder = source + self._conn = self._holder.connection + self._db_path = str(self._holder.db_path) self._write_count = 0 self._initialize_schema() @@ -175,20 +175,9 @@ def _ensure_fts(self) -> None: def _execute_write(self, sql: str, params: Any = None) -> None: """Execute a write with jitter retry for concurrent access.""" - for attempt in range(_WRITE_RETRIES): - try: - if params: - self._conn.execute(sql, params) - else: - self._conn.execute(sql) - self._write_count += 1 - return - except Exception as e: - if "locked" in str(e).lower() and attempt < _WRITE_RETRIES - 1: - jitter = random.uniform(_WRITE_JITTER_MIN_MS, _WRITE_JITTER_MAX_MS) / 1000 - time.sleep(jitter) - continue - raise + from leapflow.storage.write_buffer import execute_with_retry + execute_with_retry(self._conn, sql, params) + self._write_count += 1 def create_session( self, @@ -544,11 +533,12 @@ def import_session(self, data: Dict[str, Any]) -> ConversationSession: return session def close(self) -> None: - """Close the database connection.""" - try: - self._conn.close() - except Exception: - pass + """Close the database connection if owned by this store.""" + if self._owns_holder: + try: + self._holder.close() + except Exception: + pass def _row_to_session(self, row: tuple) -> ConversationSession: meta = {} diff --git a/src/leapflow/storage/db_repair.py b/src/leapflow/storage/db_repair.py index 2274259..5fdc41a 100644 --- a/src/leapflow/storage/db_repair.py +++ b/src/leapflow/storage/db_repair.py @@ -44,6 +44,10 @@ def check_and_repair( conn.close() return True except Exception as exc: + from leapflow.storage.duckdb_connect import is_lock_error + if is_lock_error(exc): + logger.info("db_repair: %s is locked (not corrupt), skipping repair", db_path.name) + return True logger.warning("db_repair: integrity check failed for %s: %s", db_path.name, exc) logger.warning("db_repair: database corrupt, backing up: %s", db_path.name) @@ -103,12 +107,9 @@ def _cleanup_old_backups( def safe_connect(db_path: Path | str, *, read_only: bool = False): """Connect to DuckDB with pre-flight health check and auto-repair. - Returns a duckdb.Connection. On corruption, backs up and recreates. + .. deprecated:: + Use ``leapflow.storage.duckdb_connect.connect()`` instead, + which provides lock-aware retry in addition to corruption repair. """ - import duckdb - - db_path = Path(db_path) - if db_path.exists(): - check_and_repair(db_path) - - return duckdb.connect(str(db_path), read_only=read_only) + from leapflow.storage.duckdb_connect import connect as lock_aware_connect + return lock_aware_connect(db_path, read_only=read_only) diff --git a/src/leapflow/storage/duckdb_connect.py b/src/leapflow/storage/duckdb_connect.py new file mode 100644 index 0000000..2127c97 --- /dev/null +++ b/src/leapflow/storage/duckdb_connect.py @@ -0,0 +1,133 @@ +"""Centralized DuckDB connection factory with lock detection and repair. + +Replaces bare ``duckdb.connect()`` calls throughout the codebase with a +single entry point that: + +1. Detects lock conflicts (another process holds exclusive lock) and raises + a clear, actionable ``DatabaseLockedError`` instead of a cryptic traceback. +2. Performs corruption detection via read-only probe before opening read-write. +3. Auto-repairs corrupt databases (backup + recreate). + +All stores must use ``connect()`` from this module rather than calling +``duckdb.connect()`` directly. +""" +from __future__ import annotations + +import logging +import os +import time +from pathlib import Path +from typing import Optional + +import duckdb + +logger = logging.getLogger(__name__) + +LOCK_KEYWORDS = frozenset({"lock", "locked", "exclusive", "cannot set lock"}) + +_CONNECT_RETRIES = int(os.getenv("LEAPFLOW_DB_CONNECT_RETRIES", "3")) +_CONNECT_BACKOFF_S = float(os.getenv("LEAPFLOW_DB_CONNECT_BACKOFF_S", "0.5")) + + +class DatabaseLockedError(RuntimeError): + """Raised when DuckDB cannot acquire exclusive lock on the database file. + + Provides a clear, actionable error message for the user. + """ + + def __init__(self, db_path: Path, original: Exception) -> None: + self.db_path = db_path + self.original = original + msg = ( + f"Database is locked: {db_path.name}\n" + f"Another LeapFlow instance holds the database.\n" + f"Run 'leap daemon status' to check, or close the other instance." + ) + super().__init__(msg) + + +def is_lock_error(exc: Exception) -> bool: + """Heuristic check whether exception is a DuckDB lock conflict.""" + msg = str(exc).lower() + return any(kw in msg for kw in LOCK_KEYWORDS) + + +def _probe_corruption(db_path: Path) -> bool: + """Read-only probe to detect corruption. Returns True if healthy.""" + if not db_path.exists(): + return True + try: + conn = duckdb.connect(str(db_path), read_only=True) + conn.execute("SELECT 1").fetchone() + conn.close() + return True + except Exception as exc: + if is_lock_error(exc): + return True + logger.warning("duckdb_connect: corruption detected in %s: %s", db_path.name, exc) + return False + + +def _repair(db_path: Path) -> None: + """Backup corrupt database and delete for recreation on next connect.""" + from leapflow.storage.db_repair import check_and_repair + check_and_repair(db_path) + + +def connect( + db_path: Path | str, + *, + read_only: bool = False, + retries: Optional[int] = None, +) -> duckdb.DuckDBPyConnection: + """Open a DuckDB connection with lock detection, retry, and repair. + + Parameters + ---------- + db_path : Path or str + Path to the ``.duckdb`` file. + read_only : bool + Open in read-only mode (no lock contention). + retries : int, optional + Maximum retry attempts. Defaults to env + ``LEAPFLOW_DB_CONNECT_RETRIES`` (3). + + Returns + ------- + duckdb.DuckDBPyConnection + + Raises + ------ + DatabaseLockedError + If the database is locked by another process after all retries. + RuntimeError + If the database cannot be opened for reasons other than lock. + """ + db_path = Path(db_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + max_attempts = retries if retries is not None else _CONNECT_RETRIES + + if not read_only and not _probe_corruption(db_path): + _repair(db_path) + + last_exc: Optional[Exception] = None + for attempt in range(max_attempts): + try: + conn = duckdb.connect(str(db_path), read_only=read_only) + return conn + except Exception as exc: + last_exc = exc + if is_lock_error(exc): + if attempt < max_attempts - 1: + backoff = _CONNECT_BACKOFF_S * (attempt + 1) + logger.info( + "duckdb_connect: %s locked, retry %d/%d in %.1fs", + db_path.name, attempt + 1, max_attempts, backoff, + ) + time.sleep(backoff) + continue + raise DatabaseLockedError(db_path, exc) from exc + raise + + assert last_exc is not None + raise last_exc diff --git a/src/leapflow/storage/evolution_store.py b/src/leapflow/storage/evolution_store.py index dec9ad0..47a90ed 100644 --- a/src/leapflow/storage/evolution_store.py +++ b/src/leapflow/storage/evolution_store.py @@ -14,25 +14,29 @@ import json import logging -import random import time -import uuid from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union -logger = logging.getLogger(__name__) +from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder +from leapflow.storage.write_buffer import execute_with_retry -_WRITE_RETRIES = 10 -_WRITE_JITTER_MS = (15, 120) +logger = logging.getLogger(__name__) class DuckDBEvolutionStore: - """Persistent backing store for skill episodes.""" - - def __init__(self, db_path: Path | str) -> None: - import duckdb - self._db_path = str(db_path) - self._conn = duckdb.connect(self._db_path) + """Persistent backing store for skill episodes. + + Accepts ``ConnectionHolder`` (shared) or legacy ``Path``. + """ + + def __init__(self, source: Union[ConnectionHolder, Path, str]) -> None: + self._owns_holder = isinstance(source, (str, Path)) + if self._owns_holder: + source = LocalConnectionHolder(Path(source)) + self._holder = source + self._conn = self._holder.connection + self._db_path = str(self._holder.db_path) self._initialize_schema() def _initialize_schema(self) -> None: @@ -63,19 +67,7 @@ def _initialize_schema(self) -> None: """) def _execute_write(self, sql: str, params: Any = None) -> None: - for attempt in range(_WRITE_RETRIES): - try: - if params: - self._conn.execute(sql, params) - else: - self._conn.execute(sql) - return - except Exception as e: - if "locked" in str(e).lower() and attempt < _WRITE_RETRIES - 1: - jitter = random.uniform(*_WRITE_JITTER_MS) / 1000 - time.sleep(jitter) - continue - raise + execute_with_retry(self._conn, sql, params) def save_episode( self, @@ -195,7 +187,8 @@ def integrity_check(self) -> bool: return False def close(self) -> None: - try: - self._conn.close() - except Exception: - pass + if self._owns_holder: + try: + self._holder.close() + except Exception: + pass diff --git a/src/leapflow/storage/migrate.py b/src/leapflow/storage/migrate.py new file mode 100644 index 0000000..f1cdb2e --- /dev/null +++ b/src/leapflow/storage/migrate.py @@ -0,0 +1,269 @@ +"""Migration: consolidate 6 legacy DuckDB files into single leap.duckdb. + +Idempotent — safe to run multiple times. Skips files that don't exist +or have already been migrated. Renames old files to ``.migrated.bak`` +after successful migration. + +Usage:: + + from leapflow.storage.migrate import migrate_to_unified + migrate_to_unified(data_dir=Path("~/.leapflow").expanduser()) +""" +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Dict, List, Tuple + +import duckdb + +logger = logging.getLogger(__name__) + +_TABLE_MAP: List[Tuple[str, str, str, List[str]]] = [ + # (old_db_filename, old_table, new_table, columns_to_copy) + # columns_to_copy is empty = SELECT * (we handle column mapping in SQL) + ("memory.duckdb", "leap_memory", "mem_entries", []), + ("trajectories.duckdb", "leap_trajectory", "traj_headers", []), + ("trajectories.duckdb", "leap_trajectory_step", "traj_steps", []), + ("trajectories.duckdb", "leap_episode", "traj_episodes", []), + ("trajectories.duckdb", "leap_learning_session", "learn_sessions", []), + ("skill_library.duckdb", "leap_skill_library", "skill_library", []), + ("skill_library.duckdb", "leap_parameterized_skills", "skill_parameterized", []), + ("skill_library.duckdb", "leap_skill_suggestion", "skill_suggestions", []), + ("skill_library.duckdb", "leap_skill_execution", "skill_executions", []), + ("conversations.duckdb", "conversation_sessions", "conv_sessions", []), + ("conversations.duckdb", "conversation_messages", "conv_messages", []), + ("evolution.duckdb", "skill_episodes", "evo_episodes", []), + ("evolution.duckdb", "skill_patterns", "evo_patterns", []), + ("scheduler.duckdb", "armed_tasks", "sched_tasks", []), +] + +_COLUMN_MAP: Dict[Tuple[str, str], str] = { + ("leap_memory", "mem_entries"): """ + SELECT id, kind, domain, content, path, metadata, + created_at, accessed_at, access_count, + '' AS workspace_id, '' AS session_id + FROM old_db.{old_table} + """, + ("leap_trajectory", "traj_headers"): """ + SELECT id, user_id, start_time, end_time, step_count, metadata, + created_at, + '' AS workspace_id, '' AS session_id + FROM old_db.{old_table} + """, + ("leap_learning_session", "learn_sessions"): """ + SELECT session_id, trajectory_id, goal, start_time, end_time, + status, annotations, metadata, created_at, + '' AS workspace_id, '' AS session_ref + FROM old_db.{old_table} + """, + ("leap_skill_library", "skill_library"): """ + SELECT id, title, trigger_phrases, steps, parameters, + pre_conditions, post_conditions, app_sequence, action_names, + source_trajectory_id, source_episode_id, confidence, + version, status, created_at, updated_at, + '' AS workspace_id, '' AS session_id + FROM old_db.{old_table} + """, + ("conversation_sessions", "conv_sessions"): """ + SELECT session_id, title, created_at, updated_at, + parent_session_id, model, source, cwd, + message_count, total_tokens, is_active, metadata_json, + '' AS workspace_id + FROM old_db.{old_table} + """, + ("skill_episodes", "evo_episodes"): """ + SELECT episode_id, skill_name, actions_json, outcome, reward, + context_json, created_at, + '' AS workspace_id, '' AS session_id + FROM old_db.{old_table} + """, +} + + +def migrate_to_unified( + data_dir: Path, + *, + profile: str = "default", + target_db_path: Path | None = None, +) -> Dict[str, int]: + """Migrate all legacy DuckDB files into the unified ``leap.duckdb``. + + Parameters + ---------- + data_dir : Path + The LeapFlow data directory (e.g. ``~/.leapflow``). + profile : str + Target profile name (default ``"default"``). + target_db_path : Path, optional + Explicit path for the target database. Defaults to + ``data_dir / "profiles" / profile / "db" / "leap.duckdb"``. + + Returns + ------- + dict + Mapping of ``"old_file:old_table"`` to row count migrated. + """ + target = target_db_path or (data_dir / "profiles" / profile / "db" / "leap.duckdb") + target.parent.mkdir(parents=True, exist_ok=True) + + from leapflow.storage.duckdb_connect import connect as safe_connect + from leapflow.storage.schema import ensure_schema + + conn = safe_connect(target) + ensure_schema(conn) + + results: Dict[str, int] = {} + attached_dbs: set = set() + + for old_filename, old_table, new_table, _cols in _TABLE_MAP: + old_path = data_dir / old_filename + if not old_path.exists(): + logger.debug("migrate: skipping %s (not found)", old_filename) + continue + + key = f"{old_filename}:{old_table}" + + existing = conn.execute( + f"SELECT COUNT(*) FROM {new_table}" + ).fetchone() + if existing and existing[0] > 0: + logger.info("migrate: %s already has %d rows, skipping %s", + new_table, existing[0], key) + results[key] = 0 + continue + + alias = old_filename.replace(".", "_").replace("-", "_") + if alias not in attached_dbs: + try: + conn.execute(f"ATTACH '{old_path}' AS old_db (READ_ONLY)") + attached_dbs.add(alias) + except Exception as exc: + logger.warning("migrate: cannot attach %s: %s", old_filename, exc) + continue + + try: + tables_in_old = { + r[0] for r in conn.execute("SHOW TABLES FROM old_db").fetchall() + } + if old_table not in tables_in_old: + logger.debug("migrate: table %s not in %s", old_table, old_filename) + conn.execute("DETACH old_db") + attached_dbs.discard(alias) + continue + + custom_select = _COLUMN_MAP.get((old_table, new_table)) + if custom_select: + select_sql = custom_select.format(old_table=old_table) + else: + select_sql = f"SELECT * FROM old_db.{old_table}" + + conn.execute(f"INSERT INTO {new_table} {select_sql}") + + count_row = conn.execute( + f"SELECT COUNT(*) FROM {new_table}" + ).fetchone() + count = count_row[0] if count_row else 0 + results[key] = count + logger.info("migrate: %s → %s: %d rows", key, new_table, count) + + except Exception as exc: + logger.warning("migrate: failed %s → %s: %s", key, new_table, exc) + results[key] = -1 + finally: + try: + conn.execute("DETACH old_db") + attached_dbs.discard(alias) + except Exception: + pass + + conn.close() + + _rename_old_files(data_dir, results) + return results + + +def _rename_old_files(data_dir: Path, results: Dict[str, int]) -> None: + """Rename successfully migrated old .duckdb files to .migrated.bak.""" + migrated_files: set = set() + for key, count in results.items(): + if count > 0: + filename = key.split(":")[0] + migrated_files.add(filename) + + for filename in migrated_files: + old_path = data_dir / filename + if not old_path.exists(): + continue + + all_tables_done = all( + results.get(f"{filename}:{entry[1]}", 0) >= 0 + for entry in _TABLE_MAP + if entry[0] == filename + ) + if not all_tables_done: + continue + + bak_path = old_path.with_suffix(f".duckdb.migrated.bak") + try: + old_path.rename(bak_path) + wal = old_path.with_suffix(old_path.suffix + ".wal") + if wal.exists(): + wal.rename(bak_path.with_suffix(bak_path.suffix + ".wal")) + logger.info("migrate: renamed %s → %s", old_path.name, bak_path.name) + except OSError as exc: + logger.warning("migrate: rename failed for %s: %s", old_path.name, exc) + + +def migrate_directory_layout(data_dir: Path, *, profile: str = "default") -> int: + """Migrate flat ``~/.leapflow/`` to profile-based directory structure. + + Moves ``skills/``, ``cache/``, ``audit.jsonl`` into + ``profiles//``. Idempotent — skips files that don't exist + or are already migrated. + + Returns count of items moved. + """ + import shutil + + profile_dir = data_dir / "profiles" / profile + moved = 0 + + _MOVES: List[Tuple[str, str]] = [ + ("skills", "skills"), + ("cache/frames", "cache/frames"), + ("cache/video", "cache/video"), + ("audit.jsonl", "audit.jsonl"), + ] + + for src_rel, dst_rel in _MOVES: + src = data_dir / src_rel + dst = profile_dir / dst_rel + if not src.exists(): + continue + if dst.exists() and (dst.is_dir() and any(dst.iterdir())): + logger.debug("migrate_layout: %s already populated, skipping", dst_rel) + continue + + dst.parent.mkdir(parents=True, exist_ok=True) + try: + if src.is_dir(): + if dst.exists(): + for item in src.iterdir(): + target = dst / item.name + if not target.exists(): + shutil.move(str(item), str(target)) + moved += 1 + else: + shutil.move(str(src), str(dst)) + moved += 1 + else: + if not dst.exists(): + shutil.move(str(src), str(dst)) + moved += 1 + logger.info("migrate_layout: %s → %s", src_rel, dst_rel) + except OSError as exc: + logger.warning("migrate_layout: failed %s: %s", src_rel, exc) + + return moved diff --git a/src/leapflow/storage/schema.py b/src/leapflow/storage/schema.py new file mode 100644 index 0000000..ffb877c --- /dev/null +++ b/src/leapflow/storage/schema.py @@ -0,0 +1,387 @@ +"""Unified DuckDB schema definition and migration for leap.duckdb. + +Single source of truth for all table schemas. Each store registers its +schema here rather than running ad-hoc CREATE TABLE in its own __init__. + +Migration is version-tracked via a ``_schema_version`` table. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import List + +import duckdb + +logger = logging.getLogger(__name__) + +CURRENT_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class TableDef: + """Declarative table definition.""" + name: str + ddl: str + indexes: List[str] = field(default_factory=list) + + +TABLES: List[TableDef] = [ + # ── Memory ── + TableDef( + name="mem_entries", + ddl=""" + CREATE TABLE IF NOT EXISTS mem_entries ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + domain TEXT NOT NULL DEFAULT 'system', + content TEXT NOT NULL, + path TEXT, + metadata TEXT, + created_at DOUBLE NOT NULL, + accessed_at DOUBLE NOT NULL, + access_count INTEGER NOT NULL DEFAULT 1, + workspace_id TEXT DEFAULT '', + session_id TEXT DEFAULT '' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_mem_created ON mem_entries(created_at)", + "CREATE INDEX IF NOT EXISTS idx_mem_kind ON mem_entries(kind)", + "CREATE INDEX IF NOT EXISTS idx_mem_domain ON mem_entries(domain)", + ], + ), + # ── Trajectory ── + TableDef( + name="traj_headers", + ddl=""" + CREATE TABLE IF NOT EXISTS traj_headers ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + start_time DOUBLE NOT NULL, + end_time DOUBLE NOT NULL, + step_count INTEGER NOT NULL, + metadata TEXT, + created_at DOUBLE NOT NULL, + workspace_id TEXT DEFAULT '', + session_id TEXT DEFAULT '' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_traj_time ON traj_headers(start_time)", + "CREATE INDEX IF NOT EXISTS idx_traj_user ON traj_headers(user_id)", + ], + ), + TableDef( + name="traj_steps", + ddl=""" + CREATE TABLE IF NOT EXISTS traj_steps ( + trajectory_id TEXT NOT NULL, + step_idx INTEGER NOT NULL, + timestamp DOUBLE NOT NULL, + action_type TEXT NOT NULL, + target TEXT, + target_label TEXT, + target_role TEXT, + app_bundle_id TEXT, + app_name TEXT, + params TEXT, + state_focused_app TEXT, + state_ax_digest TEXT, + state_clipboard TEXT, + visual_frame_ref TEXT, + state_ax_tree TEXT, + state_snapshot_level TEXT, + PRIMARY KEY (trajectory_id, step_idx) + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_tstep_traj ON traj_steps(trajectory_id)", + "CREATE INDEX IF NOT EXISTS idx_tstep_action ON traj_steps(action_type)", + ], + ), + TableDef( + name="traj_episodes", + ddl=""" + CREATE TABLE IF NOT EXISTS traj_episodes ( + id TEXT PRIMARY KEY, + trajectory_id TEXT NOT NULL, + start_idx INTEGER NOT NULL, + end_idx INTEGER NOT NULL, + inferred_goal TEXT, + app_sequence TEXT, + semantic_actions TEXT, + confidence DOUBLE, + created_at DOUBLE NOT NULL, + procedure_graph TEXT + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_ep_traj ON traj_episodes(trajectory_id)", + "CREATE INDEX IF NOT EXISTS idx_ep_goal ON traj_episodes(inferred_goal)", + ], + ), + # ── Learning ── + TableDef( + name="learn_sessions", + ddl=""" + CREATE TABLE IF NOT EXISTS learn_sessions ( + session_id TEXT PRIMARY KEY, + trajectory_id TEXT NOT NULL, + goal TEXT NOT NULL DEFAULT '', + start_time DOUBLE NOT NULL, + end_time DOUBLE, + status TEXT NOT NULL DEFAULT 'recording', + annotations TEXT, + metadata TEXT, + created_at DOUBLE NOT NULL, + workspace_id TEXT DEFAULT '', + session_ref TEXT DEFAULT '' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_lsess_traj ON learn_sessions(trajectory_id)", + "CREATE INDEX IF NOT EXISTS idx_lsess_status ON learn_sessions(status)", + ], + ), + # ── Skill Library ── + TableDef( + name="skill_library", + ddl=""" + CREATE TABLE IF NOT EXISTS skill_library ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + trigger_phrases TEXT, + steps TEXT, + parameters TEXT, + pre_conditions TEXT, + post_conditions TEXT, + app_sequence TEXT, + action_names TEXT, + source_trajectory_id TEXT, + source_episode_id TEXT, + confidence DOUBLE, + version INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'active', + created_at DOUBLE NOT NULL, + updated_at DOUBLE NOT NULL, + workspace_id TEXT DEFAULT '', + session_id TEXT DEFAULT '' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_skill_status ON skill_library(status)", + ], + ), + TableDef( + name="skill_parameterized", + ddl=""" + CREATE TABLE IF NOT EXISTS skill_parameterized ( + name TEXT PRIMARY KEY, + description TEXT NOT NULL, + parameters TEXT, + preconditions TEXT, + postconditions TEXT, + triggers TEXT, + source TEXT DEFAULT 'builtin', + source_trajectory_id TEXT, + source_episode_id TEXT, + confidence DOUBLE DEFAULT 1.0, + version INTEGER DEFAULT 1, + code TEXT, + created_at DOUBLE, + updated_at DOUBLE, + is_active BOOLEAN DEFAULT TRUE + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_param_active ON skill_parameterized(is_active)", + ], + ), + TableDef( + name="skill_suggestions", + ddl=""" + CREATE TABLE IF NOT EXISTS skill_suggestions ( + id TEXT PRIMARY KEY, + existing_skill_id TEXT NOT NULL, + existing_skill_title TEXT, + new_candidate_json TEXT NOT NULL, + similarity_score DOUBLE NOT NULL, + similarity_details TEXT, + suggestion_type TEXT NOT NULL, + proposed_changes TEXT, + status TEXT NOT NULL DEFAULT 'pending', + source_trajectory_id TEXT, + source_episode_id TEXT, + created_at DOUBLE NOT NULL, + resolved_at DOUBLE + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_sug_status ON skill_suggestions(status)", + ], + ), + TableDef( + name="skill_executions", + ddl=""" + CREATE TABLE IF NOT EXISTS skill_executions ( + id TEXT PRIMARY KEY, + skill_id TEXT NOT NULL, + trajectory_id TEXT NOT NULL, + episode_id TEXT NOT NULL, + similarity_score DOUBLE NOT NULL, + diff_hash TEXT NOT NULL, + diff_summary TEXT, + verdict TEXT NOT NULL, + created_at DOUBLE NOT NULL + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_sexec_skill ON skill_executions(skill_id)", + "CREATE INDEX IF NOT EXISTS idx_sexec_hash ON skill_executions(diff_hash)", + ], + ), + # ── Conversation ── + TableDef( + name="conv_sessions", + ddl=""" + CREATE TABLE IF NOT EXISTS conv_sessions ( + session_id VARCHAR PRIMARY KEY, + title VARCHAR DEFAULT '', + created_at DOUBLE DEFAULT 0.0, + updated_at DOUBLE DEFAULT 0.0, + parent_session_id VARCHAR, + model VARCHAR DEFAULT '', + source VARCHAR DEFAULT 'cli', + cwd VARCHAR DEFAULT '', + message_count INTEGER DEFAULT 0, + total_tokens INTEGER DEFAULT 0, + is_active BOOLEAN DEFAULT TRUE, + metadata_json VARCHAR DEFAULT '{}', + workspace_id TEXT DEFAULT '' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_csess_updated ON conv_sessions(updated_at DESC)", + ], + ), + TableDef( + name="conv_messages", + ddl=""" + CREATE TABLE IF NOT EXISTS conv_messages ( + message_id VARCHAR PRIMARY KEY, + session_id VARCHAR NOT NULL, + role VARCHAR NOT NULL, + content VARCHAR DEFAULT '', + created_at DOUBLE DEFAULT 0.0, + tool_name VARCHAR, + tool_call_id VARCHAR, + tool_calls_json VARCHAR, + active BOOLEAN DEFAULT TRUE, + compacted BOOLEAN DEFAULT FALSE, + token_count INTEGER DEFAULT 0, + metadata_json VARCHAR DEFAULT '{}' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_cmsg_session ON conv_messages(session_id, created_at)", + ], + ), + # ── Evolution ── + TableDef( + name="evo_episodes", + ddl=""" + CREATE TABLE IF NOT EXISTS evo_episodes ( + episode_id VARCHAR PRIMARY KEY, + skill_name VARCHAR NOT NULL, + actions_json VARCHAR DEFAULT '[]', + outcome VARCHAR DEFAULT '', + reward DOUBLE DEFAULT 0.0, + context_json VARCHAR DEFAULT '{}', + created_at DOUBLE DEFAULT 0.0, + workspace_id TEXT DEFAULT '', + session_id TEXT DEFAULT '' + ) + """, + indexes=[ + "CREATE INDEX IF NOT EXISTS idx_evep_skill ON evo_episodes(skill_name, created_at DESC)", + ], + ), + TableDef( + name="evo_patterns", + ddl=""" + CREATE TABLE IF NOT EXISTS evo_patterns ( + pattern_id VARCHAR PRIMARY KEY, + skill_name VARCHAR NOT NULL, + pattern_json VARCHAR DEFAULT '{}', + confidence DOUBLE DEFAULT 0.0, + episode_count INTEGER DEFAULT 0, + created_at DOUBLE DEFAULT 0.0 + ) + """, + indexes=[], + ), + # ── Scheduler ── + TableDef( + name="sched_tasks", + ddl=""" + CREATE TABLE IF NOT EXISTS sched_tasks ( + task_id TEXT PRIMARY KEY, + skill_name TEXT NOT NULL, + trigger_type TEXT NOT NULL, + trigger_config TEXT NOT NULL, + state TEXT DEFAULT 'armed', + execution_tier TEXT DEFAULT 'auto', + context_snapshot TEXT DEFAULT '{}', + confidence DOUBLE DEFAULT 0.0, + created_at DOUBLE NOT NULL, + next_due_at DOUBLE DEFAULT 0.0, + last_run_at DOUBLE DEFAULT 0.0, + run_count INTEGER DEFAULT 0, + max_runs INTEGER DEFAULT -1, + grace_seconds DOUBLE DEFAULT 120.0, + parameters TEXT DEFAULT '{}', + cloud_worker_id TEXT DEFAULT '', + metadata TEXT DEFAULT '{}' + ) + """, + indexes=[], + ), + # ── Schema version tracking ── + TableDef( + name="_schema_version", + ddl=""" + CREATE TABLE IF NOT EXISTS _schema_version ( + version INTEGER NOT NULL, + applied_at DOUBLE NOT NULL + ) + """, + indexes=[], + ), +] + + +def ensure_schema(conn: duckdb.DuckDBPyConnection) -> int: + """Create all tables and indexes if they don't exist. + + Returns the current schema version after applying. + """ + for table_def in TABLES: + conn.execute(table_def.ddl) + for idx_sql in table_def.indexes: + conn.execute(idx_sql) + + row = conn.execute( + "SELECT MAX(version) FROM _schema_version" + ).fetchone() + current = row[0] if row and row[0] is not None else 0 + + if current < CURRENT_SCHEMA_VERSION: + import time + conn.execute( + "INSERT INTO _schema_version VALUES (?, ?)", + [CURRENT_SCHEMA_VERSION, time.time()], + ) + logger.info("schema: applied version %d", CURRENT_SCHEMA_VERSION) + + return CURRENT_SCHEMA_VERSION diff --git a/src/leapflow/storage/session_store.py b/src/leapflow/storage/session_store.py index d56a758..781232c 100644 --- a/src/leapflow/storage/session_store.py +++ b/src/leapflow/storage/session_store.py @@ -10,24 +10,31 @@ import logging import time from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union -import duckdb +from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder +from leapflow.storage.write_buffer import execute_with_retry logger = logging.getLogger(__name__) class LearningSessionStore: - """DuckDB-backed CRUD for learning session metadata.""" + """DuckDB-backed CRUD for learning session metadata. - def __init__(self, db_path: Path) -> None: - self._path = db_path - self._path.parent.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(str(self._path)) + Accepts ``ConnectionHolder`` (shared) or legacy ``Path``. + """ + + def __init__(self, source: Union[ConnectionHolder, Path]) -> None: + self._owns_holder = isinstance(source, Path) + if self._owns_holder: + source = LocalConnectionHolder(source) + self._holder = source + self._con = self._holder.connection self._init_schema() def close(self) -> None: - self._con.close() + if self._owns_holder: + self._holder.close() def _init_schema(self) -> None: self._con.execute(""" @@ -66,7 +73,8 @@ def save( ) -> None: """Persist or update a learning session record.""" now = time.time() - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_learning_session " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", [ @@ -109,7 +117,8 @@ def find_by_trajectory(self, trajectory_id: str) -> Optional[Dict[str, Any]]: def mark_completed(self, session_id: str, end_time: Optional[float] = None) -> None: """Mark a session as completed (learning triggered).""" - self._con.execute( + execute_with_retry( + self._con, "UPDATE leap_learning_session SET status = 'completed', end_time = ? " "WHERE session_id = ?", [end_time or time.time(), session_id], @@ -117,7 +126,8 @@ def mark_completed(self, session_id: str, end_time: Optional[float] = None) -> N def mark_abandoned(self, session_id: str) -> None: """Mark a session as abandoned (quit without learning).""" - self._con.execute( + execute_with_retry( + self._con, "UPDATE leap_learning_session SET status = 'abandoned', end_time = ? " "WHERE session_id = ?", [time.time(), session_id], diff --git a/src/leapflow/storage/skill_library.py b/src/leapflow/storage/skill_library.py index 1361767..42a203b 100644 --- a/src/leapflow/storage/skill_library.py +++ b/src/leapflow/storage/skill_library.py @@ -16,7 +16,7 @@ import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import duckdb @@ -25,6 +25,8 @@ DistillationCandidate, RecoveryEvent, ) +from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder +from leapflow.storage.write_buffer import execute_with_retry if TYPE_CHECKING: from leapflow.skills.registry import Skill @@ -95,17 +97,28 @@ class SkillUpdateSuggestion: class SkillLibraryStore: - """DuckDB-backed CRUD for the skill library and suggestion queue.""" + """DuckDB-backed CRUD for the skill library and suggestion queue. - def __init__(self, db_path: Path, *, audit_logger: Optional[Any] = None) -> None: - self._path = db_path - self._path.parent.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(str(self._path)) + Accepts ``ConnectionHolder`` (shared) or legacy ``Path``. + """ + + def __init__( + self, + source: Union[ConnectionHolder, Path], + *, + audit_logger: Optional[Any] = None, + ) -> None: + self._owns_holder = isinstance(source, Path) + if self._owns_holder: + source = LocalConnectionHolder(source) + self._holder = source + self._con = self._holder.connection self._audit_logger = audit_logger self._init_schema() def close(self) -> None: - self._con.close() + if self._owns_holder: + self._holder.close() # ── Schema ── @@ -205,7 +218,8 @@ def _init_schema(self) -> None: def save_skill(self, skill: StoredSkill) -> None: now = time.time() - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_skill_library VALUES " "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", [ @@ -286,7 +300,8 @@ def save_from_candidate( # ── Suggestion CRUD ── def save_suggestion(self, s: SkillUpdateSuggestion) -> None: - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_skill_suggestion VALUES " "(?,?,?,?,?,?,?,?,?,?,?,?,?)", [ @@ -357,7 +372,8 @@ def count_pending(self) -> int: return int(result[0]) if result else 0 def resolve_suggestion(self, suggestion_id: str, status: str) -> None: - self._con.execute( + execute_with_retry( + self._con, "UPDATE leap_skill_suggestion SET status = ?, resolved_at = ? WHERE id = ?", [status, time.time(), suggestion_id], ) @@ -365,7 +381,8 @@ def resolve_suggestion(self, suggestion_id: str, status: str) -> None: # ── Execution log CRUD ── def save_execution(self, execution: SkillExecution) -> None: - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_skill_execution VALUES " "(?,?,?,?,?,?,?,?,?)", [ @@ -446,7 +463,8 @@ def save_parameterized_skill( "default": p.default, "description": p.description} for p in skill.parameters] ) - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_parameterized_skills VALUES " "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", [ @@ -489,12 +507,12 @@ def load_all_active_parameterized(self) -> List[Dict[str, Any]]: def deactivate_parameterized(self, name: str) -> bool: """Soft-delete a parameterized skill. Returns True if it existed.""" - result = self._con.execute( + execute_with_retry( + self._con, "UPDATE leap_parameterized_skills SET is_active = FALSE, updated_at = ? " "WHERE name = ? AND is_active = TRUE", [time.time(), name], ) - result.fetchone() # DuckDB returns None on success # Check if row existed check = self._con.execute( "SELECT 1 FROM leap_parameterized_skills WHERE name = ?", [name] @@ -503,7 +521,8 @@ def deactivate_parameterized(self, name: str) -> bool: def update_skill_confidence(self, name: str, confidence: float) -> None: """Update confidence for a parameterized skill by name.""" - self._con.execute( + execute_with_retry( + self._con, "UPDATE leap_parameterized_skills " "SET confidence = ?, updated_at = ? WHERE name = ?", [confidence, time.time(), name], @@ -520,7 +539,8 @@ def update_parameterized_version( if row is None: raise KeyError(f"Parameterized skill '{name}' not found") new_version = row[0] + 1 - self._con.execute( + execute_with_retry( + self._con, "UPDATE leap_parameterized_skills " "SET version = ?, code = ?, updated_at = ? WHERE name = ?", [new_version, new_code, time.time(), name], diff --git a/src/leapflow/storage/trajectory_store.py b/src/leapflow/storage/trajectory_store.py index bada258..b4cbc82 100644 --- a/src/leapflow/storage/trajectory_store.py +++ b/src/leapflow/storage/trajectory_store.py @@ -10,9 +10,7 @@ import logging import time from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence - -import duckdb +from typing import Any, Dict, List, Optional, Sequence, Union from leapflow.domain.trajectory import ( ActionType, @@ -23,30 +21,35 @@ Trajectory, TrajectoryStep, ) +from leapflow.storage.connection import ConnectionHolder, LocalConnectionHolder +from leapflow.storage.write_buffer import WriteBuffer, execute_with_retry logger = logging.getLogger(__name__) -_MAX_WRITE_BUFFER = 500 - - class TrajectoryStore: """Append-oriented store for trajectories, steps, and episodes. Write failures are buffered in memory and retried on the next successful write, preventing data loss during transient DuckDB errors. + + Accepts either a ``ConnectionHolder`` (shared connection) or a legacy + ``Path`` (auto-wrapped in ``LocalConnectionHolder``). """ - def __init__(self, db_path: Path) -> None: - self._path = db_path - self._path.parent.mkdir(parents=True, exist_ok=True) - self._con = duckdb.connect(str(self._path)) - self._write_buffer: list = [] + def __init__(self, source: Union[ConnectionHolder, Path]) -> None: + self._owns_holder = isinstance(source, Path) + if self._owns_holder: + source = LocalConnectionHolder(source) + self._holder = source + self._con = self._holder.connection + self._write_buffer = WriteBuffer(self._con) self._init_schema() def close(self) -> None: - self._flush_buffer() - self._con.close() + self._write_buffer.flush() + if self._owns_holder: + self._holder.close() # ── Schema ── @@ -164,7 +167,8 @@ def _migrate_episode_table(self) -> None: def save_trajectory(self, traj: Trajectory) -> None: """Persist a complete trajectory with all its steps.""" now = time.time() - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_trajectory VALUES (?, ?, ?, ?, ?, ?, ?)", [ traj.trajectory_id, @@ -177,7 +181,12 @@ def save_trajectory(self, traj: Trajectory) -> None: ], ) for idx, step in enumerate(traj.steps): - self._insert_step(traj.trajectory_id, idx, step) + execute_with_retry( + self._con, + "INSERT OR REPLACE INTO leap_trajectory_step VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + self._step_params(traj.trajectory_id, idx, step), + ) logger.info( "trajectory.saved id=%s steps=%d duration=%.1fs", traj.trajectory_id, @@ -186,55 +195,37 @@ def save_trajectory(self, traj: Trajectory) -> None: ) def append_step(self, trajectory_id: str, step_idx: int, step: TrajectoryStep) -> None: - """Append a single step (buffered on failure).""" + """Append a single step (immediate write, buffered on failure).""" + sql = ( + "INSERT OR REPLACE INTO leap_trajectory_step VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ) + params = self._step_params(trajectory_id, step_idx, step) try: - self._insert_step(trajectory_id, step_idx, step) - self._flush_buffer() - except Exception as e: - logger.warning("store.append_step buffered: %s", e) - self._buffer_write("step", (trajectory_id, step_idx, step)) + execute_with_retry(self._con, sql, params) + self._write_buffer.flush() + except Exception as exc: + logger.warning("store.append_step buffered: %s", exc) + self._write_buffer.append("step", sql, params) def finalize_trajectory(self, traj: Trajectory) -> None: - """Update trajectory header (buffered on failure).""" + """Update trajectory header (immediate write, buffered on failure).""" + sql = "INSERT OR REPLACE INTO leap_trajectory VALUES (?, ?, ?, ?, ?, ?, ?)" + params = [ + traj.trajectory_id, + traj.user_id, + traj.start_time, + traj.end_time, + traj.step_count, + json.dumps(traj.metadata, ensure_ascii=False), + time.time(), + ] try: - self._finalize_sql(traj) - self._flush_buffer() - except Exception as e: - logger.warning("store.finalize buffered: %s", e) - self._buffer_write("finalize", (traj,)) - - def _finalize_sql(self, traj: Trajectory) -> None: - self._con.execute( - "INSERT OR REPLACE INTO leap_trajectory VALUES (?, ?, ?, ?, ?, ?, ?)", - [ - traj.trajectory_id, - traj.user_id, - traj.start_time, - traj.end_time, - traj.step_count, - json.dumps(traj.metadata, ensure_ascii=False), - time.time(), - ], - ) - - def _buffer_write(self, op_type: str, args: tuple) -> None: - self._write_buffer.append((op_type, args)) - if len(self._write_buffer) > _MAX_WRITE_BUFFER: - self._write_buffer.pop(0) - - def _flush_buffer(self) -> None: - if not self._write_buffer: - return - remaining = [] - for op_type, args in self._write_buffer: - try: - if op_type == "step": - self._insert_step(*args) - elif op_type == "finalize": - self._finalize_sql(*args) - except Exception: - remaining.append((op_type, args)) - self._write_buffer = remaining + execute_with_retry(self._con, sql, params) + self._write_buffer.flush() + except Exception as exc: + logger.warning("store.finalize buffered: %s", exc) + self._write_buffer.append("finalize", sql, params) def load_trajectory(self, trajectory_id: str) -> Optional[Trajectory]: """Load a trajectory with all its steps.""" @@ -295,7 +286,8 @@ def list_trajectories( def save_episode(self, episode: Episode) -> None: """Persist an episode.""" - self._con.execute( + execute_with_retry( + self._con, "INSERT OR REPLACE INTO leap_episode " "(id, trajectory_id, start_idx, end_idx, inferred_goal, " "app_sequence, semantic_actions, confidence, created_at, procedure_graph) " @@ -319,11 +311,17 @@ def save_episode(self, episode: Episode) -> None: def delete_episodes(self, trajectory_id: str) -> int: """Delete all episodes for a trajectory. Returns count deleted.""" - result = self._con.execute( - "DELETE FROM leap_episode WHERE trajectory_id = ? RETURNING id", + count = self._con.execute( + "SELECT COUNT(*) FROM leap_episode WHERE trajectory_id = ?", [trajectory_id], - ).fetchall() - return len(result) + ).fetchone()[0] + if count > 0: + execute_with_retry( + self._con, + "DELETE FROM leap_episode WHERE trajectory_id = ?", + [trajectory_id], + ) + return count def load_episodes(self, trajectory_id: str) -> List[Episode]: """Load all episodes for a trajectory.""" @@ -356,31 +354,28 @@ def search_episodes_by_goal( # ── Internal helpers ── - def _insert_step(self, trajectory_id: str, idx: int, step: TrajectoryStep) -> None: - self._con.execute( - "INSERT OR REPLACE INTO leap_trajectory_step VALUES " - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - [ - trajectory_id, - idx, - step.action.timestamp, - step.action.action_type.value, - step.action.target, - step.action.target_label, - step.action.target_role, - step.action.app_bundle_id, - step.action.app_name, - json.dumps(step.action.params, ensure_ascii=False) if step.action.params else None, - step.state.focused_app, - step.state.ax_tree_digest, - step.state.clipboard_text, - step.state.visual_frame_ref, - json.dumps(step.state.ax_tree_snapshot, ensure_ascii=False) - if step.state.ax_tree_snapshot - else None, - step.state.snapshot_level, - ], - ) + @staticmethod + def _step_params(trajectory_id: str, idx: int, step: TrajectoryStep) -> list: + return [ + trajectory_id, + idx, + step.action.timestamp, + step.action.action_type.value, + step.action.target, + step.action.target_label, + step.action.target_role, + step.action.app_bundle_id, + step.action.app_name, + json.dumps(step.action.params, ensure_ascii=False) if step.action.params else None, + step.state.focused_app, + step.state.ax_tree_digest, + step.state.clipboard_text, + step.state.visual_frame_ref, + json.dumps(step.state.ax_tree_snapshot, ensure_ascii=False) + if step.state.ax_tree_snapshot + else None, + step.state.snapshot_level, + ] def _load_steps(self, trajectory_id: str) -> List[TrajectoryStep]: rows = self._con.execute( diff --git a/src/leapflow/storage/write_buffer.py b/src/leapflow/storage/write_buffer.py new file mode 100644 index 0000000..60feb6a --- /dev/null +++ b/src/leapflow/storage/write_buffer.py @@ -0,0 +1,146 @@ +"""Batched write buffer for DuckDB stores. + +High-frequency signal writes to DuckDB benefit from batching: +- Reduces WAL pressure and MVCC overhead +- Provides resilience against transient lock errors +- Flushes on count threshold OR timer, whichever fires first + +Each store that does frequent inserts should use ``WriteBuffer`` to +accumulate operations and flush them as a batch. + +Additionally provides ``execute_with_retry`` for stores that need +jitter-based retry on individual write operations. +""" +from __future__ import annotations + +import logging +import os +import random +import time +from typing import Any, List, Tuple + +import duckdb + +from leapflow.storage.duckdb_connect import is_lock_error + +logger = logging.getLogger(__name__) + +_WRITE_RETRIES = int(os.getenv("LEAPFLOW_DB_WRITE_RETRIES", "10")) +_JITTER_MIN_MS = float(os.getenv("LEAPFLOW_DB_JITTER_MIN_MS", "15")) +_JITTER_MAX_MS = float(os.getenv("LEAPFLOW_DB_JITTER_MAX_MS", "120")) + + +def execute_with_retry( + conn: duckdb.DuckDBPyConnection, + sql: str, + params: Any = None, + *, + retries: int = _WRITE_RETRIES, + jitter_ms: Tuple[float, float] = (_JITTER_MIN_MS, _JITTER_MAX_MS), +) -> None: + """Execute a write SQL with jitter retry on lock errors. + + Raises the original exception if all retries are exhausted or if the + error is not a lock conflict. + """ + for attempt in range(retries): + try: + if params: + conn.execute(sql, params) + else: + conn.execute(sql) + return + except Exception as exc: + if is_lock_error(exc) and attempt < retries - 1: + jitter = random.uniform(*jitter_ms) / 1000 + time.sleep(jitter) + continue + raise + + +_Op = Tuple[str, str, Any] # (op_tag, sql, params) + + +class WriteBuffer: + """Time-and-count gated write buffer. + + Accumulates ``(tag, sql, params)`` tuples and flushes them as a + batch when either the count threshold or the timer fires. + + Parameters + ---------- + conn : duckdb.DuckDBPyConnection + The database connection to flush to. + max_count : int + Flush when the buffer reaches this many operations. + max_interval_s : float + Flush when this many seconds have elapsed since the last flush. + max_capacity : int + Hard cap on buffer size; oldest entries are dropped when exceeded. + """ + + def __init__( + self, + conn: duckdb.DuckDBPyConnection, + *, + max_count: int = 100, + max_interval_s: float = 0.5, + max_capacity: int = 2000, + ) -> None: + self._conn = conn + self._max_count = max_count + self._max_interval_s = max_interval_s + self._max_capacity = max_capacity + self._buffer: List[_Op] = [] + self._last_flush: float = time.monotonic() + + @property + def pending(self) -> int: + return len(self._buffer) + + def append(self, tag: str, sql: str, params: Any = None) -> None: + """Add an operation to the buffer, flushing if thresholds are met.""" + self._buffer.append((tag, sql, params)) + if len(self._buffer) > self._max_capacity: + self._buffer.pop(0) + if self._should_flush(): + self.flush() + + def _should_flush(self) -> bool: + if len(self._buffer) >= self._max_count: + return True + if time.monotonic() - self._last_flush >= self._max_interval_s: + return True + return False + + def flush(self) -> int: + """Execute all buffered operations. Returns count flushed.""" + if not self._buffer: + return 0 + + flushed = 0 + remaining: List[_Op] = [] + for tag, sql, params in self._buffer: + try: + execute_with_retry(self._conn, sql, params) + flushed += 1 + except Exception as exc: + if is_lock_error(exc): + logger.warning("write_buffer: transient flush failed for %s: %s", tag, exc) + remaining.append((tag, sql, params)) + continue + logger.error( + "write_buffer: dropping permanent failed op %s: %s", + tag, + exc, + ) + + self._buffer = remaining + self._last_flush = time.monotonic() + if flushed: + logger.debug("write_buffer: flushed %d ops, %d remaining", flushed, len(remaining)) + return flushed + + def update_connection(self, conn: duckdb.DuckDBPyConnection) -> None: + """Update the underlying connection (used during connection sharing).""" + self._conn = conn diff --git a/src/leapflow/tools/file_operations.py b/src/leapflow/tools/file_operations.py index ddd6ece..5eab555 100644 --- a/src/leapflow/tools/file_operations.py +++ b/src/leapflow/tools/file_operations.py @@ -34,12 +34,14 @@ "token.json", "secrets.yaml", "secrets.yml", ".kube/config", "id_rsa", "id_ed25519", + "gateway.yaml", ".credential_key", }) _SENSITIVE_EXACT_NAMES: FrozenSet[str] = frozenset({ ".env", ".env.local", ".env.production", ".env.staging", "credentials.json", "service_account.json", "token.json", "secrets.yaml", "secrets.yml", ".netrc", ".npmrc", ".pypirc", + "gateway.yaml", ".credential_key", }) # Binary extensions that should not be read as text diff --git a/src/leapflow/tools/gateway_tool.py b/src/leapflow/tools/gateway_tool.py new file mode 100644 index 0000000..bedbe9b --- /dev/null +++ b/src/leapflow/tools/gateway_tool.py @@ -0,0 +1,432 @@ +"""Gateway tools for the agent — configuration AND messaging. + +Two tools: +- ``gateway_connect``: conversational platform configuration +- ``gateway_send``: proactive outbound messaging to connected platforms + +SECURITY: Tool results NEVER contain credential values. +Credentials flow: user message → LLM parse → tool handler → vault. +Tool returns only status information back to LLM. +""" +from __future__ import annotations + +import logging +from typing import Any, Dict, List + +logger = logging.getLogger(__name__) + +_gateway_server_ref: Any = None +_approval_gate: Any = None + + +def set_gateway_server(server: Any) -> None: + """Install ``GatewayServer`` reference for tool dispatch (late-bound).""" + global _gateway_server_ref + _gateway_server_ref = server + + +def set_gateway_approval_gate(gate: Any) -> None: + """Install approval gate for outbound messaging (late-bound).""" + global _approval_gate + _approval_gate = gate + + +# ═══════════════════════════════════════════════════════════════ +# Tool handler +# ═══════════════════════════════════════════════════════════════ + +async def gateway_connect_handler(params: Dict[str, Any]) -> Dict[str, Any]: + """Handle ``gateway_connect`` tool calls from the agent. + + Actions: + list — show available and connected platforms + guide — get setup instructions for a platform + connect — connect with provided credentials + disconnect — disconnect a platform + status — check connection health + """ + if _gateway_server_ref is None: + return {"ok": False, "error": "Gateway not initialised"} + + action = params.get("action", "list") + platform = params.get("platform", "") + + dispatch = { + "list": _action_list, + "guide": _action_guide, + "connect": _action_connect, + "disconnect": _action_disconnect, + "remove": _action_remove, + "status": _action_status, + } + handler = dispatch.get(action) + if handler is None: + return {"ok": False, "error": f"Unknown action: {action}"} + + return await handler(platform, params) + + +# ── Action implementations ─────────────────────────────────── + +async def _action_list( + _platform: str, _params: Dict[str, Any], +) -> Dict[str, Any]: + """List available platforms and their connection status.""" + import time + + statuses = _gateway_server_ref.platform_status() + platforms = [] + for s in statuses: + manifest = _gateway_server_ref.manifests.get(s.platform_id) + state = "connected" if s.connected else ( + "configured" if s.error == "configured but not connected" else "available" + ) + entry: Dict[str, Any] = { + "id": s.platform_id, + "name": manifest.display_name if manifest else s.platform_id, + "state": state, + "category": manifest.category if manifest else "", + } + if s.connected and s.connected_since > 0: + uptime_s = int(time.time() - s.connected_since) + entry["uptime"] = _format_uptime(uptime_s) + platforms.append(entry) + return {"ok": True, "platforms": platforms} + + +def _format_uptime(seconds: int) -> str: + """Format seconds into a human-readable duration string.""" + if seconds < 60: + return f"{seconds}s" + if seconds < 3600: + return f"{seconds // 60}m {seconds % 60}s" + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + return f"{hours}h {minutes}m" + + +async def _action_guide( + platform: str, _params: Dict[str, Any], +) -> Dict[str, Any]: + """Return setup guide for a platform (no credentials exposed).""" + manifest = _gateway_server_ref.manifests.get(platform) + if manifest is None: + available = list(_gateway_server_ref.manifests.keys()) + return { + "ok": False, + "error": f"Unknown platform: {platform}. Available: {available}", + } + + fields = [ + { + "key": c.key, + "label": c.label, + "required": c.required, + "help": c.help_zh or c.help_en, + } + for c in manifest.credentials + ] + options = [ + { + "key": o.key, + "label": o.label, + "default": o.default, + "choices": list(o.choices), + "help": o.help_zh or o.help_en, + } + for o in manifest.options + if not o.advanced + ] + + result: Dict[str, Any] = { + "ok": True, + "platform": manifest.display_name, + "setup_guide": ( + manifest.setup_guide.summary_zh or manifest.setup_guide.summary_en + ), + "required_fields": fields, + "console_url": manifest.setup_guide.console_url, + } + + if manifest.setup_guide.steps_zh: + result["setup_steps"] = list(manifest.setup_guide.steps_zh) + elif manifest.setup_guide.steps_en: + result["setup_steps"] = list(manifest.setup_guide.steps_en) + + if options: + result["optional_settings"] = options + + result["setup_form"] = { + "fields": [ + { + "key": c.key, + "label": c.label, + "type": "password" if c.secret else "text", + "required": c.required, + } + for c in manifest.credentials + ], + "console_url": manifest.setup_guide.console_url, + } + + required = [c for c in manifest.credentials if c.required] + labels = " / ".join(c.label for c in required) + result["prompt_hint"] = ( + f"Present the setup steps above, then ask the user to provide " + f"{labels} in a single reply. Do NOT repeat credential values " + f"in your response." + ) + + return result + + +async def _action_connect( + platform: str, params: Dict[str, Any], +) -> Dict[str, Any]: + """Connect a platform with provided credentials. + + SECURITY: credentials flow into ``connect_platform()`` which encrypts + and persists them. Return value contains **only** status. + """ + credentials = params.get("credentials", {}) + options = params.get("options", {}) + + if not platform: + return {"ok": False, "error": "Platform ID is required"} + if not credentials: + return await _action_guide(platform, params) + + return await _gateway_server_ref.connect_platform( + platform, credentials, options, + ) + + +async def _action_disconnect( + platform: str, _params: Dict[str, Any], +) -> Dict[str, Any]: + """Disconnect a platform (keeps saved credentials for reconnect).""" + if not platform: + return {"ok": False, "error": "Platform ID is required"} + return await _gateway_server_ref.disconnect_platform(platform) + + +async def _action_remove( + platform: str, _params: Dict[str, Any], +) -> Dict[str, Any]: + """Disconnect AND delete saved credentials for a platform. + + Unlike ``disconnect``, this fully removes the platform configuration + from ``gateway.yaml`` and the ``auto_connect`` list. + """ + if not platform: + return {"ok": False, "error": "Platform ID is required"} + + await _gateway_server_ref.disconnect_platform(platform) + _gateway_server_ref.remove_platform_config(platform) + return {"ok": True, "status": "removed"} + + +async def _action_status( + platform: str, _params: Dict[str, Any], +) -> Dict[str, Any]: + """Check status of a specific platform or all platforms.""" + import time + + statuses = _gateway_server_ref.platform_status() + + def _status_entry(s: Any) -> Dict[str, Any]: + entry: Dict[str, Any] = { + "id": s.platform_id, + "connected": s.connected, + } + if s.error: + entry["error"] = s.error + if s.connected and s.connected_since > 0: + entry["uptime"] = _format_uptime(int(time.time() - s.connected_since)) + return entry + + if platform: + for s in statuses: + if s.platform_id == platform: + result = _status_entry(s) + result["ok"] = True + result["platform"] = result.pop("id") + return result + return {"ok": False, "error": f"Platform not found: {platform}"} + return {"ok": True, "platforms": [_status_entry(s) for s in statuses]} + + +# ═══════════════════════════════════════════════════════════════ +# gateway_send handler — proactive outbound messaging +# ═══════════════════════════════════════════════════════════════ + +async def gateway_send_handler(params: Dict[str, Any]) -> Dict[str, Any]: + """Handle ``gateway_send`` tool calls — send messages to connected platforms. + + Enables the agent to proactively message any connected platform + conversation (e.g. post to a Feishu group, reply in a Telegram chat). + + First use per platform requires user approval (session-scoped). + """ + if _gateway_server_ref is None: + return {"ok": False, "error": "Gateway not initialised"} + + platform = params.get("platform", "") + chat_id = params.get("chat_id", "") + text = params.get("text", "") + + if not platform: + return {"ok": False, "error": "platform is required"} + if not chat_id: + return {"ok": False, "error": "chat_id is required"} + if not text: + return {"ok": False, "error": "text is required"} + + if _approval_gate is not None: + try: + from leapflow.security.approval import ApprovalDecision, ApprovalRequest + + preview = text[:80] + ("…" if len(text) > 80 else "") + decision = await _approval_gate.request_approval(ApprovalRequest( + category=f"gateway_send:{platform}", + detail=f"Send to {platform}/{chat_id}: {preview}", + risk_hint=0.5, + metadata={"platform": platform, "chat_id": chat_id}, + )) + if decision == ApprovalDecision.DENY: + return {"ok": False, "error": "Outbound message denied by approval gate"} + except Exception: + logger.debug("gateway_send approval check failed", exc_info=True) + + return await _gateway_server_ref.send_message( + platform, + chat_id, + text, + thread_id=params.get("thread_id", ""), + ) + + +# ═══════════════════════════════════════════════════════════════ +# Tool registration (OpenAI function calling schema) +# ═══════════════════════════════════════════════════════════════ + +GATEWAY_TOOL_DEFINITIONS: List[Dict[str, Any]] = [ + { + "type": "function", + "function": { + "name": "gateway_send", + "description": ( + "Send a message to a connected external platform " + "(Feishu group, Telegram chat, DingTalk conversation, etc.). " + "Requires the platform to be connected via gateway_connect first. " + "Use gateway_connect with action='list' to see connected platforms " + "and available chat IDs." + ), + "parameters": { + "type": "object", + "properties": { + "platform": { + "type": "string", + "description": "Platform ID (feishu, telegram, dingtalk, etc.)", + }, + "chat_id": { + "type": "string", + "description": "Target chat/group/channel ID", + }, + "text": { + "type": "string", + "description": "Message text to send", + }, + "thread_id": { + "type": "string", + "description": "Thread/topic ID for threaded replies (optional)", + }, + }, + "required": ["platform", "chat_id", "text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "gateway_connect", + "description": ( + "Connect, configure, or manage external platform integrations " + "(Feishu, DingTalk, Telegram, Slack, Discord, etc.). " + "Conversational flow: 1) call 'guide' to get setup steps + " + "required fields, 2) present the steps to the user and ask " + "for ALL required credentials in a single message, 3) call " + "'connect' with the credentials. Goal: complete in 1–2 user " + "turns. NEVER include credential values in your text response." + ), + "parameters": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "list", "guide", "connect", "disconnect", + "remove", "status", + ], + "description": ( + "Action to perform. 'disconnect' pauses the " + "connection (credentials kept for reconnect); " + "'remove' deletes saved credentials entirely." + ), + }, + "platform": { + "type": "string", + "description": ( + "Platform ID (feishu, dingtalk, telegram, etc.)" + ), + }, + "credentials": { + "type": "object", + "description": ( + "Platform credentials (keys vary by platform)" + ), + }, + "options": { + "type": "object", + "description": ( + "Optional platform configuration overrides" + ), + }, + }, + "required": ["action"], + }, + }, + }, +] + +GATEWAY_BRIDGE_TOOLS: List[Dict[str, Any]] = [ + { + "name": "gp_gateway_connect", + "description": "Connect or manage external platform integrations.", + "parameters": { + "action": "string (required) — list/guide/connect/disconnect/remove/status", + "platform": "string (optional) — platform ID", + "credentials": "object (optional) — platform credentials", + "options": "object (optional) — configuration overrides", + }, + "handler": gateway_connect_handler, + }, + { + "name": "gp_gateway_send", + "description": "Send a message to a connected external platform.", + "parameters": { + "platform": "string (required) — platform ID", + "chat_id": "string (required) — target chat/group ID", + "text": "string (required) — message text", + "thread_id": "string (optional) — thread ID for replies", + }, + "handler": gateway_send_handler, + }, +] + +GATEWAY_TOOL_HANDLERS: Dict[str, Any] = { + "gateway_connect": gateway_connect_handler, + "gp_gateway_connect": gateway_connect_handler, + "gateway_send": gateway_send_handler, + "gp_gateway_send": gateway_send_handler, +} diff --git a/src/leapflow/tools/registry_bootstrap.py b/src/leapflow/tools/registry_bootstrap.py index eb71267..fd6359c 100644 --- a/src/leapflow/tools/registry_bootstrap.py +++ b/src/leapflow/tools/registry_bootstrap.py @@ -18,6 +18,12 @@ HUB_TOOL_DEFINITIONS, HUB_TOOL_HANDLERS, ) +from leapflow.tools.gateway_tool import ( + GATEWAY_BRIDGE_TOOLS, + GATEWAY_TOOL_DEFINITIONS, + GATEWAY_TOOL_HANDLERS, + set_gateway_server, +) # ───────────────────────────────────────────────────────────────────── @@ -214,7 +220,7 @@ }, }, }, -] + HUB_TOOL_DEFINITIONS +] + HUB_TOOL_DEFINITIONS + GATEWAY_TOOL_DEFINITIONS # ───────────────────────────────────────────────────────────────────── @@ -314,7 +320,7 @@ }, "handler": skill_view, }, -] + HUB_BRIDGE_TOOLS +] + HUB_BRIDGE_TOOLS + GATEWAY_BRIDGE_TOOLS # ───────────────────────────────────────────────────────────────────── @@ -328,6 +334,8 @@ TOOL_HANDLERS: Dict[str, Any] = {t["name"]: t["handler"] for t in _BRIDGE_TOOLS} # Add hub tool handlers TOOL_HANDLERS.update(HUB_TOOL_HANDLERS) +# Add gateway tool handlers +TOOL_HANDLERS.update(GATEWAY_TOOL_HANDLERS) # Add unprefixed aliases matching TOOL_DEFINITIONS names for native tool_calls for _td in TOOL_DEFINITIONS: _func_name = _td.get("function", {}).get("name", "") diff --git a/tests/conftest.py b/tests/conftest.py index 40f2e07..4a1cc53 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -96,7 +96,7 @@ def trajectory_store(tmp_db: Path) -> TrajectoryStore: @pytest.fixture def long_term_memory(tmp_db: Path) -> SemanticMemoryProvider: - lt = SemanticMemoryProvider(db_path=tmp_db) + lt = SemanticMemoryProvider(source=tmp_db) lt._ensure_connection() yield lt lt.close() diff --git a/tests/test_agent_execution.py b/tests/test_agent_execution.py index df86bca..f4875a8 100644 --- a/tests/test_agent_execution.py +++ b/tests/test_agent_execution.py @@ -70,7 +70,7 @@ async def test_simple_query_returns_answer() -> None: rpc = MockBridge() llm = StubLLM([answer]) wm = WorkingMemoryProvider(max_tokens=1024) - lt = SemanticMemoryProvider(db_path=settings.duckdb_path) + lt = SemanticMemoryProvider(source=settings.duckdb_path) imm = EpisodicMemoryProvider() imm.ingest("event.fs_change", "File modified: /tmp/README.md", path="/tmp/README.md") try: @@ -102,7 +102,7 @@ async def test_react_loop_tool_then_answer() -> None: ] ) wm = WorkingMemoryProvider(max_tokens=1024) - lt = SemanticMemoryProvider(db_path=settings.duckdb_path) + lt = SemanticMemoryProvider(source=settings.duckdb_path) imm = EpisodicMemoryProvider() try: reg = build_default_registry(rpc, llm, wm, lt) @@ -125,7 +125,7 @@ async def test_dag_execution_end_to_end() -> None: rpc = MockBridge() llm = StubLLM([]) wm = WorkingMemoryProvider(max_tokens=1024) - lt = SemanticMemoryProvider(db_path=settings.duckdb_path) + lt = SemanticMemoryProvider(source=settings.duckdb_path) imm = EpisodicMemoryProvider() planned = TaskGraph(goal="organize downloads") @@ -176,7 +176,7 @@ async def test_engine_remembers_context() -> None: ] ) wm = WorkingMemoryProvider(max_tokens=1024) - lt = SemanticMemoryProvider(db_path=settings.duckdb_path) + lt = SemanticMemoryProvider(source=settings.duckdb_path) imm = EpisodicMemoryProvider() try: reg = build_default_registry(rpc, llm, wm, lt) @@ -197,6 +197,46 @@ async def test_engine_remembers_context() -> None: lt.close() +@pytest.mark.asyncio +async def test_streaming_engine_estimates_context_tokens_without_provider_usage() -> None: + """Streaming providers often omit usage; status still needs prompt utilization.""" + from leapflow.llm.base import LLMChatResponse, LLMProvider + from leapflow.platform.mock import MockBridge + + class StreamingOnlyLLM(LLMProvider): + async def achat(self, messages, *, stream=True, enable_thinking=False, on_chunk=None, **kwargs): + return LLMChatResponse(content="fallback") + + async def achat_stream(self, messages, *, enable_thinking=False, **kwargs): + yield "streamed answer" + + with tempfile.TemporaryDirectory() as td: + settings = make_settings(td) + settings = settings.__class__( + **{ + **settings.__dict__, + "stream_output": True, + "native_tool_calling_enabled": False, + } + ) + rpc = MockBridge() + llm = StreamingOnlyLLM() + wm = WorkingMemoryProvider(max_tokens=1024) + lt = SemanticMemoryProvider(source=settings.duckdb_path) + imm = EpisodicMemoryProvider() + try: + reg = build_default_registry(rpc, llm, wm, lt) + classifier = _FixedClassifier("complex") + engine = AgentEngine(settings, rpc, llm, wm, lt, imm, reg, classifier) + + events = [event async for event in engine.run_stream("Summarize a long conversation")] + + assert any(event.type == "final" for event in events) + assert engine.context_token_count > 0 + finally: + lt.close() + + @pytest.mark.asyncio async def test_immediate_memory_integration() -> None: """EpisodicMemoryProvider fragments surface in memory_recent responses.""" @@ -207,7 +247,7 @@ async def test_immediate_memory_integration() -> None: rpc = MockBridge() llm = StubLLM(["You recently modified /tmp/README.md"]) wm = WorkingMemoryProvider(max_tokens=1024) - lt = SemanticMemoryProvider(db_path=settings.duckdb_path) + lt = SemanticMemoryProvider(source=settings.duckdb_path) imm = EpisodicMemoryProvider() imm.ingest("event.fs_change", "File modified: /tmp/README.md", path="/tmp/README.md") try: diff --git a/tests/test_cli_entrypoint.py b/tests/test_cli_entrypoint.py new file mode 100644 index 0000000..0512f88 --- /dev/null +++ b/tests/test_cli_entrypoint.py @@ -0,0 +1,528 @@ +from __future__ import annotations + +import argparse +import logging +from dataclasses import replace +from pathlib import Path + +import pytest + +from conftest import make_settings + + +def test_context_constructs_approval_gate_before_initialize(tmp_path) -> None: + from leapflow.cli.context import Context + + ctx = Context(make_settings(str(tmp_path)), mock_host=True) + + assert hasattr(ctx, "_approval_gate") + assert hasattr(ctx, "_tui_approval") + + +@pytest.mark.asyncio +async def test_context_initialize_wires_gateway_approval_gate(tmp_path) -> None: + from leapflow.cli.context import Context + import leapflow.tools.gateway_tool as gateway_tool + + ctx = Context(make_settings(str(tmp_path)), mock_host=True) + await ctx.initialize() + try: + assert gateway_tool._approval_gate is ctx._approval_gate + finally: + await ctx.cleanup() + + +@pytest.mark.asyncio +async def test_context_initialize_degrades_when_primary_db_is_locked( + tmp_path, + monkeypatch, +) -> None: + from pathlib import Path + + from leapflow.cli.context import Context + from leapflow.storage.duckdb_connect import DatabaseLockedError + import leapflow.storage.connection as connection_module + + settings = make_settings(str(tmp_path)) + original_connect = connection_module._lock_aware_connect + + def locked_primary_connect(db_path: Path): + if Path(db_path) == settings.duckdb_path: + raise DatabaseLockedError(settings.duckdb_path, RuntimeError("locked")) + return original_connect(db_path) + + monkeypatch.setattr( + connection_module, + "_lock_aware_connect", + locked_primary_connect, + ) + + ctx = Context(settings, mock_host=True) + await ctx.initialize() + try: + assert ctx.storage_volatile is True + assert ctx.engine is not None + assert ctx._db_holder.db_path != settings.duckdb_path + finally: + await ctx.cleanup() + + +def test_visual_track_defaults_off_without_env(monkeypatch, tmp_path) -> None: + from leapflow.config import DEFAULT_LLM_CONTEXT_LENGTH, _build_settings_from_env + + monkeypatch.delenv("LEAPFLOW_VISUAL_TRACK_ENABLED", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_VLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_CONTEXT_LENGTH", raising=False) + monkeypatch.setenv("LEAPFLOW_DATA_DIR", str(tmp_path)) + + settings = _build_settings_from_env() + + assert settings.visual_track_enabled is False + assert settings.has_vlm_credentials is False + assert settings.llm_context_length == DEFAULT_LLM_CONTEXT_LENGTH + + +def test_profile_name_rejects_path_traversal(monkeypatch, tmp_path) -> None: + from leapflow.config import _build_settings_from_env + + monkeypatch.setenv("LEAPFLOW_DATA_DIR", str(tmp_path)) + monkeypatch.setenv("LEAPFLOW_PROFILE", "../../escape") + + with pytest.raises(ValueError, match="Invalid LEAPFLOW_PROFILE"): + _build_settings_from_env() + + assert not (tmp_path.parent / "escape").exists() + + +def test_context_length_is_exposed_in_env_templates() -> None: + from leapflow.config import DEFAULT_LLM_CONTEXT_LENGTH + from leapflow._env_template import ENV_TEMPLATE + + expected = f"LEAPFLOW_LLM_CONTEXT_LENGTH={DEFAULT_LLM_CONTEXT_LENGTH}" + example = (Path(__file__).parents[1] / ".env.example").read_text(encoding="utf-8") + + assert expected in ENV_TEMPLATE + assert expected in example + assert "Runtime context budget" in ENV_TEMPLATE + + +def test_build_visual_components_degrades_without_credentials( + caplog, + tmp_path, +) -> None: + from leapflow.cli.context import _build_visual_components + + settings = replace( + make_settings(str(tmp_path)), + llm_api_key="", + vlm_api_key="", + visual_track_enabled=True, + ) + + with caplog.at_level(logging.WARNING, logger="leapflow.cli.context"): + perception_session = _build_visual_components(settings, rpc=object()) + + assert perception_session is None + assert any("Visual perception disabled" in record.message for record in caplog.records) + + +def test_build_visual_components_accepts_vlm_only_credentials( + monkeypatch, + tmp_path, +) -> None: + import leapflow.cli.context as context_module + + captured = {} + + class FakeOpenAIChat: + def __init__(self, *, api_key: str, base_url: str, model: str) -> None: + captured["api_key"] = api_key + captured["base_url"] = base_url + captured["model"] = model + + monkeypatch.setattr(context_module, "OpenAIChat", FakeOpenAIChat) + settings = replace( + make_settings(str(tmp_path)), + llm_api_key="", + vlm_api_key="vlm-test-key", + vlm_base_url="https://vlm.example.invalid/v1", + vlm_model="vlm-test-model", + visual_track_enabled=True, + ) + + perception_session = context_module._build_visual_components(settings, rpc=object()) + + assert perception_session is not None + assert captured == { + "api_key": "vlm-test-key", + "base_url": "https://vlm.example.invalid/v1", + "model": "vlm-test-model", + } + + +@pytest.mark.asyncio +async def test_context_initialize_degrades_visual_track_without_credentials( + caplog, + tmp_path, +) -> None: + from leapflow.cli.context import Context + + settings = replace( + make_settings(str(tmp_path)), + llm_api_key="", + vlm_api_key="", + visual_track_enabled=True, + ) + + ctx = Context(settings, mock_host=True) + with caplog.at_level(logging.WARNING, logger="leapflow.cli.context"): + await ctx.initialize() + try: + assert ctx.perception_session is None + assert ctx.engine is not None + assert any("Visual perception disabled" in record.message for record in caplog.records) + finally: + await ctx.cleanup() + + +@pytest.mark.asyncio +async def test_context_hot_reloads_llm_credentials_from_env_file( + monkeypatch, + tmp_path, +) -> None: + from leapflow.cli.context import Context + + data_dir = tmp_path / "leap-home" + data_dir.mkdir() + env_path = data_dir / ".env" + env_path.write_text( + "LEAPFLOW_LLM_API_KEY=\n" + "LEAPFLOW_LLM_BASE_URL=https://old.example.invalid/v1\n" + "LEAPFLOW_LLM_MODEL=old-model\n" + "LEAPFLOW_LLM_CONTEXT_LENGTH=128000\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("LEAPFLOW_LLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_BASE_URL", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_MODEL", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_CONTEXT_LENGTH", raising=False) + + settings = replace( + make_settings(str(tmp_path)), + data_dir=data_dir, + llm_api_key="", + llm_base_url="https://old.example.invalid/v1", + llm_model="old-model", + llm_context_length=128_000, + vlm_api_key="", + visual_track_enabled=False, + ) + + ctx = Context(settings, mock_host=True) + await ctx.initialize() + try: + assert ctx.settings.has_llm_credentials is False + assert ctx.engine is not None + assert ctx.engine._settings.has_llm_credentials is False + + env_path.write_text( + "LEAPFLOW_LLM_API_KEY=sk-hot-reload\n" + "LEAPFLOW_LLM_BASE_URL=https://new.example.invalid/v1\n" + "LEAPFLOW_LLM_MODEL=new-model\n" + "LEAPFLOW_LLM_CONTEXT_LENGTH=512000\n", + encoding="utf-8", + ) + + assert ctx.reload_runtime_config_if_changed() is True + assert ctx.settings.llm_api_key == "sk-hot-reload" + assert ctx.settings.llm_base_url == "https://new.example.invalid/v1" + assert ctx.settings.llm_model == "new-model" + assert ctx.settings.llm_context_length == 512_000 + assert ctx.engine._settings.llm_api_key == "sk-hot-reload" + assert ctx.engine._settings.llm_context_length == 512_000 + assert ctx.engine.model_capabilities.resolve("new-model").context_length == 512_000 + assert ctx.engine._settings.has_llm_credentials is True + assert ctx.intent_classifier.__class__.__name__ == "LLMIntentClassifier" + finally: + await ctx.cleanup() + + +@pytest.mark.asyncio +async def test_context_hot_reloads_llm_credentials_from_config_yaml( + monkeypatch, + tmp_path, +) -> None: + from leapflow.cli.context import Context + + data_dir = tmp_path / "leap-home" + data_dir.mkdir() + (data_dir / ".env").write_text("LEAPFLOW_LLM_API_KEY=\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("LEAPFLOW_LLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_BASE_URL", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_MODEL", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_CONTEXT_LENGTH", raising=False) + + settings = replace( + make_settings(str(tmp_path)), + data_dir=data_dir, + llm_api_key="", + llm_base_url="https://old.example.invalid/v1", + llm_model="old-model", + llm_context_length=128_000, + vlm_api_key="", + visual_track_enabled=False, + ) + + ctx = Context(settings, mock_host=True) + await ctx.initialize() + try: + assert ctx.settings.has_llm_credentials is False + (data_dir / "config.yaml").write_text( + "llm:\n" + " api_key: sk-yaml-hot-reload\n" + " base_url: https://yaml.example.invalid/v1\n" + " model: yaml-model\n" + " context_length: 640000\n", + encoding="utf-8", + ) + + assert ctx.reload_runtime_config_if_changed() is True + assert ctx.settings.llm_api_key == "sk-yaml-hot-reload" + assert ctx.settings.llm_base_url == "https://yaml.example.invalid/v1" + assert ctx.settings.llm_model == "yaml-model" + assert ctx.settings.llm_context_length == 640_000 + assert ctx.engine._settings.llm_api_key == "sk-yaml-hot-reload" + assert ctx.engine.model_capabilities.resolve("yaml-model").context_length == 640_000 + finally: + await ctx.cleanup() + + +@pytest.mark.asyncio +async def test_known_model_table_does_not_override_explicit_context_length( + tmp_path, +) -> None: + from leapflow.cli.context import Context + + settings = replace( + make_settings(str(tmp_path)), + llm_model="qwen3.7-plus", + llm_context_length=300_000, + visual_track_enabled=False, + ) + + ctx = Context(settings, mock_host=True) + await ctx.initialize() + try: + assert ctx.engine is not None + caps = ctx.engine.model_capabilities.resolve("qwen3.7-plus") + assert caps.context_length == 300_000 + assert caps.supports_thinking is True + finally: + await ctx.cleanup() + + +@pytest.mark.asyncio +async def test_daemon_fallback_initializes_local_interactive_with_real_config( + monkeypatch, + tmp_path, +) -> None: + from leapflow.cli import cli + from leapflow.cli.helpers import require_initialized + import leapflow.cli.commands.interactive as interactive_module + import leapflow.daemon.client as daemon_client + + data_dir = tmp_path / "leap-home" + events: list[str] = [] + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("LEAPFLOW_DATA_DIR", str(data_dir)) + monkeypatch.delenv("LEAPFLOW_LLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_VLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_VISUAL_TRACK_ENABLED", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_CONTEXT_LENGTH", raising=False) + + async def fake_ensure_daemon_client(*args, **kwargs): + raise daemon_client.DaemonUnavailableError("daemon unavailable") + + async def fake_interactive(ctx, *, resume_id=None) -> int: + require_initialized(ctx) + assert resume_id is None + assert ctx.settings.llm_api_key == "" + assert ctx.settings.visual_track_enabled is False + assert ctx.perception_session is None + events.append("interactive") + return 0 + + monkeypatch.setattr(daemon_client, "ensure_daemon_client", fake_ensure_daemon_client) + monkeypatch.setattr(interactive_module, "cmd_interactive", fake_interactive) + + result = await cli._async_daemon_main( + argparse.Namespace(command="interactive", mock_host=True, resume=None) + ) + + assert result == 0 + assert events == ["interactive"] + assert (data_dir / ".env").exists() + + +def test_leap_default_command_uses_daemon_client(monkeypatch) -> None: + from leapflow.cli import cli + + captured = {} + + async def fake_daemon_main(args): + captured["command"] = args.command + captured["no_daemon"] = args.no_daemon + return 0 + + monkeypatch.setattr(cli, "_async_daemon_main", fake_daemon_main) + + assert cli.main([]) == 0 + assert captured == {"command": "interactive", "no_daemon": False} + + +def test_leap_no_daemon_initializes_and_runs_interactive(monkeypatch, tmp_path) -> None: + from leapflow.cli import cli + from leapflow.cli.helpers import require_initialized + import leapflow.cli.commands.interactive as interactive_module + + data_dir = tmp_path / "leap-home" + events: list[str] = [] + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("LEAPFLOW_DATA_DIR", str(data_dir)) + monkeypatch.delenv("LEAPFLOW_LLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_VLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_VISUAL_TRACK_ENABLED", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_CONTEXT_LENGTH", raising=False) + + async def fake_interactive(ctx, *, resume_id=None) -> int: + require_initialized(ctx) + assert resume_id is None + assert ctx.settings.llm_api_key == "" + assert ctx.settings.visual_track_enabled is False + assert ctx.settings.has_vlm_credentials is False + assert ctx.perception_session is None + assert ctx.engine is not None + events.append("interactive") + return 0 + + monkeypatch.setattr(interactive_module, "cmd_interactive", fake_interactive) + + assert cli.main(["--no-daemon", "--mock-host"]) == 0 + assert events == ["interactive"] + assert (data_dir / ".env").exists() + + +def test_leap_daemon_restart_routes_to_daemon_command(monkeypatch) -> None: + from leapflow.cli import cli + import leapflow.cli.commands.daemon as daemon_module + + captured = {} + + def fake_cmd_daemon(args): + captured["action"] = args.daemon_action + return 0 + + monkeypatch.setattr(daemon_module, "cmd_daemon", fake_cmd_daemon) + + assert cli.main(["daemon", "restart"]) == 0 + assert captured == {"action": "restart"} + + +@pytest.mark.asyncio +async def test_daemon_tui_exit_prompt_stops_by_default(monkeypatch, tmp_path) -> None: + from leapflow.cli.commands import interactive as interactive_module + import leapflow.daemon.lifecycle as lifecycle_module + + class Client: + async def status(self): + return {"pid": 1234} + + class Console: + def __init__(self) -> None: + self.systems: list[str] = [] + self.warnings: list[str] = [] + + def system(self, message: str) -> None: + self.systems.append(message) + + def warning(self, message: str) -> None: + self.warnings.append(message) + + class Settings: + profile_dir = tmp_path + + async def yes(prompt: str) -> bool: + return True + + def record_signal(run_dir, sig): + calls.append((run_dir, sig)) + return True + + calls = [] + monkeypatch.setattr(interactive_module, "_ask_yes_no_default_yes", yes) + monkeypatch.setattr(lifecycle_module, "send_signal", record_signal) + console = Console() + + await interactive_module._prompt_stop_daemon_on_exit(Client(), Settings(), console) + + assert calls + assert "Sent SIGTERM" in console.systems[-1] + + +@pytest.mark.asyncio +async def test_daemon_tui_exit_prompt_can_keep_daemon(monkeypatch, tmp_path) -> None: + from leapflow.cli.commands import interactive as interactive_module + import leapflow.daemon.lifecycle as lifecycle_module + + class Client: + async def status(self): + return {"pid": 1234} + + class Console: + def __init__(self) -> None: + self.systems: list[str] = [] + self.warnings: list[str] = [] + + def system(self, message: str) -> None: + self.systems.append(message) + + def warning(self, message: str) -> None: + self.warnings.append(message) + + class Settings: + profile_dir = tmp_path + + def fail_send_signal(*args, **kwargs): + raise AssertionError("daemon should be kept running") + + async def no(prompt: str) -> bool: + return False + + monkeypatch.setattr(interactive_module, "_ask_yes_no_default_yes", no) + monkeypatch.setattr(lifecycle_module, "send_signal", fail_send_signal) + console = Console() + + await interactive_module._prompt_stop_daemon_on_exit(Client(), Settings(), console) + + assert any("kept running" in message for message in console.systems) + + +def test_leap_prompt_uses_daemon_chat_route(monkeypatch) -> None: + from leapflow.cli import cli + + captured = {} + + async def fake_daemon_main(args): + captured["command"] = args.command + captured["prompt"] = args.prompt + return 0 + + monkeypatch.setattr(cli, "_async_daemon_main", fake_daemon_main) + + assert cli.main(["hello", "world"]) == 0 + assert captured == {"command": "chat", "prompt": "hello world"} diff --git a/tests/test_daemon_rpc.py b/tests/test_daemon_rpc.py new file mode 100644 index 0000000..dcce213 --- /dev/null +++ b/tests/test_daemon_rpc.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import asyncio +import tempfile +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest + +from leapflow.daemon.client import DaemonClient, DaemonUnavailableError, ensure_daemon_client +from leapflow.daemon.lifecycle import DaemonInfo +from leapflow.daemon.protocol import StreamChunk +from leapflow.daemon.server import UnixRpcServer + + +class _FakeService: + def __init__(self) -> None: + self._client_count = lambda: 0 + + def set_client_count_provider(self, provider) -> None: + self._client_count = provider + + async def engine_chat(self, message: str, **kwargs: Any) -> AsyncIterator[StreamChunk]: + yield StreamChunk( + request_id="", + content=f"hello {message}", + event_type="chunk", + metadata={"session_id": "sess-1"}, + ) + yield StreamChunk(request_id="", content="done", event_type="final") + + async def status(self) -> dict[str, Any]: + return {"pid": 123, "active_clients": self._client_count(), "profile": "test"} + + +class _FailingStreamService(_FakeService): + async def engine_chat(self, message: str, **kwargs: Any) -> AsyncIterator[StreamChunk]: + yield StreamChunk(request_id="", content="partial", event_type="chunk") + raise RuntimeError("stream exploded") + + +async def _start_server(run_dir: Path, service=None): + server = UnixRpcServer( + service or _FakeService(), + sock_path=run_dir / "leapd.sock", + run_dir=run_dir, + ) + task = asyncio.create_task(server.serve_forever()) + for _ in range(50): + if (run_dir / "leapd.sock").exists(): + return server, task, run_dir + await asyncio.sleep(0.02) + task.cancel() + raise AssertionError("server did not start") + + +@pytest.mark.asyncio +async def test_daemon_client_receives_stream_events() -> None: + with tempfile.TemporaryDirectory(prefix="lfd-", dir="/tmp") as root: + server, task, run_dir = await _start_server(Path(root) / "run") + client = DaemonClient(run_dir / "leapd.sock") + + try: + events = [event async for event in client.engine_chat("world")] + finally: + task.cancel() + await server.stop() + try: + await task + except asyncio.CancelledError: + pass + + assert [(event.type, event.content) for event in events] == [ + ("chunk", "hello world"), + ("final", "done"), + ] + assert events[0].metadata == {"session_id": "sess-1"} + + +@pytest.mark.asyncio +async def test_daemon_client_reports_unknown_method() -> None: + with tempfile.TemporaryDirectory(prefix="lfd-", dir="/tmp") as root: + server, task, run_dir = await _start_server(Path(root) / "run") + client = DaemonClient(run_dir / "leapd.sock") + + try: + with pytest.raises(DaemonUnavailableError, match="Unknown method"): + await client.request("missing.method") + finally: + task.cancel() + await server.stop() + try: + await task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_ensure_daemon_client_reuses_healthy_daemon() -> None: + from conftest import make_settings + + with tempfile.TemporaryDirectory(prefix="lfd-", dir="/tmp") as root: + root_path = Path(root) + settings = make_settings(str(root_path)) + server, task, run_dir = await _start_server(settings.profile_dir / "run") + + try: + client = await ensure_daemon_client(settings) + assert client.sock_path == run_dir / "leapd.sock" + status = await client.status() + finally: + task.cancel() + await server.stop() + try: + await task + except asyncio.CancelledError: + pass + + assert status["profile"] == "test" + + +@pytest.mark.asyncio +async def test_daemon_client_surfaces_stream_errors() -> None: + with tempfile.TemporaryDirectory(prefix="lfd-", dir="/tmp") as root: + server, task, run_dir = await _start_server( + Path(root) / "run", + service=_FailingStreamService(), + ) + client = DaemonClient(run_dir / "leapd.sock") + events = [] + + try: + with pytest.raises(DaemonUnavailableError, match="Daemon stream failed"): + async for event in client.engine_chat("world"): + events.append(event) + finally: + task.cancel() + await server.stop() + try: + await task + except asyncio.CancelledError: + pass + + assert [(event.type, event.content) for event in events] == [("chunk", "partial")] + + +@pytest.mark.asyncio +async def test_runtime_service_hot_reloads_config_before_daemon_chat( + monkeypatch, + tmp_path, +) -> None: + from conftest import make_settings + from leapflow.daemon.service import RuntimeLeapService + + data_dir = tmp_path / "leap-home" + data_dir.mkdir() + env_path = data_dir / ".env" + env_path.write_text("LEAPFLOW_LLM_API_KEY=\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("LEAPFLOW_LLM_API_KEY", raising=False) + monkeypatch.delenv("LEAPFLOW_LLM_CONTEXT_LENGTH", raising=False) + + settings = make_settings(str(tmp_path)) + settings = settings.__class__( + **{ + **settings.__dict__, + "data_dir": data_dir, + "llm_api_key": "", + "llm_context_length": 128_000, + "vlm_api_key": "", + "visual_track_enabled": False, + } + ) + service = RuntimeLeapService(settings, mock_host=True) + await service.start() + stream = None + try: + env_path.write_text( + "LEAPFLOW_LLM_API_KEY=sk-daemon-hot-reload\n" + "LEAPFLOW_LLM_CONTEXT_LENGTH=700000\n", + encoding="utf-8", + ) + stream = service.engine_chat("hello") + first = await anext(stream) + + assert first.event_type == "status" + assert "Configuration reloaded" in first.content + assert first.metadata == { + "llm_model": service.context.settings.llm_model, + "llm_context_length": 700_000, + "context_used": 0, + } + assert service.context.settings.llm_api_key == "sk-daemon-hot-reload" + assert service.context.settings.llm_context_length == 700_000 + assert service.context.engine._settings.llm_api_key == "sk-daemon-hot-reload" + assert service.context.engine._settings.llm_context_length == 700_000 + status = await service.status() + assert status["config_path"] == str(data_dir / ".env") + assert status["project_env_path"] == str(tmp_path / ".env") + assert status["llm_context_length"] == 700_000 + assert status["context_used"] == 0 + assert status["runtime_source"].endswith("leapflow/__init__.py") + assert status["runtime_executable"] + assert status["runtime_version"] + finally: + if stream is not None: + await stream.aclose() + await service.shutdown() + + +@pytest.mark.asyncio +async def test_runtime_service_serializes_engine_chat_streams(tmp_path) -> None: + from conftest import make_settings + from leapflow.daemon.service import RuntimeLeapService + from leapflow.engine import StreamEvent + + class SlowEngine: + def __init__(self) -> None: + self.active = 0 + self.max_active = 0 + self.context_token_count = 2_048 + self._current_session_id = "sess-daemon" + + async def run_stream(self, message: str, *, enable_thinking: bool = False): + self.active += 1 + self.max_active = max(self.max_active, self.active) + try: + yield StreamEvent(type="chunk", content=f"start:{message}") + await asyncio.sleep(0.05) + yield StreamEvent(type="final", content=f"done:{message}") + finally: + self.active -= 1 + + class FakeContext: + def __init__(self) -> None: + self.engine = SlowEngine() + + def reload_runtime_config_if_changed(self) -> bool: + return False + + settings = make_settings(str(tmp_path)) + service = RuntimeLeapService(settings, mock_host=True) + context = FakeContext() + service._ctx = context + + async def collect(message: str) -> list: + return [event async for event in service.engine_chat(message)] + + first, second = await asyncio.gather(collect("one"), collect("two")) + status = await service.status() + + assert [event.content for event in first] == ["start:one", "done:one"] + assert [event.content for event in second] == ["start:two", "done:two"] + assert first[0].metadata["context_used"] == 2_048 + assert first[0].metadata["session_id"] == "sess-daemon" + assert second[0].metadata["context_used"] == 2_048 + assert status["context_used"] == 2_048 + assert context.engine.max_active == 1 + + +@pytest.mark.asyncio +async def test_ensure_daemon_client_does_not_spawn_when_daemon_unhealthy( + tmp_path, + monkeypatch, +) -> None: + from conftest import make_settings + import leapflow.daemon.client as client_module + + settings = make_settings(str(tmp_path)) + unhealthy = DaemonInfo( + pid=4321, + sock_path=settings.profile_dir / "run" / "leapd.sock", + start_time=1.0, + is_running=True, + is_healthy=False, + ) + + monkeypatch.setattr(client_module.DaemonInfo, "discover", lambda run_dir: unhealthy) + + def fail_spawn(*args, **kwargs): + raise AssertionError("spawn_daemon must not be called for a running unhealthy daemon") + + monkeypatch.setattr(client_module, "spawn_daemon", fail_spawn) + + with pytest.raises(DaemonUnavailableError, match="running but unhealthy"): + await ensure_daemon_client(settings) + + +def test_daemon_runtime_status_prints_diagnostics(capsys) -> None: + from leapflow.cli.commands.daemon import _print_runtime_status + + _print_runtime_status({ + "profile": "default", + "active_clients": 1, + "volatile": False, + "model": "qwen3.7-plus", + "context_used": 256, + "llm_context_length": 256_000, + "session_id": "sess-1", + "runtime_version": "0.0.test", + "runtime_source": "/repo/src/leapflow/__init__.py", + "runtime_executable": "/venv/bin/python", + "config_path": "/home/.leapflow/.env", + "project_env_path": "/repo/.env", + "db_path": "/home/.leapflow/db/leap.duckdb", + }) + + output = capsys.readouterr().out + assert "runtime: profile=default clients=1 volatile=False" in output + assert "model: qwen3.7-plus context=256/256000" in output + assert "version: 0.0.test" in output + assert "source: /repo/src/leapflow/__init__.py" in output + assert "python: /venv/bin/python" in output + + +def test_daemon_restart_stops_waits_and_starts(monkeypatch, tmp_path) -> None: + from leapflow.cli.commands import daemon as daemon_module + + calls: list[str] = [] + + class Settings: + profile_dir = tmp_path + + monkeypatch.setattr(daemon_module, "_stop", lambda run_dir: calls.append("stop") or 0) + monkeypatch.setattr(daemon_module, "_wait_stopped", lambda run_dir: calls.append("wait") or True) + monkeypatch.setattr(daemon_module, "_start", lambda settings, mock_host: calls.append("start") or 0) + + assert daemon_module._restart(Settings(), mock_host=True) == 0 + assert calls == ["stop", "wait", "start"] + + +def test_stream_chunk_notification_preserves_event_shape() -> None: + notification = StreamChunk( + request_id="req1", + content="payload", + event_type="thinking", + metadata={"session_id": "s1"}, + ).to_notification() + + assert notification.method == "stream.chunk" + assert notification.params == { + "id": "req1", + "content": "payload", + "done": False, + "event_type": "thinking", + "metadata": {"session_id": "s1"}, + } diff --git a/tests/test_memory_and_storage.py b/tests/test_memory_and_storage.py index b69dd62..cfbc1eb 100644 --- a/tests/test_memory_and_storage.py +++ b/tests/test_memory_and_storage.py @@ -4,8 +4,10 @@ import asyncio import time +from pathlib import Path import pytest +import duckdb from leapflow.domain.trajectory import ( ActionType, @@ -20,6 +22,60 @@ from leapflow.platform.event_bus import EventBus from leapflow.platform.protocol import EventTypes from leapflow.storage.skill_library import StoredSkill +from leapflow.storage.connection import LocalConnectionHolder +from leapflow.storage.duckdb_connect import DatabaseLockedError + + +# ── Storage lock fallback ─────────────────────────────────────────── + + +def test_connection_holder_uses_volatile_duckdb_when_primary_is_locked( + tmp_path, + monkeypatch, +) -> None: + import leapflow.storage.connection as connection_module + + primary_path = tmp_path / "leap.duckdb" + original_connect = connection_module._lock_aware_connect + + def flaky_connect(db_path: Path): + if Path(db_path) == primary_path: + raise DatabaseLockedError(primary_path, RuntimeError("locked")) + return original_connect(db_path) + + monkeypatch.setattr(connection_module, "_lock_aware_connect", flaky_connect) + + holder = LocalConnectionHolder(primary_path, volatile_on_lock=True) + conn = holder.connection + volatile_path = holder.db_path + + try: + assert holder.is_volatile is True + assert holder.locked_error is not None + assert volatile_path != primary_path + assert volatile_path.name == "leap.duckdb" + assert conn.execute("SELECT 1").fetchone() == (1,) + assert volatile_path.exists() + finally: + holder.close() + + assert volatile_path.exists() is False + + +def test_write_buffer_drops_permanent_failures(tmp_path) -> None: + from leapflow.storage.write_buffer import WriteBuffer + + db_path = tmp_path / "buffer.duckdb" + conn = duckdb.connect(str(db_path)) + try: + conn.execute("CREATE TABLE items(id INTEGER PRIMARY KEY)") + buffer = WriteBuffer(conn, max_count=10) + buffer.append("bad-sql", "INSERT INTO missing_table VALUES (?)", [1]) + + assert buffer.flush() == 0 + assert buffer.pending == 0 + finally: + conn.close() # ── Working memory ───────────────────────────────────────────────── @@ -288,7 +344,7 @@ def test_skill_library_crud(skill_library) -> None: def test_memory_tiers_integration(tmp_db) -> None: - lt = SemanticMemoryProvider(db_path=tmp_db) + lt = SemanticMemoryProvider(source=tmp_db) lt._ensure_connection() try: diff --git a/tests/test_safety_and_policy.py b/tests/test_safety_and_policy.py index 33ea2f2..81debf8 100644 --- a/tests/test_safety_and_policy.py +++ b/tests/test_safety_and_policy.py @@ -137,3 +137,119 @@ async def summarize(execution, perception, **params): assert result["ok"] is True assert '"count": 3' in result["payload"] assert '"items": ["a", "b", "c"]' in result["payload"] + + +# ═══════════════════════════════════════════════════════════════════ +# Gateway safety boundaries +# ═══════════════════════════════════════════════════════════════════ + + +def test_session_key_normalizes_non_string_fields() -> None: + from leapflow.gateway.session_router import SessionKey + + key = SessionKey( + profile="default", + platform="telegram", + chat_type="group", + chat_id=123456, + thread_id=789, + user_id=42, + ) + + assert key.chat_id == "123456" + assert key.thread_id == "789" + assert key.user_id == "42" + assert str(key) == "default:telegram:group:123456:789:42" + + +def test_session_key_rejects_unsafe_serialized_fields() -> None: + from leapflow.gateway.session_router import SessionKey + + with pytest.raises(ValueError, match="Unsafe"): + SessionKey( + profile="default", + platform="telegram", + chat_type="group", + chat_id="safe", + thread_id="../escape", + ) + + +def test_gateway_adapter_import_error_preserves_internal_imports(tmp_path, monkeypatch) -> None: + import sys + + from leapflow.gateway.manifest import AdapterSpec, PlatformManifest + from leapflow.gateway.server import GatewayServer + + adapter_module = tmp_path / "adapter_mod.py" + adapter_module.write_text("import missing_subdependency_for_test\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + sys.modules.pop("adapter_mod", None) + + manifest = PlatformManifest( + platform_id="demo", + display_name="Demo", + adapter=AdapterSpec( + module="adapter_mod", + class_name="Adapter", + dependencies=("demo-sdk",), + ), + ) + + with pytest.raises(ModuleNotFoundError, match="missing_subdependency_for_test"): + GatewayServer._instantiate_adapter(manifest, {}, {}) + + +def test_gateway_adapter_missing_module_has_install_hint() -> None: + from leapflow.gateway.manifest import AdapterSpec, PlatformManifest + from leapflow.gateway.server import GatewayServer + + manifest = PlatformManifest( + platform_id="demo", + display_name="Demo", + adapter=AdapterSpec( + module="missing_adapter_module_for_test", + class_name="Adapter", + dependencies=("demo-sdk",), + ), + ) + + with pytest.raises(ImportError, match="Install dependencies: pip install demo-sdk"): + GatewayServer._instantiate_adapter(manifest, {}, {}) + + +@pytest.mark.asyncio +async def test_gateway_start_skips_failed_auto_connect_platform(tmp_path, monkeypatch) -> None: + from leapflow.gateway.config_store import GatewayConfig, PlatformConfig + from leapflow.gateway.manifest import PlatformManifest + from leapflow.gateway.server import GatewayServer + + class FakeConfigStore: + def load(self) -> GatewayConfig: + return GatewayConfig( + platforms={ + "bad": PlatformConfig(enabled=True), + "good": PlatformConfig(enabled=True), + }, + auto_connect=["bad", "good"], + ) + + def load_platform_credentials(self, platform_id, manifest): + if platform_id == "bad": + raise RuntimeError("corrupt credentials") + return {"token": "ok"} + + server = GatewayServer(tmp_path) + server._manifests = { + "bad": PlatformManifest(platform_id="bad", display_name="Bad"), + "good": PlatformManifest(platform_id="good", display_name="Good"), + } + server._config_store = FakeConfigStore() + monkeypatch.setattr(server, "discover_manifests", lambda: server._manifests) + + async def connect_platform(platform_id, credentials, options=None, *, is_reconnect=False): + return {"ok": platform_id == "good"} + + monkeypatch.setattr(server, "connect_platform", connect_platform) + + assert await server.start() == 1 diff --git a/tests/test_tui_command_queue.py b/tests/test_tui_command_queue.py new file mode 100644 index 0000000..4adb809 --- /dev/null +++ b/tests/test_tui_command_queue.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import asyncio +from contextlib import suppress + +import pytest + +from leapflow.cli.tui_app.app import LeapApp +from leapflow.cli.tui_app.command import TuiCommand, TuiCommandStatus +from leapflow.cli.tui_app.theme import _LIGHT, resolve_theme + + +class _FakeConsole: + def __init__(self) -> None: + self.cards: list[TuiCommand] = [] + self.errors: list[str] = [] + self.systems: list[str] = [] + + def command_card(self, command: TuiCommand) -> None: + self.cards.append(command) + + def error(self, message: str) -> None: + self.errors.append(message) + + def system(self, message: str) -> None: + self.systems.append(message) + + +class _FakeStatus: + def __init__(self) -> None: + self.counts: list[tuple[int, int]] = [] + + def __call__(self) -> list[tuple[str, str]]: + return [] + + def update_task_counts(self, *, running: int, queued: int) -> None: + self.counts.append((running, queued)) + + +async def _wait_for(condition, *, timeout: float = 1.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if condition(): + return + await asyncio.sleep(0.01) + raise AssertionError("condition was not met before timeout") + + +def _make_app(on_input=None) -> tuple[LeapApp, _FakeConsole, _FakeStatus]: + console = _FakeConsole() + status = _FakeStatus() + app = LeapApp( + console=console, + theme=resolve_theme(_LIGHT, terminal_bg="#FFFFFF"), + status=status, + on_input=on_input, + ) + return app, console, status + + +def test_submit_text_assigns_ids_and_keeps_input_editable() -> None: + app, console, status = _make_app() + + first = app.submit_text("first command") + second = app.submit_text("second command") + + assert first.id == 1 + assert second.id == 2 + assert app._pending_input.qsize() == 2 + assert status.counts[-1] == (0, 2) + assert [card.status for card in console.cards] == [TuiCommandStatus.QUEUED] + assert app._input_area.buffer.read_only() is False + + +def test_submit_text_rejects_empty_commands() -> None: + app, console, status = _make_app() + + with pytest.raises(ValueError): + app.submit_text(" \n ") + + assert app._pending_input.qsize() == 0 + assert console.cards == [] + assert status.counts == [] + + +@pytest.mark.asyncio +async def test_process_loop_marks_failed_commands_and_recovers_counts() -> None: + async def on_input(text: str) -> None: + raise RuntimeError(f"boom from {text}") + + app, console, status = _make_app(on_input=on_input) + app.submit_text("failing command") + + worker = asyncio.create_task(app._process_loop()) + try: + await _wait_for( + lambda: len(console.cards) == 2 + and console.cards[-1].status is TuiCommandStatus.FAILED + ) + finally: + app._should_exit = True + worker.cancel() + with suppress(asyncio.CancelledError): + await worker + + assert [(card.id, card.status) for card in console.cards] == [ + (1, TuiCommandStatus.RUNNING), + (1, TuiCommandStatus.FAILED), + ] + assert "RuntimeError: boom from failing command" in console.cards[-1].error + assert console.errors == ["boom from failing command"] + assert status.counts[-1] == (0, 0) + + +def test_failed_command_error_is_single_line_and_truncated() -> None: + command = TuiCommand.create(command_id=1, text="demo") + failed = command.mark_failed("line1\n" + "x" * 400) + + assert "\n" not in failed.error + assert len(failed.error) == 240 + assert failed.error.endswith("…") + + +class _FakeBuffer: + def __init__(self) -> None: + self.text = "" + + def insert_text(self, text: str) -> None: + self.text += text + + +def test_large_paste_is_compacted_but_submits_full_text() -> None: + app, _console, _status = _make_app() + buffer = _FakeBuffer() + pasted = "line\n" * 2_000 + + app._insert_paste_text(buffer, pasted) + + assert pasted not in buffer.text + assert "pasted block #1" in buffer.text + assert "full text will be submitted" in buffer.text + assert buffer.text.isascii() + command = app.submit_text(f"Please summarize:\n{buffer.text}") + + assert command.text == f"Please summarize:\n{pasted}".strip() + assert app._paste_blocks == {} + + +@pytest.mark.asyncio +async def test_buffer_insert_compacts_large_english_paste() -> None: + app, _console, _status = _make_app() + pasted = "long english line\n" * 400 + + app._input_area.buffer.insert_text(pasted) + + visible = app._input_area.buffer.text + assert pasted not in visible + assert visible.startswith("[pasted block #1:") + assert "full text will be submitted" in visible + assert len(visible) < 120 + assert app.submit_text(visible).text == pasted.strip() + + +@pytest.mark.asyncio +async def test_buffer_insert_compacts_large_chinese_paste_with_ascii_marker() -> None: + app, _console, _status = _make_app() + pasted = "这是一段用于验证中文大段粘贴不会直接渲染的内容。\n" * 300 + + app._input_area.buffer.insert_text(pasted) + + visible = app._input_area.buffer.text + assert pasted not in visible + assert "这是一段" not in visible + assert visible.startswith("[pasted block #1:") + assert visible.isascii() + assert len(visible) < 120 + assert app.submit_text(visible).text == pasted.strip() + + +def test_small_paste_stays_inline() -> None: + app, _console, _status = _make_app() + buffer = _FakeBuffer() + pasted = "short pasted text" + + app._insert_paste_text(buffer, pasted) + + assert buffer.text == pasted + assert app._paste_blocks == {} + + +def test_submit_clears_stale_compacted_paste_when_marker_removed() -> None: + app, _console, _status = _make_app() + buffer = _FakeBuffer() + app._insert_paste_text(buffer, "line\n" * 2_000) + + command = app.submit_text("marker was deleted by user") + + assert command.text == "marker was deleted by user" + assert app._paste_blocks == {} + + +@pytest.mark.asyncio +async def test_process_loop_runs_submitted_commands_serially() -> None: + processed: list[str] = [] + + async def on_input(text: str) -> None: + processed.append(text) + await asyncio.sleep(0.01) + + app, console, status = _make_app(on_input=on_input) + app.submit_text("first command") + app.submit_text("second command") + + worker = asyncio.create_task(app._process_loop()) + try: + await _wait_for( + lambda: processed == ["first command", "second command"] + and len(console.cards) == 5 + ) + finally: + app._should_exit = True + worker.cancel() + with suppress(asyncio.CancelledError): + await worker + + rendered = [(card.id, card.status) for card in console.cards] + assert rendered == [ + (2, TuiCommandStatus.QUEUED), + (1, TuiCommandStatus.RUNNING), + (1, TuiCommandStatus.DONE), + (2, TuiCommandStatus.RUNNING), + (2, TuiCommandStatus.DONE), + ] + assert status.counts[-1] == (0, 0) diff --git a/tests/test_tui_session_summary.py b/tests/test_tui_session_summary.py new file mode 100644 index 0000000..74fe5e3 --- /dev/null +++ b/tests/test_tui_session_summary.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from leapflow.cli.tui_app.session_summary import ( + SessionExitStats, + build_exit_summary_lines, + format_duration, + summarize_messages, +) +from leapflow.cli.tui_app.stream import StreamRenderer + + +@dataclass(frozen=True) +class _Message: + role: str + tool_name: str | None = None + tool_calls_json: str | None = None + + +class _Console: + def __init__(self) -> None: + self.markdown_calls: list[str] = [] + self.thinking_calls: list[str] = [] + self.labels: list[tuple[float, int]] = [] + self.lines = 0 + + def markdown(self, text: str) -> None: + self.markdown_calls.append(text) + + def thinking(self, text: str) -> None: + self.thinking_calls.append(text) + + def response_label(self, elapsed_s: float, *, tool_count: int = 0) -> None: + self.labels.append((elapsed_s, tool_count)) + + def newline(self) -> None: + self.lines += 1 + + def print(self, *args, **kwargs) -> None: + return None + + +def test_format_duration_compacts_common_ranges() -> None: + assert format_duration(11.9) == "11s" + assert format_duration(61) == "1m 1s" + assert format_duration(3661) == "1h 1m 1s" + + +def test_build_exit_summary_lines_matches_resume_shape() -> None: + lines = build_exit_summary_lines( + session_id="abc123", + duration_s=11, + message_count=2, + user_messages=1, + tool_calls=0, + ) + + assert lines == [ + "Resume this session with:", + " leap --resume abc123", + "", + "Session: abc123", + "Duration: 11s", + "Messages: 2 (1 user, 0 tool calls)", + ] + + +def test_empty_session_summary_degrades_to_goodbye() -> None: + assert build_exit_summary_lines( + session_id="", + duration_s=0, + message_count=0, + user_messages=0, + tool_calls=0, + ) == ["Goodbye!"] + + +def test_volatile_session_summary_does_not_offer_resume() -> None: + lines = build_exit_summary_lines( + session_id="volatile123", + duration_s=5, + message_count=2, + user_messages=1, + tool_calls=0, + resumable=False, + ) + + assert lines == [ + "Session not saved:", + " This window used volatile storage because the primary database was locked.", + "", + "Session: volatile123", + "Duration: 5s", + "Messages: 2 (1 user, 0 tool calls)", + ] + + +def test_summarize_messages_counts_tool_call_shapes() -> None: + total, users, tools = summarize_messages([ + _Message("user"), + _Message("assistant", tool_calls_json="[]"), + _Message("tool", tool_name="shell"), + ]) + + assert total == 3 + assert users == 1 + assert tools == 2 + + +def test_session_exit_stats_tracks_fallback_counts() -> None: + stats = SessionExitStats() + stats.record_user_message() + stats.record_assistant_message() + stats.record_tool_calls(2) + stats.record_tool_calls(-1) + + assert stats.message_count == 2 + assert stats.user_messages == 1 + assert stats.tool_calls == 2 + assert stats.duration_s >= 0 + + +def test_stream_renderer_exposes_output_without_private_access() -> None: + renderer = StreamRenderer(_Console()) + renderer.start() + assert renderer.has_output is False + + renderer.feed("hello") + assert renderer.has_output is True + + +def test_global_resume_routes_to_interactive(monkeypatch) -> None: + from leapflow.cli import cli + + captured = {} + + async def fake_daemon_main(args): + captured["command"] = args.command + captured["resume"] = args.resume + return 0 + + monkeypatch.setattr(cli, "_async_daemon_main", fake_daemon_main) + + assert cli.main(["--resume", "abc123"]) == 0 + assert captured == {"command": "interactive", "resume": "abc123"} diff --git a/tests/test_tui_theme.py b/tests/test_tui_theme.py new file mode 100644 index 0000000..4cfc448 --- /dev/null +++ b/tests/test_tui_theme.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from os import terminal_size + +from prompt_toolkit.styles import Style as PTStyle + +from leapflow.cli.banner import display_rich_banner +from leapflow.cli.tui_app.app import LeapApp +from leapflow.cli.tui_app.status import StatusBar +from leapflow.cli.tui_app.theme import ( + _DARK, + _LIGHT, + contrast_ratio, + detect_light_mode, + detect_theme, + parse_hex_color, + relative_luminance, + resolve_theme, +) + + +def _last_hex(style: str) -> str: + for part in reversed(style.split()): + if part.startswith("#"): + return part + raise AssertionError(f"No hex color found in style: {style}") + + +def _style_for(theme): + return PTStyle.from_dict({ + "input-area": f"bg:{theme.input_bg} {theme.input_text} bold", + "input-area.disabled": f"bg:{theme.input_bg} {theme.input_disabled_text}", + "prompt": theme.prompt_char, + "prompt.working": theme.accent_dim, + "prompt.recording": theme.recording, + "prompt.paused": theme.prompt_paused, + "prompt.executing": theme.executing, + "status-bar": f"bg:{theme.toolbar_bg} {theme.toolbar_fg}", + "status-bar.strong": f"bg:{theme.toolbar_bg} bold {theme.accent}", + "status-bar.dim": f"bg:{theme.toolbar_bg} {theme.text_muted}", + "status-bar.good": f"bg:{theme.toolbar_bg} {theme.success}", + "status-bar.warn": f"bg:{theme.toolbar_bg} {theme.warning}", + "status-bar.bad": f"bg:{theme.toolbar_bg} {theme.error}", + "hint": theme.text_dim, + "auto-suggest": theme.auto_suggest, + "placeholder": theme.input_placeholder, + "selection": f"bg:{theme.input_selection_bg} {theme.input_selection_fg}", + }) + + +def test_hex_parsing_and_contrast_math() -> None: + assert parse_hex_color("#FFFFFF") == (255, 255, 255) + assert relative_luminance("#000000") == 0.0 + assert contrast_ratio("#FFFFFF", "#000000") == 21.0 + + +def test_dark_background_resolves_readable_input_text() -> None: + theme = resolve_theme(_LIGHT, terminal_bg="#0B1F24") + + assert theme.input_text in {"#FFFFFF", "#F8FAFC", "#E5E7EB", "#D1D5DB"} + assert contrast_ratio(theme.input_text, theme.input_bg) >= 7.0 + assert contrast_ratio(theme.auto_suggest, theme.input_bg) >= 4.5 + + +def test_light_background_resolves_readable_input_text() -> None: + theme = resolve_theme(_DARK, terminal_bg="#F8FAFC") + + assert theme.input_text in {"#111827", "#1A1A1A", "#000000", "#374151"} + assert contrast_ratio(theme.input_text, theme.input_bg) >= 7.0 + assert contrast_ratio(theme.input_placeholder, theme.input_bg) >= 4.5 + assert contrast_ratio(theme.border, theme.input_bg) >= 4.5 + assert contrast_ratio(theme.input_border, theme.input_bg) >= 4.5 + assert contrast_ratio(theme.input_focus_border, theme.input_bg) >= 5.0 + assert contrast_ratio(theme.toolbar_fg, theme.toolbar_bg) >= 4.5 + assert contrast_ratio(_last_hex(theme.panel_title), theme.input_bg) >= 5.0 + + +def test_light_theme_uses_terminal_background_for_tui_surfaces() -> None: + theme = detect_theme( + {"LEAPFLOW_TUI_THEME": "light", "LEAPFLOW_TUI_BG": "#FFFFFF"}, + is_tty=True, + ) + + assert theme.input_bg == "#FFFFFF" + assert theme.toolbar_bg == "#FFFFFF" + assert contrast_ratio(theme.input_text, theme.input_bg) >= 7.0 + assert contrast_ratio(theme.toolbar_fg, theme.toolbar_bg) >= 4.5 + + +def test_theme_and_background_overrides_are_deterministic() -> None: + env = {"LEAPFLOW_TUI_THEME": "light", "LEAPFLOW_TUI_BG": "#102A2E"} + theme = detect_theme(env, is_tty=True) + + assert theme.name == "light" + assert theme.terminal_bg == "#102A2E" + assert contrast_ratio(theme.input_text, theme.input_bg) >= 7.0 + + +def test_colorfgbg_background_detection() -> None: + assert detect_light_mode({"COLORFGBG": "0;15"}) is True + assert detect_light_mode({"COLORFGBG": "15;0"}) is False + assert detect_light_mode({"ITERM_PROFILE": "Light"}) is True + assert detect_light_mode({"ITERM_PROFILE": "Dark"}) is False + + +def test_terminal_program_alone_does_not_force_light_theme() -> None: + theme = detect_theme({"TERM_PROGRAM": "Apple_Terminal"}, is_tty=True) + + assert theme.name == "dark" + assert theme.input_bg == "#0B1F24" + assert contrast_ratio(theme.input_text, theme.input_bg) >= 7.0 + + +def test_prompt_toolkit_accepts_base_and_resolved_theme_styles() -> None: + _style_for(_DARK) + _style_for(_LIGHT) + _style_for(resolve_theme(_DARK, terminal_bg="#102A2E")) + _style_for(resolve_theme(_LIGHT, terminal_bg="#F8FAFC")) + + +def test_leap_app_style_builder_accepts_resolved_theme(tmp_path) -> None: + theme = resolve_theme(_DARK, terminal_bg="#102A2E") + app = LeapApp( + console=None, + theme=theme, + status=lambda: [], + data_dir=tmp_path, + on_input=None, + ) + + assert app._build_style() is not None + assert app._input_area.window.height.max == 4 + assert app._input_area.window.dont_extend_height() is True + + +def test_rich_banner_accepts_resolved_theme(capsys) -> None: + theme = resolve_theme(_LIGHT, terminal_bg="#FFFFFF") + + display_rich_banner( + model="provider/qwen3-plus", + cwd="/tmp/work", + platform_online=False, + tool_defs=[], + skills=[], + show_welcome=False, + theme=theme, + ) + + output = capsys.readouterr().out + assert "LeapFlow" in output + assert "#FFF8DC" not in output + + +def test_status_bar_compacts_on_narrow_terminal(monkeypatch) -> None: + monkeypatch.setattr( + "leapflow.cli.tui_app.status.shutil.get_terminal_size", + lambda: terminal_size((48, 24)), + ) + status = StatusBar(resolve_theme(_LIGHT, terminal_bg="#FFFFFF")) + status.update( + model_name="very-long-model-name-that-would-overflow", + context_used=50_000, + context_max=100_000, + ) + + rendered = "".join(text for _, text in status()) + assert "very-long-model" not in rendered + assert "50%" in rendered + assert "[" not in rendered + + +def test_status_bar_shows_fractional_k_for_small_context_usage(monkeypatch) -> None: + monkeypatch.setattr( + "leapflow.cli.tui_app.status.shutil.get_terminal_size", + lambda: terminal_size((120, 24)), + ) + status = StatusBar(resolve_theme(_LIGHT, terminal_bg="#FFFFFF")) + status.update( + model_name="qwen3.7-plus", + context_used=240, + context_max=256_000, + ) + + rendered = "".join(text for _, text in status()) + assert "0.2K/256K" in rendered + assert "0.1%" in rendered + assert "[█░░░░░░░░░]" in rendered \ No newline at end of file diff --git a/tests/test_world_model.py b/tests/test_world_model.py index 9c4ef1d..8d13123 100644 --- a/tests/test_world_model.py +++ b/tests/test_world_model.py @@ -28,7 +28,7 @@ def _open_store(tmp_path) -> tuple[SemanticMemoryProvider, ExperienceStore]: - lt = SemanticMemoryProvider(db_path=tmp_path / "world_model.duckdb") + lt = SemanticMemoryProvider(source=tmp_path / "world_model.duckdb") # _ensure_connection() auto-initializes on first legacy method call lt._ensure_connection() return lt, ExperienceStore(lt)