From 4537c48a84541f1278ddfdb69cb9b6f4672baece Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 00:40:29 +0530 Subject: [PATCH 1/8] feat(agent): move, resize and snap windows window_control gains move (x, y), resize (width, height) and snap (left, right, top, bottom, the four corners, or center). Coordinates are relative to the usable area of the window's screen (without the taskbar or menu bar), and placements stay inside it with a 200 px minimum size, so a window can't end up off screen. - Windows uses SetWindowPos with the monitor's work area. - macOS uses System Events, below the menu bar. - Linux uses wmctrl -e with the desktop's work area. --- src/locus/tools/desktop/window_control.py | 260 +++++++++++++++++++++- tests/tools/test_desktop_tools.py | 149 +++++++++++++ 2 files changed, 399 insertions(+), 10 deletions(-) diff --git a/src/locus/tools/desktop/window_control.py b/src/locus/tools/desktop/window_control.py index d36404d..089ec8a 100644 --- a/src/locus/tools/desktop/window_control.py +++ b/src/locus/tools/desktop/window_control.py @@ -1,4 +1,4 @@ -"""List, focus, minimize, maximize and restore windows. +"""List, focus, minimize, maximize, restore, move, resize and snap windows. Windows uses the window API through ctypes, macOS uses System Events (Locus needs Accessibility permission), and Linux uses wmctrl, which works on X11 @@ -15,15 +15,104 @@ from locus.tools._stubs import BaseTool, ToolSpec from locus.tools.desktop import _os -ACTIONS = ("list", "focus", "minimize", "maximize", "restore") +ACTIONS = ( + "list", + "focus", + "minimize", + "maximize", + "restore", + "move", + "resize", + "snap", +) +PLACE_ACTIONS = ("move", "resize", "snap") +SNAP_POSITIONS = ( + "left", + "right", + "top", + "bottom", + "top_left", + "top_right", + "bottom_left", + "bottom_right", + "center", +) +#: Smallest window placement Locus will make, so a window can't vanish. +MIN_SIZE = 200 _PAST = { "focus": "Focused", "minimize": "Minimized", "maximize": "Maximized", "restore": "Restored", + "move": "Moved", + "resize": "Resized", + "snap": "Snapped", } +#: (left, top, width, height) in screen pixels. +Rect = Tuple[int, int, int, int] + + +def placement_rect(action: str, placement: dict, current: Rect, work: Rect) -> Rect: + """Where a move, resize or snap puts a window. + + ``work`` is the usable area of the window's screen (without the taskbar + or menu bar). Move coordinates are relative to it, and the result always + stays inside it. + """ + wl, wt, ww, wh = work + cx, cy, cw, ch = current + if action == "snap": + position = placement["position"] + half_w, half_h = ww // 2, wh // 2 + if position == "center": + w, h = ww * 2 // 3, wh * 2 // 3 + return wl + (ww - w) // 2, wt + (wh - h) // 2, w, h + x = wl + (half_w if "right" in position else 0) + y = wt + (half_h if "bottom" in position else 0) + w = ww if position in ("top", "bottom") else half_w + h = wh if position in ("left", "right") else half_h + if "right" in position: + w = ww - half_w + if "bottom" in position: + h = wh - half_h + return x, y, w, h + if action == "resize": + w = max(MIN_SIZE, min(int(placement["width"]), ww)) + h = max(MIN_SIZE, min(int(placement["height"]), wh)) + x, y = cx, cy + else: # move + w, h = min(cw, ww), min(ch, wh) + x, y = wl + int(placement["x"]), wt + int(placement["y"]) + x = max(wl, min(x, wl + ww - w)) + y = max(wt, min(y, wt + wh - h)) + return x, y, w, h + + +def _number(raw: Any) -> Union[int, None]: + try: + return int(round(float(raw))) + except (TypeError, ValueError): + return None + + +def _placement(action: str, params: dict) -> Union[dict, str]: + """The move, resize or snap arguments, or a message saying what's wrong.""" + if action == "snap": + position = params.get("position") + if position not in SNAP_POSITIONS: + return f"Give a position: {', '.join(SNAP_POSITIONS)}." + return {"position": position} + keys = ("x", "y") if action == "move" else ("width", "height") + values = {key: _number(params.get(key)) for key in keys} + if any(v is None for v in values.values()): + return f"Give {' and '.join(keys)} in pixels." + if action == "move" and any(v < 0 for v in values.values()): + return "x and y start at the screen's top-left corner; they can't be negative." + return values + + #: (handle, title): the handle is whatever the OS needs to act on the window. Window = Tuple[Any, str] @@ -87,7 +176,45 @@ def _apply(hwnd: int, action: str) -> bool: return True -def _windows_act(hwnd: int, title: str, action: str) -> Union[None, str]: +def _windows_place(hwnd: int, action: str, placement: dict) -> bool: + import ctypes + from ctypes import wintypes + + class MONITORINFO(ctypes.Structure): + _fields_ = [ + ("cbSize", wintypes.DWORD), + ("rcMonitor", wintypes.RECT), + ("rcWork", wintypes.RECT), + ("dwFlags", wintypes.DWORD), + ] + + user32 = ctypes.windll.user32 # type: ignore[attr-defined] + if user32.IsIconic(hwnd) or user32.IsZoomed(hwnd): + user32.ShowWindow(hwnd, _SW["restore"]) + rect = wintypes.RECT() + user32.GetWindowRect(hwnd, ctypes.byref(rect)) + info = MONITORINFO() + info.cbSize = ctypes.sizeof(MONITORINFO) + monitor = user32.MonitorFromWindow(hwnd, 2) # MONITOR_DEFAULTTONEAREST + user32.GetMonitorInfoW(monitor, ctypes.byref(info)) + work = info.rcWork + x, y, w, h = placement_rect( + action, + placement, + (rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top), + (work.left, work.top, work.right - work.left, work.bottom - work.top), + ) + # SWP_NOZORDER | SWP_NOACTIVATE: change only position and size. + return bool(user32.SetWindowPos(hwnd, 0, x, y, w, h, 0x0014)) + + +def _windows_act( + hwnd: int, title: str, action: str, placement: Union[dict, None] = None +) -> Union[None, str]: + if action in PLACE_ACTIONS: + if not _windows_place(hwnd, action, placement or {}): + return f"Windows didn't let Locus move {title}." + return None if not _apply(hwnd, action): return f"Windows didn't let Locus focus {title}." return None @@ -128,6 +255,36 @@ def _windows_act(hwnd: int, title: str, action: str) -> Union[None, str]: end run""" +_MAC_PLACE = """on run argv + set appName to item 1 of argv + set winName to item 2 of argv + set x to (item 3 of argv) as integer + set y to (item 4 of argv) as integer + set w to (item 5 of argv) as integer + set h to (item 6 of argv) as integer + tell application "System Events" + tell process appName + set win to (first window whose name is winName) + set value of attribute "AXMinimized" of win to false + if x >= 0 and y >= 0 then set position of win to {x, y} + if w > 0 and h > 0 then set size of win to {w, h} + end tell + end tell +end run""" +_MAC_SCREEN = 'tell application "Finder" to get bounds of window of desktop' +_MAC_MENU_BAR = 25 + + +def _mac_work_area() -> Union[Rect, None]: + done = _os.run(["osascript", "-e", _MAC_SCREEN]) + try: + left, top, right, bottom = (int(v) for v in done.stdout.split(",")) + except ValueError: + return None + top = max(top, _MAC_MENU_BAR) + return left, top, right - left, bottom - top + + def _mac_windows() -> Union[List[Window], ToolResult]: done = _os.run(["osascript", "-e", _MAC_LIST]) if not done.ok: @@ -144,8 +301,30 @@ def _mac_windows() -> Union[List[Window], ToolResult]: return windows -def _mac_act(handle: Tuple[str, str], title: str, action: str) -> Union[None, str]: +def _mac_act( + handle: Tuple[str, str], + title: str, + action: str, + placement: Union[dict, None] = None, +) -> Union[None, str]: app, name = handle + if action in PLACE_ACTIONS: + placement = placement or {} + if action == "resize": + x, y = -1, -1 + w, h = (max(MIN_SIZE, placement[k]) for k in ("width", "height")) + else: + work = _mac_work_area() + if work is None: + return "Couldn't read the screen size." + if action == "move": + x, y = work[0] + placement["x"], work[1] + placement["y"] + w = h = -1 + else: + x, y, w, h = placement_rect("snap", placement, work, work) + args = [str(v) for v in (x, y, w, h)] + done = _os.run(["osascript", "-e", _MAC_PLACE, app, name, *args]) + return None if done.ok else f"Couldn't move {title}: {done.error()}" done = _os.run(["osascript", "-e", _MAC_ACT, app, name, action]) return None if done.ok else f"Couldn't change {title}: {done.error()}" @@ -175,7 +354,45 @@ def _linux_windows() -> Union[List[Window], ToolResult]: return windows -def _linux_act(window_id: str, title: str, action: str) -> Union[None, str]: +def _linux_work_area() -> Union[Rect, None]: + done = _os.run(["wmctrl", "-d"]) + for line in done.stdout.splitlines(): + parts = line.split() + if len(parts) > 1 and parts[1] == "*" and "WA:" in parts: + i = parts.index("WA:") + try: + x, y = (int(v) for v in parts[i + 1].split(",")) + w, h = (int(v) for v in parts[i + 2].split("x")) + except (IndexError, ValueError): + return None + return x, y, w, h + return None + + +def _linux_act( + window_id: str, title: str, action: str, placement: Union[dict, None] = None +) -> Union[None, str]: + if action in PLACE_ACTIONS: + placement = placement or {} + if action == "resize": + geometry = ( + -1, + -1, + *(max(MIN_SIZE, placement[k]) for k in ("width", "height")), + ) + else: + work = _linux_work_area() + if work is None: + return "Couldn't read the screen's work area from wmctrl." + if action == "move": + geometry = (work[0] + placement["x"], work[1] + placement["y"], -1, -1) + else: + geometry = placement_rect("snap", placement, work, work) + _os.run(["wmctrl", "-i", "-r", window_id, "-b", _WMCTRL_STATES["restore"]]) + done = _os.run( + ["wmctrl", "-i", "-r", window_id, "-e", "0," + ",".join(map(str, geometry))] + ) + return None if done.ok else f"Couldn't move {title}: {done.error()}" if action in _WMCTRL_STATES: done = _os.run(["wmctrl", "-i", "-r", window_id, "-b", _WMCTRL_STATES[action]]) if not done.ok: @@ -209,9 +426,13 @@ def spec(self) -> ToolSpec: name="window_control", description=( "Work with open windows. 'list' shows window titles. 'focus'," - " 'minimize', 'maximize' and 'restore' act on the first window" - " whose title contains 'title' (not case-sensitive). To close" - " an app, use app_control instead." + " 'minimize', 'maximize', 'restore', 'move', 'resize' and 'snap'" + " act on the first window whose title contains 'title' (not" + " case-sensitive). 'move' takes x and y and 'resize' takes width" + " and height, in pixels from the top-left of the screen's usable" + " area. 'snap' puts the window in a 'position' (left, right, top," + " bottom, top_left, top_right, bottom_left, bottom_right or" + " center). To close an app, use app_control instead." ), parameters={ "type": "object", @@ -221,6 +442,11 @@ def spec(self) -> ToolSpec: "type": "string", "description": "Part of the window's title, like 'Notepad'.", }, + "x": {"type": "number"}, + "y": {"type": "number"}, + "width": {"type": "number"}, + "height": {"type": "number"}, + "position": {"type": "string", "enum": list(SNAP_POSITIONS)}, }, "required": ["action"], }, @@ -238,6 +464,11 @@ def execute(self, **params: Any) -> ToolResult: if platform is None: return _os.unsupported("window_control", "Controlling windows") list_fn, act_fn = platform + placement = None + if action in PLACE_ACTIONS: + placement = _placement(action, params) + if isinstance(placement, str): + return _os.fail("window_control", placement) windows = list_fn() if isinstance(windows, ToolResult): @@ -256,10 +487,19 @@ def execute(self, **params: Any) -> ToolResult: "window_control", f"No open window has {wanted!r} in its title." ) handle, title = match - error = act_fn(handle, title, action) + error = ( + act_fn(handle, title, action, placement) + if placement is not None + else act_fn(handle, title, action) + ) if error: return _os.fail("window_control", error) - return _os.ok("window_control", f"{_PAST[action]} {title}.") + where = ( + f" to the {placement['position'].replace('_', ' ')}" + if action == "snap" + else "" + ) + return _os.ok("window_control", f"{_PAST[action]} {title}{where}.") __all__ = ["WindowControlTool", "list_windows"] diff --git a/tests/tools/test_desktop_tools.py b/tests/tools/test_desktop_tools.py index 8a3bec1..3dc58a7 100644 --- a/tests/tools/test_desktop_tools.py +++ b/tests/tools/test_desktop_tools.py @@ -557,3 +557,152 @@ def test_radio_failures_are_reported(monkeypatch): def test_radio_risk(action, risk): tool = SystemControlTool() assert tool_risk("system_control", tool, {"action": action}) == risk + + +# ── Moving, resizing and snapping windows ──────────────────────── + +WORK = (0, 40, 1600, 900) # a screen with a 40 px bar on top + + +@pytest.mark.parametrize( + "position, rect", + [ + ("left", (0, 40, 800, 900)), + ("right", (800, 40, 800, 900)), + ("top", (0, 40, 1600, 450)), + ("bottom", (0, 490, 1600, 450)), + ("top_left", (0, 40, 800, 450)), + ("bottom_right", (800, 490, 800, 450)), + ("center", (267, 190, 1066, 600)), + ], +) +def test_snap_rects(position, rect): + current = (10, 50, 500, 400) + assert ( + window_control.placement_rect("snap", {"position": position}, current, WORK) + == rect + ) + + +def test_snapping_an_odd_sized_screen_covers_it(): + work = (0, 0, 1365, 767) + left = window_control.placement_rect("snap", {"position": "left"}, work, work) + right = window_control.placement_rect("snap", {"position": "right"}, work, work) + assert left[2] + right[2] == 1365 + assert right[0] == left[2] + + +def test_moves_and_resizes_stay_on_screen(): + current = (100, 100, 600, 400) + place = window_control.placement_rect + assert place("move", {"x": 50, "y": 20}, current, WORK) == (50, 60, 600, 400) + # Too far right or down: pulled back so the whole window shows. + assert place("move", {"x": 5000, "y": 5000}, current, WORK) == (1000, 540, 600, 400) + assert place("resize", {"width": 800, "height": 500}, current, WORK) == ( + 100, + 100, + 800, + 500, + ) + # Bigger than the screen, or tiny: limited to the screen and MIN_SIZE. + assert place("resize", {"width": 9999, "height": 10}, current, WORK) == ( + 0, + 100, + 1600, + window_control.MIN_SIZE, + ) + + +@pytest.mark.parametrize( + "params, message", + [ + ({"action": "snap", "title": "x", "position": "middle"}, "Give a position"), + ({"action": "move", "title": "x", "x": 10}, "Give x and y"), + ({"action": "move", "title": "x", "x": -5, "y": 0}, "can't be negative"), + ({"action": "resize", "title": "x", "width": "big", "height": 4}, "Give width"), + ], +) +def test_placement_arguments_are_checked(monkeypatch, params, message): + FakeOS(monkeypatch, "windows") + result = WindowControlTool().execute(**params) + assert not result.success + assert message in result.content + + +def test_snapping_on_windows(monkeypatch): + FakeOS(monkeypatch, "windows") + placed = [] + monkeypatch.setattr( + window_control, "list_windows", lambda: [(7, "notes.txt - Notepad")] + ) + monkeypatch.setattr( + window_control, + "_windows_place", + lambda hwnd, action, placement: ( + placed.append((hwnd, action, placement)) or True + ), + ) + + result = WindowControlTool().execute( + action="snap", title="notepad", position="top_right" + ) + + assert result.content == "Snapped notes.txt - Notepad to the top right." + assert placed == [(7, "snap", {"position": "top_right"})] + + +def test_snapping_on_linux_uses_the_work_area(monkeypatch): + fake = FakeOS(monkeypatch, "linux") + outputs = { + "-l": "0x05 0 host Editor\n", + "-d": "0 * DG: 1920x1080 VP: 0,0 WA: 0,27 1920x1053 Workspace 1\n", + } + + def run(args, *, env=None, input=None, timeout=0): + fake.calls.append({"args": args}) + return _os.Completed(0, outputs.get(args[1], ""), "") + + monkeypatch.setattr(_os, "run", run) + + assert ( + WindowControlTool() + .execute(action="snap", title="editor", position="left") + .success + ) + assert fake.calls[-1]["args"] == [ + "wmctrl", + "-i", + "-r", + "0x05", + "-e", + "0,0,27,960,1053", + ] + WindowControlTool().execute(action="resize", title="editor", width=900, height=700) + assert fake.calls[-1]["args"][-1] == "0,-1,-1,900,700" + + +def test_moving_on_macos(monkeypatch): + fake = FakeOS(monkeypatch, "macos") + + def run(args, *, env=None, input=None, timeout=0): + fake.calls.append({"args": args}) + script = args[2] if len(args) > 2 else "" + if "bounds of window of desktop" in script: + return _os.Completed(0, "0, 0, 1440, 900", "") + if "repeat with p in" in script: + return _os.Completed(0, "Notes\tShopping list\n", "") + return _os.Completed(0, "", "") + + monkeypatch.setattr(_os, "run", run) + + assert ( + WindowControlTool().execute(action="move", title="shopping", x=10, y=10).success + ) + assert fake.calls[-1]["args"][-6:] == [ + "Notes", + "Shopping list", + "10", + "35", + "-1", + "-1", + ] From fef678d5856cfea723661f29dfb3df04828292a5 Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 00:42:25 +0530 Subject: [PATCH 2/8] feat(agent): control windows on Wayland window_control on Linux used wmctrl, which only works on X11. On a Wayland session it now talks to the compositor: - Sway uses swaymsg (minimize moves to the scratchpad). - Hyprland uses hyprctl (minimize uses a special workspace, since Hyprland has none). - KDE Plasma uses kdotool. Move, resize and snap use the focused screen's area. GNOME doesn't let apps control other windows, so the tool says so, and it names the tool to install when one is missing. --- src/locus/tools/desktop/wayland.py | 249 ++++++++++++++++++++++ src/locus/tools/desktop/window_control.py | 11 +- tests/tools/test_desktop_tools.py | 112 ++++++++++ 3 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 src/locus/tools/desktop/wayland.py diff --git a/src/locus/tools/desktop/wayland.py b/src/locus/tools/desktop/wayland.py new file mode 100644 index 0000000..885559c --- /dev/null +++ b/src/locus/tools/desktop/wayland.py @@ -0,0 +1,249 @@ +"""Window control on Wayland, one backend per compositor. + +Wayland has no shared window API, so Locus talks to the compositor itself: +``swaymsg`` on Sway, ``hyprctl`` on Hyprland and ``kdotool`` on KDE Plasma. +GNOME doesn't let other apps control windows, so it isn't supported. +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from locus.core.types import ToolResult +from locus.tools.desktop import _os + +Window = Tuple[Any, str] +Rect = Tuple[int, int, int, int] +Backend = Tuple[Callable[[], Any], Callable[..., Optional[str]]] + + +def is_wayland() -> bool: + return os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland" or bool( + os.environ.get("WAYLAND_DISPLAY") + ) + + +def _placement_rect(action: str, placement: dict, current: Rect, work: Rect) -> Rect: + from locus.tools.desktop.window_control import placement_rect + + return placement_rect(action, placement, current, work) + + +def _json(args: List[str]) -> Union[Any, ToolResult]: + done = _os.run(args) + if not done.ok: + return _os.fail("window_control", f"Couldn't list windows: {done.error()}") + try: + return json.loads(done.stdout or "null") + except ValueError: + return _os.fail("window_control", f"{args[0]} gave output Locus can't read.") + + +# ── Sway ───────────────────────────────────────────────────────── + + +def _sway_nodes(tree: Any) -> List[dict]: + """Every window in a sway tree, including floating and scratchpad ones.""" + found: List[dict] = [] + stack = [tree] if isinstance(tree, dict) else [] + while stack: + node = stack.pop() + children = node.get("nodes", []) + node.get("floating_nodes", []) + if ( + not children + and node.get("type") in ("con", "floating_con") + and node.get("name") + ): + found.append(node) + stack.extend(children) + return found + + +def _sway_windows() -> Union[List[Window], ToolResult]: + tree = _json(["swaymsg", "-t", "get_tree", "-r"]) + if isinstance(tree, ToolResult): + return tree + return [(node, node["name"]) for node in _sway_nodes(tree)] + + +def _sway_output_rect() -> Optional[Rect]: + outputs = _json(["swaymsg", "-t", "get_outputs", "-r"]) + if isinstance(outputs, ToolResult) or not isinstance(outputs, list): + return None + for output in outputs: + if output.get("focused") and output.get("rect"): + r = output["rect"] + return r["x"], r["y"], r["width"], r["height"] + return None + + +def _sway_act(node: dict, title: str, action: str, placement: Optional[dict] = None): + target = f"[con_id={node['id']}]" + commands = { + "focus": f"{target} focus", + "minimize": f"{target} move scratchpad", + "maximize": f"{target} fullscreen enable", + "restore": f"{target} scratchpad show, {target} floating disable, " + f"{target} fullscreen disable, {target} focus", + } + if action in commands: + command = commands[action] + else: + rect = node.get("rect", {}) + current = ( + rect.get("x", 0), + rect.get("y", 0), + rect.get("width", 800), + rect.get("height", 600), + ) + work = _sway_output_rect() or current + x, y, w, h = _placement_rect(action, placement or {}, current, work) + command = ( + f"{target} floating enable, {target} resize set {w} px {h} px, " + f"{target} move absolute position {x} px {y} px" + ) + done = _os.run(["swaymsg", command]) + return None if done.ok else f"Couldn't change {title}: {done.error()}" + + +# ── Hyprland ───────────────────────────────────────────────────── + + +def _hypr_windows() -> Union[List[Window], ToolResult]: + clients = _json(["hyprctl", "clients", "-j"]) + if isinstance(clients, ToolResult): + return clients + return [(c, c["title"]) for c in clients or [] if c.get("title")] + + +def _hypr_monitor_rect() -> Optional[Rect]: + monitors = _json(["hyprctl", "monitors", "-j"]) + if isinstance(monitors, ToolResult) or not isinstance(monitors, list): + return None + for m in monitors: + if m.get("focused"): + scale = float(m.get("scale") or 1) or 1 + reserved = m.get("reserved") or [0, 0, 0, 0] # left, top, right, bottom + width = int(m["width"] / scale) - reserved[0] - reserved[2] + height = int(m["height"] / scale) - reserved[1] - reserved[3] + return m["x"] + reserved[0], m["y"] + reserved[1], width, height + return None + + +def _hypr_act(client: dict, title: str, action: str, placement: Optional[dict] = None): + window = f"address:{client['address']}" + if action == "focus": + batches = [f"dispatch focuswindow {window}"] + elif action == "minimize": + # Hyprland has no minimize; a special workspace hides the window. + batches = [f"dispatch movetoworkspacesilent special:minimized,{window}"] + elif action == "maximize": + batches = [f"dispatch focuswindow {window}", "dispatch fullscreen 1"] + elif action == "restore": + batches = [ + f"dispatch movetoworkspace e+0,{window}", + f"dispatch focuswindow {window}", + ] + else: + at, size = client.get("at", [0, 0]), client.get("size", [800, 600]) + current = (at[0], at[1], size[0], size[1]) + work = _hypr_monitor_rect() or current + x, y, w, h = _placement_rect(action, placement or {}, current, work) + batches = [] + if not client.get("floating"): + batches.append(f"dispatch setfloating {window}") + batches += [ + f"dispatch resizewindowpixel exact {w} {h},{window}", + f"dispatch movewindowpixel exact {x} {y},{window}", + ] + done = _os.run(["hyprctl", "--batch", " ; ".join(batches)]) + return None if done.ok else f"Couldn't change {title}: {done.error()}" + + +# ── KDE Plasma (kdotool) ───────────────────────────────────────── + + +def _kde_windows() -> Union[List[Window], ToolResult]: + done = _os.run(["kdotool", "search", "--name", "."]) + if not done.ok: + return _os.fail("window_control", f"Couldn't list windows: {done.error()}") + windows: List[Window] = [] + for window_id in done.stdout.split(): + name = _os.run(["kdotool", "getwindowname", window_id]) + if name.ok and name.stdout.strip(): + windows.append((window_id, name.stdout.strip())) + return windows + + +def _kde_act(window_id: str, title: str, action: str, placement: Optional[dict] = None): + if action in ("focus", "restore"): + steps = [["kdotool", "windowactivate", window_id]] + elif action == "minimize": + steps = [["kdotool", "windowminimize", window_id]] + elif action == "maximize": + steps = [["kdotool", "windowstate", "--add", "MAXIMIZED", window_id]] + else: + placement = placement or {} + if action == "move": + steps = [ + [ + "kdotool", + "windowmove", + window_id, + str(placement["x"]), + str(placement["y"]), + ] + ] + elif action == "resize": + steps = [ + [ + "kdotool", + "windowsize", + window_id, + str(placement["width"]), + str(placement["height"]), + ] + ] + else: + return "Snapping isn't available through kdotool; use KDE's own shortcuts." + for step in steps: + done = _os.run(step) + if not done.ok: + return f"Couldn't change {title}: {done.error()}" + return None + + +# ── Choosing one ───────────────────────────────────────────────── + +_BACKENDS: Dict[str, Tuple[str, Backend]] = { + "sway": ("swaymsg", (lambda: _sway_windows(), lambda *a: _sway_act(*a))), + "hyprland": ("hyprctl", (lambda: _hypr_windows(), lambda *a: _hypr_act(*a))), + "kde": ("kdotool", (lambda: _kde_windows(), lambda *a: _kde_act(*a))), +} + + +def backend() -> Union[Backend, ToolResult]: + """The running compositor's backend, or a result explaining what's missing.""" + if os.environ.get("SWAYSOCK"): + name = "sway" + elif os.environ.get("HYPRLAND_INSTANCE_SIGNATURE"): + name = "hyprland" + elif "kde" in os.environ.get("XDG_CURRENT_DESKTOP", "").lower(): + name = "kde" + else: + desktop = os.environ.get("XDG_CURRENT_DESKTOP") or "this desktop" + return _os.fail( + "window_control", + f"Window control on Wayland works on Sway, Hyprland and KDE Plasma," + f" not {desktop}. GNOME doesn't let apps control other windows.", + ) + command, functions = _BACKENDS[name] + if not _os.has_command(command): + hint = " (install kdotool)" if name == "kde" else "" + return _os.fail("window_control", f"{command} isn't available{hint}.") + return functions + + +__all__ = ["backend", "is_wayland"] diff --git a/src/locus/tools/desktop/window_control.py b/src/locus/tools/desktop/window_control.py index 089ec8a..2b5cda8 100644 --- a/src/locus/tools/desktop/window_control.py +++ b/src/locus/tools/desktop/window_control.py @@ -1,8 +1,8 @@ """List, focus, minimize, maximize, restore, move, resize and snap windows. Windows uses the window API through ctypes, macOS uses System Events (Locus -needs Accessibility permission), and Linux uses wmctrl, which works on X11 -but not Wayland. +needs Accessibility permission), and Linux uses wmctrl on X11 or the +compositor's own tool on Wayland (see ``wayland.py``). """ from __future__ import annotations @@ -463,6 +463,13 @@ def execute(self, **params: Any) -> ToolResult: platform = _PLATFORMS.get(_os.system()) if platform is None: return _os.unsupported("window_control", "Controlling windows") + if _os.system() == "linux": + from locus.tools.desktop import wayland + + if wayland.is_wayland(): + platform = wayland.backend() + if isinstance(platform, ToolResult): + return platform list_fn, act_fn = platform placement = None if action in PLACE_ACTIONS: diff --git a/tests/tools/test_desktop_tools.py b/tests/tools/test_desktop_tools.py index 3dc58a7..127b7e9 100644 --- a/tests/tools/test_desktop_tools.py +++ b/tests/tools/test_desktop_tools.py @@ -24,6 +24,19 @@ ) +@pytest.fixture(autouse=True) +def _no_wayland(monkeypatch): + """Linux tests mean X11 unless they set a Wayland session themselves.""" + for name in ( + "WAYLAND_DISPLAY", + "XDG_SESSION_TYPE", + "SWAYSOCK", + "HYPRLAND_INSTANCE_SIGNATURE", + "XDG_CURRENT_DESKTOP", + ): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture(autouse=True) def _isolate_reminders(tmp_path, monkeypatch): """Keep reminders out of the real ``~/.locus`` and drop leftover timers.""" @@ -706,3 +719,102 @@ def run(args, *, env=None, input=None, timeout=0): "-1", "-1", ] + + +# ── Wayland ────────────────────────────────────────────────────── + + +def _scripted_run(monkeypatch, fake, outputs): + """Answer commands by their first two words; record every call.""" + + def run(args, *, env=None, input=None, timeout=0): + fake.calls.append({"args": args}) + return _os.Completed(0, outputs.get(" ".join(args[:2]), ""), "") + + monkeypatch.setattr(_os, "run", run) + + +def test_sway_windows_are_listed_and_snapped(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-1") + monkeypatch.setenv("SWAYSOCK", "/run/sway.sock") + fake = FakeOS(monkeypatch, "linux") + tree = { + "type": "root", + "nodes": [ + { + "type": "workspace", + "nodes": [ + { + "type": "con", + "id": 12, + "name": "notes - Kate", + "rect": {"x": 0, "y": 0, "width": 900, "height": 700}, + "nodes": [], + } + ], + "floating_nodes": [], + } + ], + } + _scripted_run(monkeypatch, fake, {"swaymsg -t": json.dumps(tree)}) + monkeypatch.setattr( + "locus.tools.desktop.wayland._sway_output_rect", lambda: (0, 0, 1920, 1080) + ) + + listed = WindowControlTool().execute(action="list") + snapped = WindowControlTool().execute(action="snap", title="kate", position="left") + + assert listed.metadata["titles"] == ["notes - Kate"] + assert snapped.success + command = fake.calls[-1]["args"][1] + assert "[con_id=12] floating enable" in command + assert "resize set 960 px 1080 px" in command + assert "move absolute position 0 px 0 px" in command + + +def test_hyprland_uses_hyprctl(monkeypatch): + monkeypatch.setenv("XDG_SESSION_TYPE", "wayland") + monkeypatch.setenv("HYPRLAND_INSTANCE_SIGNATURE", "abc") + fake = FakeOS(monkeypatch, "linux") + clients = [ + {"address": "0xa1", "title": "Firefox", "at": [0, 0], "size": [800, 600]} + ] + _scripted_run(monkeypatch, fake, {"hyprctl clients": json.dumps(clients)}) + + assert WindowControlTool().execute(action="minimize", title="fire").success + assert fake.calls[-1]["args"] == [ + "hyprctl", + "--batch", + "dispatch movetoworkspacesilent special:minimized,address:0xa1", + ] + + +def test_kde_uses_kdotool(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + monkeypatch.setenv("XDG_CURRENT_DESKTOP", "KDE") + fake = FakeOS(monkeypatch, "linux") + _scripted_run( + monkeypatch, + fake, + {"kdotool search": "{aa}\n", "kdotool getwindowname": "Dolphin"}, + ) + + assert WindowControlTool().execute(action="focus", title="dolphin").success + assert fake.calls[-1]["args"] == ["kdotool", "windowactivate", "{aa}"] + + +def test_gnome_on_wayland_is_explained(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + monkeypatch.setenv("XDG_CURRENT_DESKTOP", "GNOME") + FakeOS(monkeypatch, "linux") + result = WindowControlTool().execute(action="list") + assert not result.success + assert "GNOME" in result.content + + +def test_wayland_tool_must_be_installed(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + monkeypatch.setenv("XDG_CURRENT_DESKTOP", "KDE") + FakeOS(monkeypatch, "linux") + monkeypatch.setattr(_os, "has_command", lambda name: False) + assert "install kdotool" in WindowControlTool().execute(action="list").content From 0a1074c9f8e29bcfe2f1aea1dea076a3c3d6feae Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 00:47:14 +0530 Subject: [PATCH 3/8] feat(agent): let vision models see screenshots The agent could take a screenshot but only got its file path back. Now: - screenshot takes look=true and also returns the screen as a base64 JPEG of at most 1280 px a side (Pillow is already in the server and desktop extras). - After each round of tools, the orchestrator shows the model any images the tools returned, as one user message, since tool messages can't carry images in most chat APIs. - Agent and developer mode tell the agent whether the model can see images (Ollama's vision capability), so a text-only model gets only the text. --- src/locus/agents/orchestrator.py | 32 +++++++++++++++ src/locus/server/routes.py | 16 +++++++- src/locus/tools/desktop/screenshot.py | 58 +++++++++++++++++++++++++-- tests/server/test_agent_mode.py | 37 +++++++++++++++++ tests/tools/test_desktop_tools.py | 41 +++++++++++++++++++ 5 files changed, 179 insertions(+), 5 deletions(-) diff --git a/src/locus/agents/orchestrator.py b/src/locus/agents/orchestrator.py index 45560b5..2215872 100644 --- a/src/locus/agents/orchestrator.py +++ b/src/locus/agents/orchestrator.py @@ -141,6 +141,32 @@ def _check_tool_allowed(self, tc: ToolCall) -> Optional[ToolResult]: return None return self._governance_denial(tc.name, "policy denied the call") + @staticmethod + def _attach_tool_images( + messages: list[Message], results: list[ToolResult], vision: bool + ) -> None: + """Show the model images its tools returned (``metadata["images"]``). + + Tool messages can't carry images in most chat APIs, so they follow as + one user message after this round's tool results. + """ + if not vision: + return + images = [ + image + for result in results + for image in (result.metadata or {}).get("images") or [] + if isinstance(image, str) and image + ] + if images: + messages.append( + Message( + role=Role.USER, + content="Here is the image from the last tool call.", + images=images, + ) + ) + # ------------------------------------------------------------------ # Structured mode (THOUGHT/TOOL/INPUT/FINAL_ANSWER) # ------------------------------------------------------------------ @@ -364,6 +390,9 @@ def _run_function_calling( turns = 0 total_prompt_tokens = 0 total_completion_tokens = 0 + # Callers set metadata["vision"] = False for models that can't see + # images, so tool images (screenshots) aren't sent to them. + vision = (context.metadata.get("vision") if context else None) is not False for _turn in range(self._max_turns): turns += 1 @@ -422,6 +451,7 @@ def _run_function_calling( ) # Execute each tool (with loop guard check) and append results + round_start = len(all_tool_results) if self._parallel_tools and len(tool_calls) > 1: # Governance callbacks often wrap stateful policy engines and # are not assumed to be thread-safe. Evaluate them in request @@ -525,6 +555,8 @@ def _exec_tool(tc: ToolCall) -> tuple: ) ) + self._attach_tool_images(messages, all_tool_results[round_start:], vision) + # Max turns exceeded final_content = self._strip_think_tags(content) if content else "" self._emit_turn_end(turns=turns, max_turns_exceeded=True) diff --git a/src/locus/server/routes.py b/src/locus/server/routes.py index 859c523..11f08e6 100644 --- a/src/locus/server/routes.py +++ b/src/locus/server/routes.py @@ -8,7 +8,7 @@ import uuid import weakref from dataclasses import replace -from typing import Any +from typing import Any, Optional from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse @@ -292,6 +292,12 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request await _drop_images_without_vision(request_body, engine, model) + # Agent and developer mode tools can return images (screenshots); show + # them only to models that can see images. + agent_vision: Optional[bool] = None + if use_server_agent and request_body.mode in ("agent", "developer"): + agent_vision = await _model_accepts_images(engine, model) + # Search the web for the latest user message (before memory context, which # rewrites messages with attachments folded in). web_result = None @@ -459,6 +465,7 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request trace_store=getattr(request.app.state, "trace_store", None), bus=getattr(request.app.state, "bus", None), memory_service=getattr(request.app.state, "memory_service", None), + vision=agent_vision, ) else: response = await _handle_stream( @@ -502,6 +509,7 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request complexity_info, trace_store=getattr(request.app.state, "trace_store", None), bus=getattr(request.app.state, "bus", None), + vision=agent_vision, ) else: bus = getattr(request.app.state, "bus", None) @@ -800,6 +808,7 @@ def _handle_agent( *, trace_store=None, bus=None, + vision: Optional[bool] = None, ) -> ChatCompletionResponse: """Run through agent. @@ -822,6 +831,9 @@ def _handle_agent( input_text = messages[-1].content if messages else "" if messages and messages[-1].images: ctx.metadata["input_images"] = list(messages[-1].images) + if vision is not None: + # Whether tool images (screenshots) may be shown to the model. + ctx.metadata["vision"] = vision # Override agent model for this request if the caller specified one. # Locked for the full override-run-restore cycle (#759): only the @@ -941,6 +953,7 @@ async def _handle_agent_stream( trace_store=None, bus=None, memory_service=None, + vision: Optional[bool] = None, ): """Run the configured agent and return its result as an SSE response. @@ -996,6 +1009,7 @@ def run_agent(): complexity_info, trace_store=trace_store, bus=bus, + vision=vision, ) finally: current_session.reset(token) diff --git a/src/locus/tools/desktop/screenshot.py b/src/locus/tools/desktop/screenshot.py index ed34235..513b9d5 100644 --- a/src/locus/tools/desktop/screenshot.py +++ b/src/locus/tools/desktop/screenshot.py @@ -2,9 +2,11 @@ from __future__ import annotations +import base64 +import io import time from pathlib import Path -from typing import Any +from typing import Any, Optional from locus.core.registry import ToolRegistry from locus.core.types import ToolResult @@ -22,6 +24,27 @@ ) +#: Screenshots sent to a model are scaled down to fit this many pixels a side. +LOOK_MAX_PIXELS = 1280 + + +def image_for_model(path: Path) -> Optional[str]: + """The screenshot as a base64 JPEG small enough to send to a model.""" + try: + from PIL import Image + except ImportError: + return None + try: + with Image.open(path) as image: + image = image.convert("RGB") + image.thumbnail((LOOK_MAX_PIXELS, LOOK_MAX_PIXELS)) + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=70) + except OSError: + return None + return base64.b64encode(buffer.getvalue()).decode("ascii") + + def screenshots_dir() -> Path: from locus.core.paths import get_config_dir @@ -40,9 +63,19 @@ def spec(self) -> ToolSpec: name="screenshot", description=( "Take a screenshot of the whole screen and save it as a PNG." - " Returns the file's path." + " Returns the file's path. With look=true you also see the" + " screenshot, to read what's on screen (needs a model that can" + " see images)." ), - parameters={"type": "object", "properties": {}}, + parameters={ + "type": "object", + "properties": { + "look": { + "type": "boolean", + "description": "Also show you the screenshot.", + }, + }, + }, category="desktop", ) @@ -74,7 +107,24 @@ def execute(self, **params: Any) -> ToolResult: return _os.unsupported("screenshot", "Screenshots") if not done.ok or not path.exists(): return _os.fail("screenshot", f"Couldn't take a screenshot: {done.error()}") - return _os.ok("screenshot", f"Saved a screenshot to {path}.", path=str(path)) + if not params.get("look"): + return _os.ok( + "screenshot", f"Saved a screenshot to {path}.", path=str(path) + ) + image = image_for_model(path) + if image is None: + return _os.ok( + "screenshot", + f"Saved a screenshot to {path}, but couldn't prepare it for you" + " to see (Pillow is needed).", + path=str(path), + ) + return _os.ok( + "screenshot", + f"Saved a screenshot to {path}. It's attached so you can see it.", + path=str(path), + images=[image], + ) __all__ = ["ScreenshotTool", "screenshots_dir"] diff --git a/tests/server/test_agent_mode.py b/tests/server/test_agent_mode.py index c5df629..8d70e73 100644 --- a/tests/server/test_agent_mode.py +++ b/tests/server/test_agent_mode.py @@ -340,3 +340,40 @@ def test_agents_send_input_images_with_the_last_message(): assert with_images[-1].content == "what is this?" assert with_images[-1].images == ["aW1n"] assert without[-1].images is None + + +def test_tool_images_are_shown_only_to_vision_models(): + from locus.agents.orchestrator import OrchestratorAgent + from locus.core.types import Message, Role, ToolResult + + shot = ToolResult( + tool_name="screenshot", content="ok", metadata={"images": ["aW1n"]} + ) + plain = ToolResult(tool_name="file_read", content="text") + + messages: list = [] + OrchestratorAgent._attach_tool_images(messages, [plain, shot], vision=True) + assert len(messages) == 1 + assert messages[0].role == Role.USER + assert messages[0].images == ["aW1n"] + + blind: list[Message] = [] + OrchestratorAgent._attach_tool_images(blind, [shot], vision=False) + OrchestratorAgent._attach_tool_images(blind, [plain], vision=True) + assert blind == [] + + +@pytest.mark.parametrize("sees", [True, False]) +def test_agent_mode_tells_the_agent_whether_the_model_sees_images(monkeypatch, sees): + agent = FakeAgent() + monkeypatch.setattr(task_agent, "_build_task_agent", lambda state: agent) + + async def accepts(engine, model): + return sees + + monkeypatch.setattr("locus.server.routes._model_accepts_images", accepts) + app = create_app(_engine([]), "test-model", config=_config()) + + _post(TestClient(app), mode="agent") + + assert agent.contexts[0].metadata["vision"] is sees diff --git a/tests/tools/test_desktop_tools.py b/tests/tools/test_desktop_tools.py index 127b7e9..f6b7960 100644 --- a/tests/tools/test_desktop_tools.py +++ b/tests/tools/test_desktop_tools.py @@ -818,3 +818,44 @@ def test_wayland_tool_must_be_installed(monkeypatch): FakeOS(monkeypatch, "linux") monkeypatch.setattr(_os, "has_command", lambda name: False) assert "install kdotool" in WindowControlTool().execute(action="list").content + + +# ── Screenshots the model can see ──────────────────────────────── + + +def test_screenshot_look_attaches_a_small_jpeg(monkeypatch, tmp_path): + import base64 + import io + + from PIL import Image + + monkeypatch.setenv("LOCUS_HOME", str(tmp_path)) + fake = FakeOS(monkeypatch, "windows") + fake.on_call = lambda call: Image.new("RGB", (3000, 1500), "white").save( + call["args"]["path"] + ) + + plain = ScreenshotTool().execute() + looked = ScreenshotTool().execute(look=True) + + assert "images" not in plain.metadata + assert "attached" in looked.content + data = base64.b64decode(looked.metadata["images"][0]) + with Image.open(io.BytesIO(data)) as image: + assert image.format == "JPEG" + assert max(image.size) == 1280 + + +def test_screenshot_look_without_pillow(monkeypatch, tmp_path): + from locus.tools.desktop import screenshot + + monkeypatch.setenv("LOCUS_HOME", str(tmp_path)) + fake = FakeOS(monkeypatch, "windows") + fake.on_call = lambda call: Path(call["args"]["path"]).write_bytes(b"png") + monkeypatch.setattr(screenshot, "image_for_model", lambda path: None) + + result = ScreenshotTool().execute(look=True) + + assert result.success + assert "couldn't prepare it" in result.content + assert "images" not in result.metadata From 257716209b62541cbd6bbfbc5629f63b67bdc3ba Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 00:50:31 +0530 Subject: [PATCH 4/8] feat(agent): recommend a tool-capable model for Agent mode The desktop app sizes its default model by memory, which gives qwen3.5:4b on 16 GB machines, and small models often miss or garble tool calls. Agent mode now recommends a better model: - GET /v1/agent-model picks the largest installed model of 7B or more that fits in memory. Otherwise it suggests qwen3.5:9b from 12 GB (qwen3.5:4b from 8 GB, with a warning), and says when the computer has too little memory. - With Agent mode on and a model under 7B, the chat input shows the suggestion with one click: Use it if it's installed, or Download and use it. - Cloud and custom models with no size in their name aren't flagged. --- .../src/components/Chat/AgentModelHint.tsx | 78 +++++++++++++++++++ frontend/src/components/Chat/InputArea.tsx | 11 +-- frontend/src/lib/agent-model.test.ts | 17 ++++ frontend/src/lib/api.ts | 15 ++++ frontend/src/lib/model-capabilities.ts | 11 +++ src/locus/server/model_capabilities.py | 71 +++++++++++++++++ src/locus/server/routes.py | 16 ++++ tests/server/test_agent_mode.py | 42 ++++++++++ 8 files changed, 252 insertions(+), 9 deletions(-) create mode 100644 frontend/src/components/Chat/AgentModelHint.tsx create mode 100644 frontend/src/lib/agent-model.test.ts diff --git a/frontend/src/components/Chat/AgentModelHint.tsx b/frontend/src/components/Chat/AgentModelHint.tsx new file mode 100644 index 0000000..29ebdf8 --- /dev/null +++ b/frontend/src/components/Chat/AgentModelHint.tsx @@ -0,0 +1,78 @@ +import { useEffect, useState } from 'react'; +import { toast } from 'sonner'; +import { fetchAgentModel, fetchModels, pullModel, type AgentModelRecommendation } from '../../lib/api'; +import { modelSizeBillions, shouldSuggestAgentModel } from '../../lib/model-capabilities'; +import { useAppStore } from '../../lib/store'; + +/** + * In Agent mode with a small model, suggest the model that works best on + * this computer, with one click to use it (downloading it first if needed). + */ +export function AgentModelHint({ selectedModel }: { selectedModel: string }) { + const [rec, setRec] = useState(null); + const [busy, setBusy] = useState(false); + + useEffect(() => { + let cancelled = false; + fetchAgentModel() + .then((r) => { + if (!cancelled) setRec(r); + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, []); + + if (!rec) return null; + if (!rec.model) { + if (modelSizeBillions(selectedModel) === null) return null; + return ( +
+ {rec.reason} +
+ ); + } + if (!shouldSuggestAgentModel(selectedModel, rec.model)) return null; + + const useIt = async () => { + const app = useAppStore.getState(); + if (rec.installed) { + app.setSelectedModel(rec.model); + return; + } + setBusy(true); + const id = toast.loading(`Downloading ${rec.model}. This can take a few minutes…`); + try { + await pullModel(rec.model); + app.setModels(await fetchModels()); + app.setSelectedModel(rec.model); + setRec({ ...rec, installed: true }); + toast.success(`Now using ${rec.model} for Agent mode.`, { id }); + } catch (err: any) { + toast.error(err?.message || `Couldn't download ${rec.model}.`, { id }); + } finally { + setBusy(false); + } + }; + + return ( +
+ + {selectedModel} often misses tool calls. {rec.reason} + + +
+ ); +} diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx index 9ba845d..35881a3 100644 --- a/frontend/src/components/Chat/InputArea.tsx +++ b/frontend/src/components/Chat/InputArea.tsx @@ -6,13 +6,13 @@ import { streamChat, streamResearch } from '../../lib/sse'; import { fetchSavings, getBase, isTauri, stopAgentRun } from '../../lib/api'; import { listConnectors, getSyncStatus } from '../../lib/connectors-api'; import { serializeToolCallArguments } from '../../lib/tool-call'; -import { isSmallForAgent } from '../../lib/model-capabilities'; import { stoppedSummary } from '../../lib/agent-stop'; import { engineFromCompletionChunk, resolveChatEngine, } from '../../lib/chat-telemetry'; import { MicButton } from './MicButton'; +import { AgentModelHint } from './AgentModelHint'; import { AttachmentChip } from './AttachmentChip'; import { useSpeech } from '../../hooks/useSpeech'; import { useHandsFree } from '../../hooks/useHandsFree'; @@ -931,14 +931,7 @@ export function InputArea() { )} - {agentMode && selectedModel && isSmallForAgent(selectedModel) && ( -
- Small models like {selectedModel} often miss tool calls. A 4B or larger model works better in Agent mode. -
- )} + {agentMode && selectedModel && } {deepResearch && corpusSync.syncing && corpusSync.itemsSynced > 0 && (
{ + it('suggests a bigger model over small ones', () => { + expect(shouldSuggestAgentModel('qwen3.5:4b', 'qwen3.5:9b')).toBe(true); + expect(shouldSuggestAgentModel('qwen3:0.6b', 'qwen3.5:9b')).toBe(true); + }); + + it('leaves good, unknown and matching models alone', () => { + expect(shouldSuggestAgentModel('qwen3.5:9b', 'qwen3.5:9b')).toBe(false); + expect(shouldSuggestAgentModel('llama3.1:8b', 'qwen3.5:9b')).toBe(false); + expect(shouldSuggestAgentModel('gpt-4o', 'qwen3.5:9b')).toBe(false); + expect(shouldSuggestAgentModel('qwen3.5:4b', '')).toBe(false); + }); +}); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 22543e3..716373d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1273,3 +1273,18 @@ export async function removeAgentPermission(key: string): Promise { const res = await apiFetch(`/v1/chat/permissions/${encodeURIComponent(key)}`, { method: 'DELETE' }); if (!res.ok && res.status !== 404) throw new Error(`Could not remove the rule: ${res.status}`); } + +export interface AgentModelRecommendation { + /** Empty when the computer has too little memory for Agent mode. */ + model: string; + installed: boolean; + ram_gb: number; + reason: string; +} + +/** The model Agent mode works best with on this computer. */ +export async function fetchAgentModel(): Promise { + const res = await apiFetch('/v1/agent-model'); + if (!res.ok) throw new Error(`Could not get a model suggestion: ${res.status}`); + return res.json(); +} diff --git a/frontend/src/lib/model-capabilities.ts b/frontend/src/lib/model-capabilities.ts index a52b52a..eea4f97 100644 --- a/frontend/src/lib/model-capabilities.ts +++ b/frontend/src/lib/model-capabilities.ts @@ -33,3 +33,14 @@ export function isSmallForAgent(modelId: string): boolean { const size = modelSizeBillions(modelId); return size !== null && size < 4; } + +/** Below this size, Agent mode suggests a bigger model. */ +const AGENT_GOOD_SIZE_B = 7; + +/** Whether to suggest `recommended` instead of `selected` for Agent mode. */ +export function shouldSuggestAgentModel(selected: string, recommended: string): boolean { + if (!recommended || recommended === selected) return false; + const size = modelSizeBillions(selected); + // Unknown sizes are usually cloud or custom models; leave those alone. + return size !== null && size < AGENT_GOOD_SIZE_B; +} diff --git a/src/locus/server/model_capabilities.py b/src/locus/server/model_capabilities.py index 9e334e3..1907352 100644 --- a/src/locus/server/model_capabilities.py +++ b/src/locus/server/model_capabilities.py @@ -32,3 +32,74 @@ def is_embed_only_model(model_name: str) -> bool: __all__ = ["is_embed_only_model"] + + +#: Agent mode's recommended models, by the memory they need (GB). Smaller +#: models often miss or garble tool calls; 9B is the reliable floor that still +#: runs on a typical laptop. +AGENT_MODEL_LADDER = ( + ("qwen3.5:9b", 12.0), + ("qwen3.5:4b", 8.0), +) +#: Installed models this size or bigger are good enough for Agent mode. +AGENT_MIN_PARAMS_B = 7.0 + + +def model_params_b(model_name: str) -> float: + """Parameter count in billions from a name like ``qwen3.5:9b`` (else 0).""" + import re + + match = re.search(r"(?:^|[:\-_/])(\d+(?:\.\d+)?)b(?=$|[-_.:])", model_name.lower()) + return float(match.group(1)) if match else 0.0 + + +def _fits(params_b: float, ram_gb: float) -> bool: + # A 4-bit model needs about 0.75 GB per billion parameters, plus headroom. + return params_b * 0.75 + 4 <= ram_gb + + +def agent_model_recommendation(installed: list, ram_gb: float) -> dict: + """The model Agent mode should use on this computer. + + Prefers the largest installed model that's big enough and fits in memory, + then the ladder's best fit to download. ``model`` is empty when the + computer has too little memory for Agent mode to work well. + """ + usable = sorted( + ( + m + for m in installed + if not is_embed_only_model(m) + and model_params_b(m) >= AGENT_MIN_PARAMS_B + and _fits(model_params_b(m), ram_gb) + ), + key=model_params_b, + reverse=True, + ) + if usable: + return { + "model": usable[0], + "installed": True, + "ram_gb": round(ram_gb), + "reason": f"{usable[0]} is installed and handles tools well.", + } + for model, min_ram in AGENT_MODEL_LADDER: + if ram_gb >= min_ram: + reason = ( + f"{model} handles tools reliably on {round(ram_gb)} GB of memory." + if model_params_b(model) >= AGENT_MIN_PARAMS_B + else f"{model} is the largest that fits in {round(ram_gb)} GB;" + " Agent mode works, but may miss steps." + ) + return { + "model": model, + "installed": model in installed, + "ram_gb": round(ram_gb), + "reason": reason, + } + return { + "model": "", + "installed": False, + "ram_gb": round(ram_gb), + "reason": "Agent mode needs at least 8 GB of memory to work well.", + } diff --git a/src/locus/server/routes.py b/src/locus/server/routes.py index 11f08e6..8551e2a 100644 --- a/src/locus/server/routes.py +++ b/src/locus/server/routes.py @@ -544,6 +544,22 @@ async def answer_approval(approval_id: str, body: ApprovalAnswer): return {"id": approval_id, "decision": body.decision} +@router.get("/v1/agent-model") +async def agent_model(request: Request): + """The model Agent mode works best with on this computer.""" + import psutil + + from locus.server.model_capabilities import agent_model_recommendation + + engine = request.app.state.engine + try: + installed = list(await asyncio.to_thread(engine.list_models)) + except Exception: + installed = [] + ram_gb = psutil.virtual_memory().total / 1024**3 + return agent_model_recommendation(installed, ram_gb) + + @router.get("/v1/chat/permissions") async def list_agent_permissions(): """Agent mode's "always allow" rules.""" diff --git a/tests/server/test_agent_mode.py b/tests/server/test_agent_mode.py index 8d70e73..e2c9808 100644 --- a/tests/server/test_agent_mode.py +++ b/tests/server/test_agent_mode.py @@ -377,3 +377,45 @@ async def accepts(engine, model): _post(TestClient(app), mode="agent") assert agent.contexts[0].metadata["vision"] is sees + + +@pytest.mark.parametrize( + "installed, ram, model, is_installed", + [ + ([], 31, "qwen3.5:9b", False), + (["qwen3.5:9b"], 16, "qwen3.5:9b", True), + (["qwen3.5:4b", "llama3.1:8b", "nomic-embed-text"], 16, "llama3.1:8b", True), + (["qwen3.5:27b"], 16, "qwen3.5:9b", False), # 27b doesn't fit in 16 GB + (["qwen3.5:27b", "qwen3.5:9b"], 32, "qwen3.5:27b", True), + ([], 8, "qwen3.5:4b", False), + ([], 6, "", False), + ], +) +def test_agent_model_recommendation(installed, ram, model, is_installed): + from locus.server.model_capabilities import agent_model_recommendation + + rec = agent_model_recommendation(installed, ram) + + assert rec["model"] == model + assert rec["installed"] is is_installed + assert rec["reason"] + + +def test_agent_model_endpoint(monkeypatch): + import psutil + + engine = _engine([]) + engine.list_models.return_value = ["qwen3.5:4b"] + monkeypatch.setattr( + psutil, "virtual_memory", lambda: SimpleNamespace(total=16 * 1024**3) + ) + client = TestClient(create_app(engine, "test-model", config=_config())) + + body = client.get("/v1/agent-model").json() + + assert body == { + "model": "qwen3.5:9b", + "installed": False, + "ram_gb": 16, + "reason": "qwen3.5:9b handles tools reliably on 16 GB of memory.", + } From b4a303c073003c259f18fe92032983ce875d2b8c Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 00:51:59 +0530 Subject: [PATCH 5/8] feat(voice): answer approvals by voice in hands-free turns During a hands-free turn, an approval was the one step that still needed the mouse. Now Locus also reads the step aloud ("Locus wants to set the volume to 30%. Should I go ahead? Say yes or no.") and listens for the answer: - Yes, okay, go ahead or allow approves; no, don't, stop or cancel denies. An unclear answer ("yes no wait") falls back to the buttons. - Commands and code (run-level steps) are read out but always need a click, so nothing risky runs on a misheard word. - The on-screen buttons keep working the whole time. Adds speakAndWait() to the TTS store, which resolves when playback ends. --- frontend/src/components/Chat/InputArea.tsx | 41 ++++++++++++++++++++-- frontend/src/lib/tts.ts | 25 +++++++++++++ frontend/src/lib/voice-approval.test.ts | 40 +++++++++++++++++++++ frontend/src/lib/voice-approval.ts | 34 ++++++++++++++++++ 4 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 frontend/src/lib/voice-approval.test.ts create mode 100644 frontend/src/lib/voice-approval.ts diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx index 35881a3..5ee08e1 100644 --- a/frontend/src/components/Chat/InputArea.tsx +++ b/frontend/src/components/Chat/InputArea.tsx @@ -3,10 +3,12 @@ import { Send, Square, Paperclip, Search, Globe, Bot, FolderCode, X } from 'luci import { toast } from 'sonner'; import { useAppStore, generateId } from '../../lib/store'; import { streamChat, streamResearch } from '../../lib/sse'; -import { fetchSavings, getBase, isTauri, stopAgentRun } from '../../lib/api'; +import { answerApproval, fetchSavings, getBase, isTauri, stopAgentRun } from '../../lib/api'; import { listConnectors, getSyncStatus } from '../../lib/connectors-api'; import { serializeToolCallArguments } from '../../lib/tool-call'; import { stoppedSummary } from '../../lib/agent-stop'; +import { approvalQuestion, canAnswerByVoice, parseYesNo } from '../../lib/voice-approval'; +import { speakAndWait } from '../../lib/tts'; import { engineFromCompletionChunk, resolveChatEngine, @@ -237,6 +239,35 @@ export function InputArea() { cancelRecording, } = useSpeech(); + // Hands-free turns answer approvals aloud: Locus reads the step out and + // listens for yes or no. Commands and code still need a click. + const askApprovalAloud = useCallback( + async (tc: ToolCallInfo) => { + if (!tc.approvalId || !speechAvailable) return; + const spoke = await speakAndWait(`approval-${tc.approvalId}`, approvalQuestion(tc)); + if (!spoke || !canAnswerByVoice(tc)) return; + const heard = await new Promise((resolve) => { + void startRecording({ + onEndOfSpeech: (reason) => { + if (reason === 'no_speech') { + cancelRecording(); + resolve(''); + } else { + stopRecording().then(resolve, () => resolve('')); + } + }, + }); + }); + const answer = parseYesNo(heard); + if (!answer) { + void speakAndWait(`approval-${tc.approvalId}-retry`, "Sorry, I didn't catch that. Please answer on screen."); + return; + } + await answerApproval(tc.approvalId, answer === 'yes' ? 'approve' : 'deny').catch(() => false); + }, + [speechAvailable, startRecording, stopRecording, cancelRecording], + ); + // Hands-free: the transcript goes out through the prompt queue, which // sends it as soon as no reply is streaming. const handsFree = useAppStore((s) => s.settings.handsFree); @@ -591,7 +622,7 @@ export function InputArea() { } else if (eventName === 'approval_required') { try { const data = JSON.parse(sseEvent.data); - toolCalls.push({ + const pending: ToolCallInfo = { id: generateId(), tool: data.tool, arguments: serializeToolCallArguments(data.arguments), @@ -600,7 +631,10 @@ export function InputArea() { risk: data.risk, canAlways: Boolean(data.can_always), preview: data.preview || undefined, - }); + }; + toolCalls.push(pending); + // A hands-free turn asks aloud too (the buttons still work). + if (useAppStore.getState().speakNextReply) void askApprovalAloud(pending); setStreamState({ phase: 'Waiting for your approval...', activeToolCalls: [...toolCalls], @@ -817,6 +851,7 @@ export function InputArea() { setLastAssistantWebSearch, agentMode, projectDir, + askApprovalAloud, ]); const queuedPrompt = useAppStore((s) => s.queuedPrompt); diff --git a/frontend/src/lib/tts.ts b/frontend/src/lib/tts.ts index b1e9617..fc32396 100644 --- a/frontend/src/lib/tts.ts +++ b/frontend/src/lib/tts.ts @@ -154,6 +154,31 @@ export const useTtsStore = create((set, get) => ({ }, })); +/** + * Speak `text` and resolve once playback ends (or fails, or something else + * starts speaking). Resolves false when voice output isn't available. + */ +export async function speakAndWait(id: string, text: string, timeoutMs = 60000): Promise { + const tts = useTtsStore.getState(); + await tts.ensureHealth(); + if (useTtsStore.getState().available !== true) return false; + await tts.speak(id, text); + if (useTtsStore.getState().speakingId !== id) return useTtsStore.getState().errorId !== id; + return new Promise((resolve) => { + const timer = setTimeout(() => { + unsubscribe(); + resolve(true); + }, timeoutMs); + const unsubscribe = useTtsStore.subscribe((state) => { + if (state.speakingId !== id) { + clearTimeout(timer); + unsubscribe(); + resolve(state.errorId !== id); + } + }); + }); +} + /** Test seam: reset module-level playback handles between cases. */ export function __resetTtsForTests(): void { teardown(); diff --git a/frontend/src/lib/voice-approval.test.ts b/frontend/src/lib/voice-approval.test.ts new file mode 100644 index 0000000..c331c8a --- /dev/null +++ b/frontend/src/lib/voice-approval.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; + +import { approvalQuestion, canAnswerByVoice, parseYesNo } from './voice-approval'; +import type { ToolCallInfo } from '../types'; + +const tc = (tool: string, args: object, risk: 'change' | 'run'): ToolCallInfo => ({ + id: 'x', + tool, + arguments: JSON.stringify(args), + status: 'awaiting_approval', + risk, +}); + +describe('parseYesNo', () => { + it.each(['Yes', 'yeah sure', 'Okay, go ahead.', 'allow it', 'Yes please do'])('hears yes in %s', (t) => { + expect(parseYesNo(t)).toBe('yes'); + }); + + it.each(['No', 'nope', "Don't do that", 'cancel', 'stop', 'never mind'])('hears no in %s', (t) => { + expect(parseYesNo(t)).toBe('no'); + }); + + it.each(['', 'hmm', 'what was that', 'yes no wait', 'nobody knows'])('is unsure about %s', (t) => { + expect(parseYesNo(t)).toBeNull(); + }); +}); + +describe('approvals by voice', () => { + it('asks aloud for change-level steps only', () => { + const volume = tc('system_control', { action: 'set_volume', level: 30 }, 'change'); + const shell = tc('shell_exec', { command: 'dir' }, 'run'); + + expect(canAnswerByVoice(volume)).toBe(true); + expect(canAnswerByVoice(shell)).toBe(false); + expect(approvalQuestion(volume)).toBe( + 'Locus wants to set the volume to 30%. Should I go ahead? Say yes or no.', + ); + expect(approvalQuestion(shell)).toContain('please confirm it on screen'); + }); +}); diff --git a/frontend/src/lib/voice-approval.ts b/frontend/src/lib/voice-approval.ts new file mode 100644 index 0000000..2711014 --- /dev/null +++ b/frontend/src/lib/voice-approval.ts @@ -0,0 +1,34 @@ +import type { ToolCallInfo } from '../types'; +import { describeToolCall } from '../components/Chat/ToolCallCard'; + +const YES = ['yes', 'yeah', 'yep', 'yup', 'sure', 'ok', 'okay', 'allow', 'go ahead', 'do it', 'approve', 'confirm', 'please do']; +const NO = ['no', 'nope', 'nah', 'deny', "don't", 'do not', 'stop', 'cancel', 'skip', 'never mind', 'nevermind']; + +function hasPhrase(text: string, phrases: string[]): boolean { + return phrases.some((p) => new RegExp(`(^|\\b)${p.replace(/'/g, "'?")}(\\b|$)`).test(text)); +} + +/** What a spoken answer to an approval means, or null if it's unclear. */ +export function parseYesNo(transcript: string): 'yes' | 'no' | null { + const text = transcript.toLowerCase().replace(/[^a-z' ]+/g, ' ').replace(/\s+/g, ' ').trim(); + if (!text) return null; + const yes = hasPhrase(text, YES); + const no = hasPhrase(text, NO); + if (yes === no) return null; // neither, or both ("yes, no, wait") + return yes ? 'yes' : 'no'; +} + +/** Commands and code always need a click; other steps can be answered aloud. */ +export function canAnswerByVoice(tc: ToolCallInfo): boolean { + return tc.risk !== 'run'; +} + +/** What Locus says when it asks for an approval during a hands-free turn. */ +export function approvalQuestion(tc: ToolCallInfo): string { + const step = describeToolCall(tc.tool, tc.arguments); + const lowered = step.charAt(0).toLowerCase() + step.slice(1); + if (!canAnswerByVoice(tc)) { + return `Locus wants to ${lowered}. That runs a command, so please confirm it on screen.`; + } + return `Locus wants to ${lowered}. Should I go ahead? Say yes or no.`; +} From c5f7934a51b6e7de9687ce8ddde85b314e5f8d21 Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 00:53:57 +0530 Subject: [PATCH 6/8] feat(voice): add an opt-in "Hey Locus" wake word Hands-free voice needed the shortcut to start. With Say "Hey Locus" on (Settings > Speech, under Hands-free voice), saying "Hey Locus" starts listening, and "Hey Locus, mute the sound" sends the command straight away. There's no separate wake-word model: - A segmenter finds short bursts of speech against a noise floor that follows quiet moments. - Only bursts of 0.3-3.5 s are transcribed, by the local faster-whisper backend, and checked for the name. Common mishearings like "Lokus" and "Hey Lotus" count; "the locus of control" doesn't. - It refuses to run with a cloud speech backend, so audio never leaves the computer. - It pauses while Locus is listening, answering or speaking. It's off by default, and the microphone stays on while it's on. --- frontend/src/components/Chat/InputArea.tsx | 21 +++- frontend/src/hooks/useWakeWord.ts | 114 +++++++++++++++++++++ frontend/src/lib/store.ts | 3 + frontend/src/lib/wake-word.test.ts | 52 ++++++++++ frontend/src/lib/wake-word.ts | 64 ++++++++++++ frontend/src/pages/SettingsPage.tsx | 24 +++++ 6 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 frontend/src/hooks/useWakeWord.ts create mode 100644 frontend/src/lib/wake-word.test.ts create mode 100644 frontend/src/lib/wake-word.ts diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx index 5ee08e1..9b26626 100644 --- a/frontend/src/components/Chat/InputArea.tsx +++ b/frontend/src/components/Chat/InputArea.tsx @@ -8,7 +8,7 @@ import { listConnectors, getSyncStatus } from '../../lib/connectors-api'; import { serializeToolCallArguments } from '../../lib/tool-call'; import { stoppedSummary } from '../../lib/agent-stop'; import { approvalQuestion, canAnswerByVoice, parseYesNo } from '../../lib/voice-approval'; -import { speakAndWait } from '../../lib/tts'; +import { speakAndWait, useTtsStore } from '../../lib/tts'; import { engineFromCompletionChunk, resolveChatEngine, @@ -18,6 +18,7 @@ import { AgentModelHint } from './AgentModelHint'; import { AttachmentChip } from './AttachmentChip'; import { useSpeech } from '../../hooks/useSpeech'; import { useHandsFree } from '../../hooks/useHandsFree'; +import { useWakeWord } from '../../hooks/useWakeWord'; import { apiAttachmentFields, deletePayloads, @@ -271,7 +272,9 @@ export function InputArea() { // Hands-free: the transcript goes out through the prompt queue, which // sends it as soon as no reply is streaming. const handsFree = useAppStore((s) => s.settings.handsFree); - useHandsFree({ + const wakeWord = useAppStore((s) => s.settings.wakeWord); + const ttsState = useTtsStore((s) => s.state); + const { listen: listenHandsFree } = useHandsFree({ enabled: handsFree, available: speechAvailable, state: speechState, @@ -280,6 +283,20 @@ export function InputArea() { cancelRecording, send: (text) => useAppStore.getState().queuePrompt(text), }); + // "Hey Locus": a command said with it goes straight out; the name alone + // starts listening. + useWakeWord({ + enabled: handsFree && wakeWord, + busy: speechState !== 'idle' || streamState.isStreaming || ttsState !== 'idle', + onWake: (command) => { + if (command) { + useAppStore.getState().setSpeakNextReply(true); + useAppStore.getState().queuePrompt(command); + } else { + void listenHandsFree(); + } + }, + }); // Abort in-flight stream when the user switches models mid-generation. // This prevents errors from trying to continue a stream with a stale model. diff --git a/frontend/src/hooks/useWakeWord.ts b/frontend/src/hooks/useWakeWord.ts new file mode 100644 index 0000000..0870585 --- /dev/null +++ b/frontend/src/hooks/useWakeWord.ts @@ -0,0 +1,114 @@ +import { useEffect, useRef } from 'react'; +import { toast } from 'sonner'; +import { fetchSpeechHealth, transcribeAudio } from '../lib/api'; +import { rms } from '../lib/vad'; +import { LOCAL_SPEECH_BACKENDS, SpeechSegmenter, matchWakeWord } from '../lib/wake-word'; + +/** Bursts shorter than this are clicks or coughs, not words. */ +const MIN_SEGMENT_MS = 300; + +interface Options { + enabled: boolean; + /** Locus is listening, answering or speaking: ignore the microphone. */ + busy: boolean; + /** "Hey Locus" was heard; `command` is anything said after it. */ + onWake: (command: string) => void; +} + +/** + * Listens for "Hey Locus" while enabled. The microphone stays on, but only + * short bursts of speech are transcribed, by the local Whisper backend. + */ +export function useWakeWord({ enabled, busy, onWake }: Options) { + const busyRef = useRef(busy); + const onWakeRef = useRef(onWake); + useEffect(() => { + busyRef.current = busy; + onWakeRef.current = onWake; + }, [busy, onWake]); + + useEffect(() => { + if (!enabled) return; + let stopped = false; + let stream: MediaStream | null = null; + let context: AudioContext | null = null; + let timer: number | null = null; + + (async () => { + const health = await fetchSpeechHealth().catch(() => null); + if (!health?.available || !LOCAL_SPEECH_BACKENDS.includes(health.backend ?? '')) { + toast.error('"Hey Locus" needs the local Whisper speech backend (faster-whisper), so audio stays on this computer.'); + return; + } + if (stopped || !navigator.mediaDevices?.getUserMedia || typeof AudioContext === 'undefined') return; + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + toast.error('"Hey Locus" needs microphone access.'); + return; + } + if (stopped) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + context = new AudioContext(); + const analyser = context.createAnalyser(); + analyser.fftSize = 2048; + context.createMediaStreamSource(stream).connect(analyser); + const samples = new Float32Array(analyser.fftSize); + const segmenter = new SpeechSegmenter(); + let recorder: MediaRecorder | null = null; + let chunks: Blob[] = []; + let startedAt = 0; + + const finish = (rec: MediaRecorder, duration: number) => { + rec.onstop = async () => { + const blob = new Blob(chunks, { type: rec.mimeType || 'audio/webm' }); + chunks = []; + if (stopped || duration < MIN_SEGMENT_MS || busyRef.current) return; + try { + const { text } = await transcribeAudio(blob); + const { woke, command } = matchWakeWord(text || ''); + if (woke && !busyRef.current && !stopped) onWakeRef.current(command); + } catch { + // A failed transcription just means no wake word this time. + } + }; + rec.stop(); + }; + + timer = window.setInterval(() => { + if (!stream) return; + if (busyRef.current) { + if (recorder?.state === 'recording') { + recorder.onstop = null; + recorder.stop(); + } + recorder = null; + return; + } + analyser.getFloatTimeDomainData(samples); + const event = segmenter.push(rms(samples), performance.now()); + if (event === 'start') { + chunks = []; + recorder = new MediaRecorder(stream); + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + recorder.start(); + startedAt = performance.now(); + } else if (event === 'end' && recorder?.state === 'recording') { + finish(recorder, performance.now() - startedAt); + recorder = null; + } + }, 100); + })(); + + return () => { + stopped = true; + if (timer !== null) window.clearInterval(timer); + stream?.getTracks().forEach((t) => t.stop()); + void context?.close().catch(() => {}); + }; + }, [enabled]); +} diff --git a/frontend/src/lib/store.ts b/frontend/src/lib/store.ts index eae499c..06b9fcf 100644 --- a/frontend/src/lib/store.ts +++ b/frontend/src/lib/store.ts @@ -115,6 +115,8 @@ interface Settings { voiceAutoplay: boolean; /** Ctrl/Cmd+Shift+L listens, sends when you stop talking, and speaks the reply. */ handsFree: boolean; + /** With hands-free on, "Hey Locus" starts listening (local Whisper only). */ + wakeWord: boolean; } function loadSettings(): Settings { @@ -131,6 +133,7 @@ function loadSettings(): Settings { voiceOutputEnabled: false, voiceAutoplay: false, handsFree: false, + wakeWord: false, }; try { const raw = localStorage.getItem(SETTINGS_KEY); diff --git a/frontend/src/lib/wake-word.test.ts b/frontend/src/lib/wake-word.test.ts new file mode 100644 index 0000000..a85d474 --- /dev/null +++ b/frontend/src/lib/wake-word.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; + +import { SpeechSegmenter, matchWakeWord } from './wake-word'; + +describe('matchWakeWord', () => { + it.each([ + ['Hey Locus.', ''], + ['hey locus, set the volume to 30', 'set the volume to 30'], + ['Locus open my downloads folder.', 'open my downloads folder'], + ['OK Locus: what time is it?', 'what time is it'], + ['Hey Lokus', ''], + ['Hey Lotus, mute', 'mute'], + ])('wakes on %s', (transcript, command) => { + expect(matchWakeWord(transcript)).toEqual({ woke: true, command }); + }); + + it.each(['', 'Thank you.', 'the locus of control', 'lotus flower', 'focus on this'])( + 'stays asleep for %s', + (transcript) => { + expect(matchWakeWord(transcript).woke).toBe(false); + }, + ); +}); + +describe('SpeechSegmenter', () => { + function feed(seg: SpeechSegmenter, level: number, from: number, to: number) { + const events = []; + for (let t = from; t < to; t += 100) { + const e = seg.push(level, t); + if (e) events.push([e, t]); + } + return events; + } + + it('finds a burst of speech', () => { + const seg = new SpeechSegmenter(700, 3500); + expect(feed(seg, 0.005, 0, 1000)).toEqual([]); + expect(feed(seg, 0.2, 1000, 1800)).toEqual([['start', 1000]]); + expect(feed(seg, 0.005, 1800, 2700)).toEqual([['end', 2400]]); + }); + + it('cuts long speech into pieces', () => { + const seg = new SpeechSegmenter(700, 1000); + feed(seg, 0.005, 0, 500); + expect(feed(seg, 0.2, 500, 1600).map(([e]) => e)).toEqual(['start', 'end']); + }); + + it('ignores steady background noise', () => { + const seg = new SpeechSegmenter(); + expect(feed(seg, 0.012, 0, 3000)).toEqual([]); + }); +}); diff --git a/frontend/src/lib/wake-word.ts b/frontend/src/lib/wake-word.ts new file mode 100644 index 0000000..5b573d4 --- /dev/null +++ b/frontend/src/lib/wake-word.ts @@ -0,0 +1,64 @@ +/** + * Wake word ("Hey Locus") for hands-free voice. + * + * There's no separate wake-word model: Locus listens for short bursts of + * speech, transcribes only those with the local Whisper backend, and checks + * them for its name. Audio never leaves the computer, which is why this only + * runs with the `faster-whisper` backend. + */ + +/** Speech backends that run on this computer. */ +export const LOCAL_SPEECH_BACKENDS = ['faster-whisper']; + +// Whisper often hears "Locus" as one of these. +const NAME = '(?:locus|lokus|locas|lockus|low cus)'; +const WAKE = new RegExp(`^\\s*(?:(?:hey|hi|okay|ok|hello)[\\s,]+)?${NAME}\\b[\\s,.!?:;-]*(.*)$`, 'i'); +// "Hey Lotus" is a common mishearing, but "lotus" alone is a real word. +const WAKE_LOTUS = /^\s*(?:hey|hi|okay|ok)[\s,]+lotus\b[\s,.!?:;-]*(.*)$/i; + +/** Whether `transcript` starts with the wake word, and anything said after it. */ +export function matchWakeWord(transcript: string): { woke: boolean; command: string } { + const match = transcript.match(WAKE) ?? transcript.match(WAKE_LOTUS); + if (!match) return { woke: false, command: '' }; + return { woke: true, command: match[1].trim().replace(/[.!?]+$/, '').trim() }; +} + +export type SegmentEvent = 'start' | 'end' | null; + +/** + * Finds bursts of speech in the microphone's loudness (RMS every ~100 ms). + * The noise floor follows quiet moments, so a fan or traffic doesn't count. + */ +export class SpeechSegmenter { + private floor = 0.01; + private speaking = false; + private startedAt = 0; + private lastLoudAt = 0; + + constructor( + private readonly silenceMs = 700, + private readonly maxMs = 3500, + ) {} + + get threshold(): number { + return Math.max(this.floor * 3, 0.02); + } + + push(rms: number, now: number): SegmentEvent { + const loud = rms >= this.threshold; + if (!loud) this.floor = this.floor * 0.95 + rms * 0.05; + if (!this.speaking) { + if (!loud) return null; + this.speaking = true; + this.startedAt = now; + this.lastLoudAt = now; + return 'start'; + } + if (loud) this.lastLoudAt = now; + if (now - this.lastLoudAt >= this.silenceMs || now - this.startedAt >= this.maxMs) { + this.speaking = false; + return 'end'; + } + return null; + } +} diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 09d912b..16eef75 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -866,6 +866,30 @@ export function SettingsPage() { /> + + +
Date: Mon, 9 Nov 2026 00:55:32 +0530 Subject: [PATCH 7/8] ci(desktop): sign Windows installers when signing secrets are set macOS signing and notarization were already wired to the APPLE_* secrets, but Windows installers were always unsigned, so SmartScreen warned on first launch. A new "Configure Windows signing" step supports either: - a code-signing certificate (WINDOWS_CERTIFICATE, base64 .pfx, and WINDOWS_CERTIFICATE_PASSWORD), imported for the build and used by thumbprint with SHA-256 and a DigiCert timestamp, or - Azure Trusted Signing (AZURE_* secrets and variables) through trusted-signing-cli as Tauri's signCommand. It adds the setting to tauri.release.conf.json. With neither set, it warns on release tags and builds unsigned, as before. The release checklist documents every secret for Windows and macOS. --- .github/workflows/desktop.yml | 60 +++++++++++++++++++++++++++ docs/development/release-checklist.md | 26 ++++++++++++ 2 files changed, 86 insertions(+) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3898939..ff8d503 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -612,6 +612,66 @@ jobs: writeFileSync('frontend/src-tauri/tauri.release.conf.json', JSON.stringify(config)); NODE + # Windows code signing, so SmartScreen doesn't warn on first launch. + # Either a code-signing certificate (.pfx, base64 in WINDOWS_CERTIFICATE) + # or Azure Trusted Signing (AZURE_* secrets and variables); with neither, + # the installers stay unsigned as before. Updater artifacts are signed + # with the Tauri key either way. + - name: Configure Windows signing + if: runner.os == 'Windows' + shell: pwsh + env: + WIN_CERT: ${{ secrets.WINDOWS_CERTIFICATE }} + WIN_CERT_PASS: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }} + AZ_TENANT: ${{ secrets.AZURE_TENANT_ID }} + AZ_CLIENT: ${{ secrets.AZURE_CLIENT_ID }} + AZ_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZ_ENDPOINT: ${{ vars.AZURE_SIGNING_ENDPOINT }} + AZ_ACCOUNT: ${{ vars.AZURE_SIGNING_ACCOUNT }} + AZ_PROFILE: ${{ vars.AZURE_SIGNING_PROFILE }} + run: | + $configPath = 'frontend/src-tauri/tauri.release.conf.json' + $config = Get-Content $configPath -Raw | ConvertFrom-Json + $windows = $null + if ($env:WIN_CERT) { + $pfx = Join-Path $env:RUNNER_TEMP 'locus-signing.pfx' + [IO.File]::WriteAllBytes($pfx, [Convert]::FromBase64String($env:WIN_CERT)) + $password = ConvertTo-SecureString $env:WIN_CERT_PASS -AsPlainText -Force + $cert = Import-PfxCertificate -FilePath $pfx -CertStoreLocation Cert:\CurrentUser\My -Password $password + Remove-Item $pfx -Force + $windows = @{ + certificateThumbprint = $cert.Thumbprint + digestAlgorithm = 'sha256' + timestampUrl = 'http://timestamp.digicert.com' + } + Write-Host "Windows signing configured with certificate $($cert.Subject)" + } elseif ($env:AZ_SECRET) { + $missing = @('AZ_TENANT', 'AZ_CLIENT', 'AZ_ENDPOINT', 'AZ_ACCOUNT', 'AZ_PROFILE') | + Where-Object { -not [Environment]::GetEnvironmentVariable($_) } + if ($missing) { + Write-Host "::error::AZURE_CLIENT_SECRET is set but these are missing: $($missing -join ', ')" + exit 1 + } + cargo install trusted-signing-cli --locked + "AZURE_TENANT_ID=$env:AZ_TENANT" >> $env:GITHUB_ENV + "AZURE_CLIENT_ID=$env:AZ_CLIENT" >> $env:GITHUB_ENV + "AZURE_CLIENT_SECRET=$env:AZ_SECRET" >> $env:GITHUB_ENV + $windows = @{ + signCommand = "trusted-signing-cli -e $env:AZ_ENDPOINT -a $env:AZ_ACCOUNT -c $env:AZ_PROFILE -d Locus %1" + } + Write-Host 'Windows signing configured with Azure Trusted Signing' + } else { + if ('${{ github.ref }}' -like 'refs/tags/*') { + Write-Host '::warning::No Windows signing secrets; releasing unsigned Windows installers (SmartScreen will warn)' + } + Write-Host 'No Windows signing configured, skipping code signing' + } + if ($windows) { + $config.bundle | Add-Member -NotePropertyName windows -NotePropertyValue $windows -Force + # WriteAllText writes UTF-8 without a byte-order mark, which Tauri needs. + [IO.File]::WriteAllText((Resolve-Path $configPath), ($config | ConvertTo-Json -Depth 10)) + } + - name: Configure Apple signing if: runner.os == 'macOS' env: diff --git a/docs/development/release-checklist.md b/docs/development/release-checklist.md index 2eba4e4..dd59a90 100644 --- a/docs/development/release-checklist.md +++ b/docs/development/release-checklist.md @@ -50,3 +50,29 @@ Run at least one failure scenario per release; rotate which one. - All pytest tests pass: `uv run pytest tests/` - All bats tests pass: see `.github/workflows/bash-tests.yml` - Container integration matrix is green: see `.github/workflows/installer-integration.yml` + +## Code signing + +Without signing secrets, `desktop.yml` still releases, but Windows SmartScreen and macOS Gatekeeper warn on first launch. Add the secrets under **Settings > Secrets and variables > Actions** in the repository, and the next `desktop-v*` tag is signed. + +### Windows + +Use one of these. If both are set, the certificate wins. + +- **Code-signing certificate (.pfx).** An OV or EV certificate from a certificate authority. + - Secret `WINDOWS_CERTIFICATE`: the .pfx file, base64-encoded (`[Convert]::ToBase64String([IO.File]::ReadAllBytes("cert.pfx"))`). + - Secret `WINDOWS_CERTIFICATE_PASSWORD`: its password. +- **Azure Trusted Signing.** Usually the cheapest option. Create a Trusted Signing account and certificate profile, and an app registration with the *Trusted Signing Certificate Profile Signer* role. + - Secrets `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET`. + - Variables `AZURE_SIGNING_ENDPOINT` (for example `https://eus.codesigning.azure.net`), `AZURE_SIGNING_ACCOUNT` and `AZURE_SIGNING_PROFILE`. + +The "Configure Windows signing" step adds the certificate's thumbprint, or a `trusted-signing-cli` sign command, to `tauri.release.conf.json`. Installers are signed with SHA-256 and timestamped. + +### macOS + +Needs an Apple Developer ID Application certificate and notarization credentials: + +- Secrets `APPLE_CERTIFICATE` (the .p12, base64-encoded), `APPLE_CERTIFICATE_PASSWORD` and `APPLE_SIGNING_IDENTITY` (for example `Developer ID Application: Name (TEAMID)`). +- Secrets `APPLE_ID`, `APPLE_PASSWORD` (an app-specific password for that Apple ID) and `APPLE_TEAM_ID`. + +The workflow checks the notarization credentials before building, so a bad one fails in seconds rather than at the end. From 6e4518b7a59bcf1c1ebbb393397ffc78638a0dfb Mon Sep 17 00:00:00 2001 From: Aryan Date: Mon, 9 Nov 2026 01:19:15 +0530 Subject: [PATCH 8/8] feat(models): uninstall local models from Settings Models could only be removed with a hidden, unconfirmed trash icon in the model picker. Settings listed them by calling Ollama's port directly, which fails on a custom host. - Settings > Models lists installed models with their sizes and an In use badge. Uninstall asks inline ("Uninstall and free 6.6 GB?") first, then reports the space freed. If chat was using the model, it switches to another installed chat model, or says to download one. - The model picker's bin is visible now and needs a second click to confirm. Errors show instead of being swallowed. - GET /v1/models/installed lists models through the server, with sizes, parameter counts and which one Locus started with. - DELETE /v1/models/{name} now returns freed_bytes and was_default, gives a 404 for a model that isn't installed, and clears the cached vision capability. The size lookup is best-effort, so it can't block an uninstall. --- frontend/src/components/CommandPalette.tsx | 33 ++++- frontend/src/components/InstalledModels.tsx | 142 ++++++++++++++++++++ frontend/src/lib/api.ts | 42 +++++- frontend/src/lib/model-capabilities.ts | 13 ++ frontend/src/lib/uninstall-model.test.ts | 28 ++++ frontend/src/pages/SettingsPage.tsx | 35 ++--- src/locus/server/routes.py | 71 +++++++++- tests/server/test_uninstall_models.py | 134 ++++++++++++++++++ 8 files changed, 456 insertions(+), 42 deletions(-) create mode 100644 frontend/src/components/InstalledModels.tsx create mode 100644 frontend/src/lib/uninstall-model.test.ts create mode 100644 tests/server/test_uninstall_models.py diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx index 4fe1d94..768f9ac 100644 --- a/frontend/src/components/CommandPalette.tsx +++ b/frontend/src/components/CommandPalette.tsx @@ -1,9 +1,11 @@ import { useState, useRef, useEffect, useCallback } from 'react'; import { Search, Cpu, X, Download, Loader2, Trash2, Check, Cloud, Key, Eye, EyeOff } from 'lucide-react'; +import { toast } from 'sonner'; import { useAppStore } from '../lib/store'; import { pullModel, deleteModel, + formatBytes, fetchModels, preloadModel, isTauri, @@ -202,20 +204,31 @@ export function CommandPalette() { } }; + // Uninstalling takes two clicks: the first asks, the second confirms. + const [confirmDelete, setConfirmDelete] = useState(null); const handleDelete = async (modelId: string) => { + if (confirmDelete !== modelId) { + setConfirmDelete(modelId); + toast(`Click the bin again to uninstall ${modelId} from this computer.`); + return; + } + setConfirmDelete(null); setDeleting(modelId); try { - await deleteModel(modelId); + const { freedBytes } = await deleteModel(modelId); useAppStore.getState().addLogEntry({ timestamp: Date.now(), level: 'info', category: 'model', - message: `Deleted ${modelId}`, + message: `Uninstalled ${modelId}`, }); await refreshModels(); if (selectedModel === modelId) { const remaining = models.filter((m) => m.id !== modelId); if (remaining.length > 0) setSelectedModel(remaining[0].id); } - } catch {} finally { + toast.success(`Uninstalled ${modelId}${freedBytes ? ` and freed ${formatBytes(freedBytes)}` : ''}.`); + } catch (err: any) { + toast.error(err?.message || `Couldn't uninstall ${modelId}.`); + } finally { setDeleting(null); } }; @@ -400,10 +413,18 @@ export function CommandPalette() { onClick={() => handleDelete(model.id)} disabled={isDeleting} className="p-1 rounded transition-colors cursor-pointer" - style={{ color: 'var(--color-text-tertiary)', opacity: 0 }} - title="Delete model" + style={{ + color: confirmDelete === model.id ? 'var(--color-error)' : 'var(--color-text-tertiary)', + opacity: confirmDelete === model.id ? 1 : 0.5, + }} + title={confirmDelete === model.id ? `Click again to uninstall ${model.id}` : `Uninstall ${model.id}`} + aria-label={`Uninstall ${model.id}`} onMouseEnter={(e) => { e.currentTarget.style.opacity = '1'; e.currentTarget.style.color = 'var(--color-error)'; }} - onMouseLeave={(e) => { e.currentTarget.style.opacity = '0'; e.currentTarget.style.color = 'var(--color-text-tertiary)'; }} + onMouseLeave={(e) => { + if (confirmDelete === model.id) return; + e.currentTarget.style.opacity = '0.5'; + e.currentTarget.style.color = 'var(--color-text-tertiary)'; + }} > {isDeleting ? : } diff --git a/frontend/src/components/InstalledModels.tsx b/frontend/src/components/InstalledModels.tsx new file mode 100644 index 0000000..a307170 --- /dev/null +++ b/frontend/src/components/InstalledModels.tsx @@ -0,0 +1,142 @@ +import { useCallback, useEffect, useState } from 'react'; +import { Loader2, Trash2 } from 'lucide-react'; +import { toast } from 'sonner'; +import { + deleteModel, + fetchInstalledModels, + fetchModels, + formatBytes, + type InstalledModel, +} from '../lib/api'; +import { nextModelAfterUninstall } from '../lib/model-capabilities'; +import { useAppStore } from '../lib/store'; + +/** + * Local models with their sizes, each with an Uninstall button that asks + * first. Uninstalling the model chat is using switches chat to another one. + */ +export function InstalledModels() { + const [models, setModels] = useState(null); + const [error, setError] = useState(null); + const [confirming, setConfirming] = useState(null); + const [removing, setRemoving] = useState(null); + const selectedModel = useAppStore((s) => s.selectedModel); + + const load = useCallback(() => { + fetchInstalledModels() + .then((r) => { + setModels(r.models); + setError(null); + }) + .catch((err) => setError(err?.message || 'Could not list installed models.')); + }, []); + + useEffect(load, [load]); + + const uninstall = async (model: InstalledModel) => { + setConfirming(null); + setRemoving(model.name); + try { + const { freedBytes } = await deleteModel(model.name); + const remaining = (models ?? []).filter((m) => m.name !== model.name); + setModels(remaining); + const app = useAppStore.getState(); + app.setModels(await fetchModels().catch(() => app.models)); + const next = nextModelAfterUninstall( + app.selectedModel, + model.name, + useAppStore.getState().models.map((m) => m.id), + ); + if (next !== null) app.setSelectedModel(next); + app.addLogEntry({ + timestamp: Date.now(), level: 'info', category: 'model', + message: `Uninstalled ${model.name}`, + }); + const freed = freedBytes || model.size_bytes; + toast.success( + `Uninstalled ${model.name}${freed ? ` and freed ${formatBytes(freed)}` : ''}.` + + (next ? ` Chat now uses ${next}.` : next === '' ? ' Download a model to keep chatting.' : ''), + ); + } catch (err: any) { + toast.error(err?.message || `Couldn't uninstall ${model.name}.`); + } finally { + setRemoving(null); + } + }; + + if (error) { + return {error}; + } + if (models === null) { + return Loading…; + } + if (models.length === 0) { + return No models installed; + } + + return ( +
    + {models.map((model) => { + const inUse = model.name === selectedModel; + const busy = removing === model.name; + return ( +
  • + + {model.name} + + {formatBytes(model.size_bytes)} + + {inUse && ( + + In use + + )} + + {confirming === model.name ? ( + + + Uninstall and free {formatBytes(model.size_bytes)}? + + + + + ) : ( + + )} +
  • + ); + })} +
+ ); +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 716373d..65fc890 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -221,23 +221,55 @@ export async function pullModel(modelName: string): Promise { } } -export async function deleteModel(modelName: string): Promise { +export interface UninstallResult { + /** Disk space the model used, when the server could tell. */ + freedBytes: number; + /** It was the model Locus started with. */ + wasDefault: boolean; +} + +/** Uninstall a local (Ollama) model, freeing its disk space. */ +export async function deleteModel(modelName: string): Promise { if (isTauri()) { try { const { invoke } = await import('@tauri-apps/api/core'); await invoke('delete_ollama_model', { modelName }); - return; + return { freedBytes: 0, wasDefault: false }; } catch (e: any) { - throw new Error(e?.message || e || 'Delete failed'); + throw new Error(e?.message || e || 'Uninstall failed'); } } const res = await apiFetch(`/v1/models/${encodeURIComponent(modelName)}`, { method: 'DELETE', }); + const body = await res.json().catch(() => ({})); if (!res.ok) { - const detail = await res.text().catch(() => res.statusText); - throw new Error(`Failed to delete model: ${detail}`); + throw new Error(body.detail || `Couldn't uninstall ${modelName} (${res.status}).`); } + return { freedBytes: body.freed_bytes ?? 0, wasDefault: Boolean(body.was_default) }; +} + +export interface InstalledModel { + name: string; + size_bytes: number; + modified_at?: string | null; + parameter_size?: string | null; + is_default: boolean; +} + +/** Models downloaded to Ollama on this computer, with their sizes. */ +export async function fetchInstalledModels(): Promise<{ models: InstalledModel[]; total_bytes: number }> { + const res = await apiFetch('/v1/models/installed'); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.detail || `Couldn't list installed models (${res.status}).`); + return body; +} + +/** A byte count as GB or MB, like "6.6 GB". */ +export function formatBytes(bytes: number): string { + if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`; + if (bytes >= 1e6) return `${Math.round(bytes / 1e6)} MB`; + return `${bytes} bytes`; } const _CLOUD_PREFIXES = ['gpt-', 'o1-', 'o3-', 'o4-', 'claude-', 'gemini-', 'openrouter/']; diff --git a/frontend/src/lib/model-capabilities.ts b/frontend/src/lib/model-capabilities.ts index eea4f97..3d38f09 100644 --- a/frontend/src/lib/model-capabilities.ts +++ b/frontend/src/lib/model-capabilities.ts @@ -44,3 +44,16 @@ export function shouldSuggestAgentModel(selected: string, recommended: string): // Unknown sizes are usually cloud or custom models; leave those alone. return size !== null && size < AGENT_GOOD_SIZE_B; } + +/** + * After uninstalling `removed`: the model chat should switch to, '' if none + * is left, or null when chat wasn't using it. + */ +export function nextModelAfterUninstall( + selected: string, + removed: string, + remaining: string[], +): string | null { + if (selected !== removed) return null; + return remaining.find((m) => m !== removed && !isEmbedOnlyModel(m)) ?? ''; +} diff --git a/frontend/src/lib/uninstall-model.test.ts b/frontend/src/lib/uninstall-model.test.ts new file mode 100644 index 0000000..12a227f --- /dev/null +++ b/frontend/src/lib/uninstall-model.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; + +import { formatBytes } from './api'; +import { nextModelAfterUninstall } from './model-capabilities'; + +describe('nextModelAfterUninstall', () => { + it('keeps chat on its model when another one is removed', () => { + expect(nextModelAfterUninstall('qwen3.5:9b', 'qwen3.5:4b', ['qwen3.5:9b'])).toBeNull(); + }); + + it('switches chat to a remaining chat model', () => { + expect( + nextModelAfterUninstall('qwen3.5:4b', 'qwen3.5:4b', ['nomic-embed-text', 'qwen3.5:9b']), + ).toBe('qwen3.5:9b'); + }); + + it('says when no chat model is left', () => { + expect(nextModelAfterUninstall('qwen3.5:4b', 'qwen3.5:4b', ['nomic-embed-text'])).toBe(''); + }); +}); + +describe('formatBytes', () => { + it('shows GB and MB', () => { + expect(formatBytes(6_600_000_000)).toBe('6.6 GB'); + expect(formatBytes(274_000_000)).toBe('274 MB'); + expect(formatBytes(512)).toBe('512 bytes'); + }); +}); diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 16eef75..55f562a 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -37,31 +37,10 @@ import { } from '../lib/api'; import { isAutoUpdateDisabled, setAutoUpdateDisabled } from '../components/Desktop/UpdateChecker'; import { AgentPermissionsList } from '../components/Chat/AgentPermissionsList'; +import { InstalledModels } from '../components/InstalledModels'; const CLOUD_KEY_STATUS_CHANGED = 'locus-cloud-key-status-changed'; -function OllamaModelList() { - const [models, setModels] = useState>([]); - useEffect(() => { - fetch('http://localhost:11434/api/tags') - .then(r => r.json()) - .then(data => setModels((data.models || []).map((m: any) => ({ name: m.name, size: m.size })))) - .catch(() => setModels([])); - }, []); - if (models.length === 0) return No models loaded; - return ( -
- {models.map(m => ( - - - {m.name} ({(m.size / 1e9).toFixed(1)} GB) - - ))} -
- ); -} - function ApiKeyInput({ keyName, placeholder, @@ -617,11 +596,15 @@ export function SettingsPage() { {/* Models */}
- - - +
+
Local models (Ollama)
+
+ Models downloaded to this computer. Uninstall one to free its disk space; you can download it again later. +
+ +
- Run ollama pull <model-name> in your terminal to add more models + Add models from the model picker (⌘K), or run ollama pull <model-name> in a terminal
diff --git a/src/locus/server/routes.py b/src/locus/server/routes.py index 8551e2a..359d1ba 100644 --- a/src/locus/server/routes.py +++ b/src/locus/server/routes.py @@ -1530,17 +1530,67 @@ async def pull_model(request: Request): return {"status": "ok", "model": model_name} -@router.delete("/v1/models/{model_name:path}") -async def delete_model(model_name: str, request: Request): - """Delete a model from Ollama.""" +def _ollama_host(request: Request) -> str: + """Ollama's address, or 501 when the server isn't using Ollama.""" engine = request.app.state.engine engine_name = getattr(request.app.state, "engine_name", "") if engine_name != "ollama" and getattr(engine, "engine_id", "") != "ollama": raise HTTPException(status_code=501, detail="Only supported with Ollama engine") + return getattr(engine, "_host", "http://localhost:11434") + +async def _installed_ollama_models(host: str) -> list[dict]: import httpx as _httpx - host = getattr(engine, "_host", "http://localhost:11434") + try: + async with _httpx.AsyncClient(base_url=host, timeout=15.0) as client: + resp = await client.get("/api/tags") + resp.raise_for_status() + except (_httpx.ConnectError, _httpx.TimeoutException) as exc: + raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}") + except _httpx.HTTPStatusError as exc: + raise HTTPException( + status_code=exc.response.status_code, + detail=f"Ollama error: {exc.response.text[:300]}", + ) + return [ + { + "name": m.get("name") or m.get("model", ""), + "size_bytes": int(m.get("size") or 0), + "modified_at": m.get("modified_at"), + "parameter_size": (m.get("details") or {}).get("parameter_size"), + } + for m in resp.json().get("models") or [] + if m.get("name") or m.get("model") + ] + + +@router.get("/v1/models/installed") +async def installed_models(request: Request): + """Models downloaded to Ollama on this computer, with their sizes.""" + host = _ollama_host(request) + models = await _installed_ollama_models(host) + default = getattr(request.app.state, "model", "") + for m in models: + m["is_default"] = m["name"] == default + return { + "models": sorted(models, key=lambda m: m["name"]), + "total_bytes": sum(m["size_bytes"] for m in models), + } + + +@router.delete("/v1/models/{model_name:path}") +async def delete_model(model_name: str, request: Request): + """Uninstall a model from Ollama, freeing its disk space.""" + import httpx as _httpx + + host = _ollama_host(request) + # Look the model up first, so the answer says how much space it freed. + # Best effort: a failed lookup mustn't stop the uninstall. + try: + installed = {m["name"]: m for m in await _installed_ollama_models(host)} + except Exception: + installed = {} try: async with _httpx.AsyncClient(base_url=host, timeout=30.0) as client: resp = await client.request( @@ -1552,12 +1602,23 @@ async def delete_model(model_name: str, request: Request): except (_httpx.ConnectError, _httpx.TimeoutException) as exc: raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}") except _httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + raise HTTPException( + status_code=404, detail=f"{model_name} isn't installed." + ) raise HTTPException( status_code=exc.response.status_code, detail=f"Ollama error: {exc.response.text[:300]}", ) - return {"status": "deleted", "model": model_name} + _vision_cache.pop(model_name, None) + return { + "status": "deleted", + "model": model_name, + "freed_bytes": installed.get(model_name, {}).get("size_bytes", 0), + # The model Locus started with; chats need another model picked. + "was_default": model_name == getattr(request.app.state, "model", ""), + } @router.post("/v1/cloud/reload") diff --git a/tests/server/test_uninstall_models.py b/tests/server/test_uninstall_models.py new file mode 100644 index 0000000..f151fe0 --- /dev/null +++ b/tests/server/test_uninstall_models.py @@ -0,0 +1,134 @@ +"""Listing and uninstalling local (Ollama) models.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +fastapi = pytest.importorskip("fastapi") +httpx = pytest.importorskip("httpx") + +from fastapi.testclient import TestClient # noqa: E402 + +from locus.server import routes # noqa: E402 +from locus.server.app import create_app # noqa: E402 + +TAGS = { + "models": [ + { + "name": "qwen3.5:9b", + "size": 6_600_000_000, + "modified_at": "2026-11-09T10:00:00Z", + "details": {"parameter_size": "9B"}, + }, + {"name": "qwen3.5:4b", "size": 3_400_000_000, "details": {}}, + ] +} + + +class FakeOllama: + """Stands in for httpx.AsyncClient talking to Ollama.""" + + deleted: list = [] + missing: set = set() + down = False + + def __init__(self, base_url="", timeout=0): + self.base_url = base_url + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + def _response(self, method, path, status=200, body=None): + request = httpx.Request(method, f"{self.base_url}{path}") + return httpx.Response(status, json=body or {}, request=request) + + async def get(self, path): + if self.down: + raise httpx.ConnectError("refused") + return self._response("GET", path, body=TAGS) + + async def request(self, method, path, json=None): + if self.down: + raise httpx.ConnectError("refused") + name = json["name"] + if name in self.missing: + return self._response(method, path, 404, {"error": "model not found"}) + type(self).deleted.append(name) + return self._response(method, path) + + +@pytest.fixture +def client(monkeypatch): + FakeOllama.deleted = [] + FakeOllama.missing = set() + FakeOllama.down = False + monkeypatch.setattr(httpx, "AsyncClient", FakeOllama) + engine = SimpleNamespace(engine_id="ollama", _host="http://ollama:11434") + from locus.core.config import JarvisConfig + + cfg = JarvisConfig() + cfg.analytics.enabled = False + cfg.traces.enabled = False + return TestClient(create_app(engine, "qwen3.5:4b", config=cfg)) + + +def test_installed_models_have_sizes_and_the_default(client): + body = client.get("/v1/models/installed").json() + + assert body["total_bytes"] == 10_000_000_000 + assert [m["name"] for m in body["models"]] == ["qwen3.5:4b", "qwen3.5:9b"] + nine = body["models"][1] + assert nine["size_bytes"] == 6_600_000_000 + assert nine["parameter_size"] == "9B" + assert [m["is_default"] for m in body["models"]] == [True, False] + + +def test_uninstall_reports_the_space_freed(client): + routes._vision_cache["qwen3.5:9b"] = True + + body = client.delete("/v1/models/qwen3.5:9b").json() + + assert body == { + "status": "deleted", + "model": "qwen3.5:9b", + "freed_bytes": 6_600_000_000, + "was_default": False, + } + assert FakeOllama.deleted == ["qwen3.5:9b"] + assert "qwen3.5:9b" not in routes._vision_cache + + +def test_uninstalling_the_default_model_says_so(client): + assert client.delete("/v1/models/qwen3.5:4b").json()["was_default"] is True + + +def test_uninstalling_a_missing_model_is_404(client): + FakeOllama.missing = {"gone:1b"} + resp = client.delete("/v1/models/gone:1b") + assert resp.status_code == 404 + assert resp.json()["detail"] == "gone:1b isn't installed." + + +def test_ollama_down_is_502(client): + FakeOllama.down = True + assert client.get("/v1/models/installed").status_code == 502 + assert client.delete("/v1/models/qwen3.5:9b").status_code == 502 + + +def test_other_engines_cant_uninstall(monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", FakeOllama) + engine = SimpleNamespace(engine_id="vllm") + from locus.core.config import JarvisConfig + + cfg = JarvisConfig() + cfg.analytics.enabled = False + cfg.traces.enabled = False + client = TestClient(create_app(engine, "m", config=cfg)) + + assert client.get("/v1/models/installed").status_code == 501 + assert client.delete("/v1/models/m").status_code == 501