diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 3898939..ff8d503 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -612,6 +612,66 @@ jobs: writeFileSync('frontend/src-tauri/tauri.release.conf.json', JSON.stringify(config)); NODE + # Windows code signing, so SmartScreen doesn't warn on first launch. + # Either a code-signing certificate (.pfx, base64 in WINDOWS_CERTIFICATE) + # or Azure Trusted Signing (AZURE_* secrets and variables); with neither, + # the installers stay unsigned as before. Updater artifacts are signed + # with the Tauri key either way. + - name: Configure Windows signing + if: runner.os == 'Windows' + shell: pwsh + env: + WIN_CERT: ${{ secrets.WINDOWS_CERTIFICATE }} + WIN_CERT_PASS: ${{ secrets.WINDOWS_CERTIFICATE_PASSWORD }} + AZ_TENANT: ${{ secrets.AZURE_TENANT_ID }} + AZ_CLIENT: ${{ secrets.AZURE_CLIENT_ID }} + AZ_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZ_ENDPOINT: ${{ vars.AZURE_SIGNING_ENDPOINT }} + AZ_ACCOUNT: ${{ vars.AZURE_SIGNING_ACCOUNT }} + AZ_PROFILE: ${{ vars.AZURE_SIGNING_PROFILE }} + run: | + $configPath = 'frontend/src-tauri/tauri.release.conf.json' + $config = Get-Content $configPath -Raw | ConvertFrom-Json + $windows = $null + if ($env:WIN_CERT) { + $pfx = Join-Path $env:RUNNER_TEMP 'locus-signing.pfx' + [IO.File]::WriteAllBytes($pfx, [Convert]::FromBase64String($env:WIN_CERT)) + $password = ConvertTo-SecureString $env:WIN_CERT_PASS -AsPlainText -Force + $cert = Import-PfxCertificate -FilePath $pfx -CertStoreLocation Cert:\CurrentUser\My -Password $password + Remove-Item $pfx -Force + $windows = @{ + certificateThumbprint = $cert.Thumbprint + digestAlgorithm = 'sha256' + timestampUrl = 'http://timestamp.digicert.com' + } + Write-Host "Windows signing configured with certificate $($cert.Subject)" + } elseif ($env:AZ_SECRET) { + $missing = @('AZ_TENANT', 'AZ_CLIENT', 'AZ_ENDPOINT', 'AZ_ACCOUNT', 'AZ_PROFILE') | + Where-Object { -not [Environment]::GetEnvironmentVariable($_) } + if ($missing) { + Write-Host "::error::AZURE_CLIENT_SECRET is set but these are missing: $($missing -join ', ')" + exit 1 + } + cargo install trusted-signing-cli --locked + "AZURE_TENANT_ID=$env:AZ_TENANT" >> $env:GITHUB_ENV + "AZURE_CLIENT_ID=$env:AZ_CLIENT" >> $env:GITHUB_ENV + "AZURE_CLIENT_SECRET=$env:AZ_SECRET" >> $env:GITHUB_ENV + $windows = @{ + signCommand = "trusted-signing-cli -e $env:AZ_ENDPOINT -a $env:AZ_ACCOUNT -c $env:AZ_PROFILE -d Locus %1" + } + Write-Host 'Windows signing configured with Azure Trusted Signing' + } else { + if ('${{ github.ref }}' -like 'refs/tags/*') { + Write-Host '::warning::No Windows signing secrets; releasing unsigned Windows installers (SmartScreen will warn)' + } + Write-Host 'No Windows signing configured, skipping code signing' + } + if ($windows) { + $config.bundle | Add-Member -NotePropertyName windows -NotePropertyValue $windows -Force + # WriteAllText writes UTF-8 without a byte-order mark, which Tauri needs. + [IO.File]::WriteAllText((Resolve-Path $configPath), ($config | ConvertTo-Json -Depth 10)) + } + - name: Configure Apple signing if: runner.os == 'macOS' env: diff --git a/docs/development/release-checklist.md b/docs/development/release-checklist.md index 2eba4e4..dd59a90 100644 --- a/docs/development/release-checklist.md +++ b/docs/development/release-checklist.md @@ -50,3 +50,29 @@ Run at least one failure scenario per release; rotate which one. - All pytest tests pass: `uv run pytest tests/` - All bats tests pass: see `.github/workflows/bash-tests.yml` - Container integration matrix is green: see `.github/workflows/installer-integration.yml` + +## Code signing + +Without signing secrets, `desktop.yml` still releases, but Windows SmartScreen and macOS Gatekeeper warn on first launch. Add the secrets under **Settings > Secrets and variables > Actions** in the repository, and the next `desktop-v*` tag is signed. + +### Windows + +Use one of these. If both are set, the certificate wins. + +- **Code-signing certificate (.pfx).** An OV or EV certificate from a certificate authority. + - Secret `WINDOWS_CERTIFICATE`: the .pfx file, base64-encoded (`[Convert]::ToBase64String([IO.File]::ReadAllBytes("cert.pfx"))`). + - Secret `WINDOWS_CERTIFICATE_PASSWORD`: its password. +- **Azure Trusted Signing.** Usually the cheapest option. Create a Trusted Signing account and certificate profile, and an app registration with the *Trusted Signing Certificate Profile Signer* role. + - Secrets `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET`. + - Variables `AZURE_SIGNING_ENDPOINT` (for example `https://eus.codesigning.azure.net`), `AZURE_SIGNING_ACCOUNT` and `AZURE_SIGNING_PROFILE`. + +The "Configure Windows signing" step adds the certificate's thumbprint, or a `trusted-signing-cli` sign command, to `tauri.release.conf.json`. Installers are signed with SHA-256 and timestamped. + +### macOS + +Needs an Apple Developer ID Application certificate and notarization credentials: + +- Secrets `APPLE_CERTIFICATE` (the .p12, base64-encoded), `APPLE_CERTIFICATE_PASSWORD` and `APPLE_SIGNING_IDENTITY` (for example `Developer ID Application: Name (TEAMID)`). +- Secrets `APPLE_ID`, `APPLE_PASSWORD` (an app-specific password for that Apple ID) and `APPLE_TEAM_ID`. + +The workflow checks the notarization credentials before building, so a bad one fails in seconds rather than at the end. diff --git a/frontend/src/components/Chat/AgentModelHint.tsx b/frontend/src/components/Chat/AgentModelHint.tsx new file mode 100644 index 0000000..29ebdf8 --- /dev/null +++ b/frontend/src/components/Chat/AgentModelHint.tsx @@ -0,0 +1,78 @@ +import { useEffect, useState } from 'react'; +import { toast } from 'sonner'; +import { fetchAgentModel, fetchModels, pullModel, type AgentModelRecommendation } from '../../lib/api'; +import { modelSizeBillions, shouldSuggestAgentModel } from '../../lib/model-capabilities'; +import { useAppStore } from '../../lib/store'; + +/** + * In Agent mode with a small model, suggest the model that works best on + * this computer, with one click to use it (downloading it first if needed). + */ +export function AgentModelHint({ selectedModel }: { selectedModel: string }) { + const [rec, setRec] = useState(null); + const [busy, setBusy] = useState(false); + + useEffect(() => { + let cancelled = false; + fetchAgentModel() + .then((r) => { + if (!cancelled) setRec(r); + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, []); + + if (!rec) return null; + if (!rec.model) { + if (modelSizeBillions(selectedModel) === null) return null; + return ( +
+ {rec.reason} +
+ ); + } + if (!shouldSuggestAgentModel(selectedModel, rec.model)) return null; + + const useIt = async () => { + const app = useAppStore.getState(); + if (rec.installed) { + app.setSelectedModel(rec.model); + return; + } + setBusy(true); + const id = toast.loading(`Downloading ${rec.model}. This can take a few minutes…`); + try { + await pullModel(rec.model); + app.setModels(await fetchModels()); + app.setSelectedModel(rec.model); + setRec({ ...rec, installed: true }); + toast.success(`Now using ${rec.model} for Agent mode.`, { id }); + } catch (err: any) { + toast.error(err?.message || `Couldn't download ${rec.model}.`, { id }); + } finally { + setBusy(false); + } + }; + + return ( +
+ + {selectedModel} often misses tool calls. {rec.reason} + + +
+ ); +} diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx index 9ba845d..9b26626 100644 --- a/frontend/src/components/Chat/InputArea.tsx +++ b/frontend/src/components/Chat/InputArea.tsx @@ -3,19 +3,22 @@ import { Send, Square, Paperclip, Search, Globe, Bot, FolderCode, X } from 'luci import { toast } from 'sonner'; import { useAppStore, generateId } from '../../lib/store'; import { streamChat, streamResearch } from '../../lib/sse'; -import { fetchSavings, getBase, isTauri, stopAgentRun } from '../../lib/api'; +import { answerApproval, fetchSavings, getBase, isTauri, stopAgentRun } from '../../lib/api'; import { listConnectors, getSyncStatus } from '../../lib/connectors-api'; import { serializeToolCallArguments } from '../../lib/tool-call'; -import { isSmallForAgent } from '../../lib/model-capabilities'; import { stoppedSummary } from '../../lib/agent-stop'; +import { approvalQuestion, canAnswerByVoice, parseYesNo } from '../../lib/voice-approval'; +import { speakAndWait, useTtsStore } from '../../lib/tts'; import { engineFromCompletionChunk, resolveChatEngine, } from '../../lib/chat-telemetry'; import { MicButton } from './MicButton'; +import { AgentModelHint } from './AgentModelHint'; import { AttachmentChip } from './AttachmentChip'; import { useSpeech } from '../../hooks/useSpeech'; import { useHandsFree } from '../../hooks/useHandsFree'; +import { useWakeWord } from '../../hooks/useWakeWord'; import { apiAttachmentFields, deletePayloads, @@ -237,10 +240,41 @@ export function InputArea() { cancelRecording, } = useSpeech(); + // Hands-free turns answer approvals aloud: Locus reads the step out and + // listens for yes or no. Commands and code still need a click. + const askApprovalAloud = useCallback( + async (tc: ToolCallInfo) => { + if (!tc.approvalId || !speechAvailable) return; + const spoke = await speakAndWait(`approval-${tc.approvalId}`, approvalQuestion(tc)); + if (!spoke || !canAnswerByVoice(tc)) return; + const heard = await new Promise((resolve) => { + void startRecording({ + onEndOfSpeech: (reason) => { + if (reason === 'no_speech') { + cancelRecording(); + resolve(''); + } else { + stopRecording().then(resolve, () => resolve('')); + } + }, + }); + }); + const answer = parseYesNo(heard); + if (!answer) { + void speakAndWait(`approval-${tc.approvalId}-retry`, "Sorry, I didn't catch that. Please answer on screen."); + return; + } + await answerApproval(tc.approvalId, answer === 'yes' ? 'approve' : 'deny').catch(() => false); + }, + [speechAvailable, startRecording, stopRecording, cancelRecording], + ); + // Hands-free: the transcript goes out through the prompt queue, which // sends it as soon as no reply is streaming. const handsFree = useAppStore((s) => s.settings.handsFree); - useHandsFree({ + const wakeWord = useAppStore((s) => s.settings.wakeWord); + const ttsState = useTtsStore((s) => s.state); + const { listen: listenHandsFree } = useHandsFree({ enabled: handsFree, available: speechAvailable, state: speechState, @@ -249,6 +283,20 @@ export function InputArea() { cancelRecording, send: (text) => useAppStore.getState().queuePrompt(text), }); + // "Hey Locus": a command said with it goes straight out; the name alone + // starts listening. + useWakeWord({ + enabled: handsFree && wakeWord, + busy: speechState !== 'idle' || streamState.isStreaming || ttsState !== 'idle', + onWake: (command) => { + if (command) { + useAppStore.getState().setSpeakNextReply(true); + useAppStore.getState().queuePrompt(command); + } else { + void listenHandsFree(); + } + }, + }); // Abort in-flight stream when the user switches models mid-generation. // This prevents errors from trying to continue a stream with a stale model. @@ -591,7 +639,7 @@ export function InputArea() { } else if (eventName === 'approval_required') { try { const data = JSON.parse(sseEvent.data); - toolCalls.push({ + const pending: ToolCallInfo = { id: generateId(), tool: data.tool, arguments: serializeToolCallArguments(data.arguments), @@ -600,7 +648,10 @@ export function InputArea() { risk: data.risk, canAlways: Boolean(data.can_always), preview: data.preview || undefined, - }); + }; + toolCalls.push(pending); + // A hands-free turn asks aloud too (the buttons still work). + if (useAppStore.getState().speakNextReply) void askApprovalAloud(pending); setStreamState({ phase: 'Waiting for your approval...', activeToolCalls: [...toolCalls], @@ -817,6 +868,7 @@ export function InputArea() { setLastAssistantWebSearch, agentMode, projectDir, + askApprovalAloud, ]); const queuedPrompt = useAppStore((s) => s.queuedPrompt); @@ -931,14 +983,7 @@ export function InputArea() { )} - {agentMode && selectedModel && isSmallForAgent(selectedModel) && ( -
- Small models like {selectedModel} often miss tool calls. A 4B or larger model works better in Agent mode. -
- )} + {agentMode && selectedModel && } {deepResearch && corpusSync.syncing && corpusSync.itemsSynced > 0 && (
(null); const handleDelete = async (modelId: string) => { + if (confirmDelete !== modelId) { + setConfirmDelete(modelId); + toast(`Click the bin again to uninstall ${modelId} from this computer.`); + return; + } + setConfirmDelete(null); setDeleting(modelId); try { - await deleteModel(modelId); + const { freedBytes } = await deleteModel(modelId); useAppStore.getState().addLogEntry({ timestamp: Date.now(), level: 'info', category: 'model', - message: `Deleted ${modelId}`, + message: `Uninstalled ${modelId}`, }); await refreshModels(); if (selectedModel === modelId) { const remaining = models.filter((m) => m.id !== modelId); if (remaining.length > 0) setSelectedModel(remaining[0].id); } - } catch {} finally { + toast.success(`Uninstalled ${modelId}${freedBytes ? ` and freed ${formatBytes(freedBytes)}` : ''}.`); + } catch (err: any) { + toast.error(err?.message || `Couldn't uninstall ${modelId}.`); + } finally { setDeleting(null); } }; @@ -400,10 +413,18 @@ export function CommandPalette() { onClick={() => handleDelete(model.id)} disabled={isDeleting} className="p-1 rounded transition-colors cursor-pointer" - style={{ color: 'var(--color-text-tertiary)', opacity: 0 }} - title="Delete model" + style={{ + color: confirmDelete === model.id ? 'var(--color-error)' : 'var(--color-text-tertiary)', + opacity: confirmDelete === model.id ? 1 : 0.5, + }} + title={confirmDelete === model.id ? `Click again to uninstall ${model.id}` : `Uninstall ${model.id}`} + aria-label={`Uninstall ${model.id}`} onMouseEnter={(e) => { e.currentTarget.style.opacity = '1'; e.currentTarget.style.color = 'var(--color-error)'; }} - onMouseLeave={(e) => { e.currentTarget.style.opacity = '0'; e.currentTarget.style.color = 'var(--color-text-tertiary)'; }} + onMouseLeave={(e) => { + if (confirmDelete === model.id) return; + e.currentTarget.style.opacity = '0.5'; + e.currentTarget.style.color = 'var(--color-text-tertiary)'; + }} > {isDeleting ? : } diff --git a/frontend/src/components/InstalledModels.tsx b/frontend/src/components/InstalledModels.tsx new file mode 100644 index 0000000..a307170 --- /dev/null +++ b/frontend/src/components/InstalledModels.tsx @@ -0,0 +1,142 @@ +import { useCallback, useEffect, useState } from 'react'; +import { Loader2, Trash2 } from 'lucide-react'; +import { toast } from 'sonner'; +import { + deleteModel, + fetchInstalledModels, + fetchModels, + formatBytes, + type InstalledModel, +} from '../lib/api'; +import { nextModelAfterUninstall } from '../lib/model-capabilities'; +import { useAppStore } from '../lib/store'; + +/** + * Local models with their sizes, each with an Uninstall button that asks + * first. Uninstalling the model chat is using switches chat to another one. + */ +export function InstalledModels() { + const [models, setModels] = useState(null); + const [error, setError] = useState(null); + const [confirming, setConfirming] = useState(null); + const [removing, setRemoving] = useState(null); + const selectedModel = useAppStore((s) => s.selectedModel); + + const load = useCallback(() => { + fetchInstalledModels() + .then((r) => { + setModels(r.models); + setError(null); + }) + .catch((err) => setError(err?.message || 'Could not list installed models.')); + }, []); + + useEffect(load, [load]); + + const uninstall = async (model: InstalledModel) => { + setConfirming(null); + setRemoving(model.name); + try { + const { freedBytes } = await deleteModel(model.name); + const remaining = (models ?? []).filter((m) => m.name !== model.name); + setModels(remaining); + const app = useAppStore.getState(); + app.setModels(await fetchModels().catch(() => app.models)); + const next = nextModelAfterUninstall( + app.selectedModel, + model.name, + useAppStore.getState().models.map((m) => m.id), + ); + if (next !== null) app.setSelectedModel(next); + app.addLogEntry({ + timestamp: Date.now(), level: 'info', category: 'model', + message: `Uninstalled ${model.name}`, + }); + const freed = freedBytes || model.size_bytes; + toast.success( + `Uninstalled ${model.name}${freed ? ` and freed ${formatBytes(freed)}` : ''}.` + + (next ? ` Chat now uses ${next}.` : next === '' ? ' Download a model to keep chatting.' : ''), + ); + } catch (err: any) { + toast.error(err?.message || `Couldn't uninstall ${model.name}.`); + } finally { + setRemoving(null); + } + }; + + if (error) { + return {error}; + } + if (models === null) { + return Loading…; + } + if (models.length === 0) { + return No models installed; + } + + return ( +
    + {models.map((model) => { + const inUse = model.name === selectedModel; + const busy = removing === model.name; + return ( +
  • + + {model.name} + + {formatBytes(model.size_bytes)} + + {inUse && ( + + In use + + )} + + {confirming === model.name ? ( + + + Uninstall and free {formatBytes(model.size_bytes)}? + + + + + ) : ( + + )} +
  • + ); + })} +
+ ); +} diff --git a/frontend/src/hooks/useWakeWord.ts b/frontend/src/hooks/useWakeWord.ts new file mode 100644 index 0000000..0870585 --- /dev/null +++ b/frontend/src/hooks/useWakeWord.ts @@ -0,0 +1,114 @@ +import { useEffect, useRef } from 'react'; +import { toast } from 'sonner'; +import { fetchSpeechHealth, transcribeAudio } from '../lib/api'; +import { rms } from '../lib/vad'; +import { LOCAL_SPEECH_BACKENDS, SpeechSegmenter, matchWakeWord } from '../lib/wake-word'; + +/** Bursts shorter than this are clicks or coughs, not words. */ +const MIN_SEGMENT_MS = 300; + +interface Options { + enabled: boolean; + /** Locus is listening, answering or speaking: ignore the microphone. */ + busy: boolean; + /** "Hey Locus" was heard; `command` is anything said after it. */ + onWake: (command: string) => void; +} + +/** + * Listens for "Hey Locus" while enabled. The microphone stays on, but only + * short bursts of speech are transcribed, by the local Whisper backend. + */ +export function useWakeWord({ enabled, busy, onWake }: Options) { + const busyRef = useRef(busy); + const onWakeRef = useRef(onWake); + useEffect(() => { + busyRef.current = busy; + onWakeRef.current = onWake; + }, [busy, onWake]); + + useEffect(() => { + if (!enabled) return; + let stopped = false; + let stream: MediaStream | null = null; + let context: AudioContext | null = null; + let timer: number | null = null; + + (async () => { + const health = await fetchSpeechHealth().catch(() => null); + if (!health?.available || !LOCAL_SPEECH_BACKENDS.includes(health.backend ?? '')) { + toast.error('"Hey Locus" needs the local Whisper speech backend (faster-whisper), so audio stays on this computer.'); + return; + } + if (stopped || !navigator.mediaDevices?.getUserMedia || typeof AudioContext === 'undefined') return; + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + toast.error('"Hey Locus" needs microphone access.'); + return; + } + if (stopped) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + context = new AudioContext(); + const analyser = context.createAnalyser(); + analyser.fftSize = 2048; + context.createMediaStreamSource(stream).connect(analyser); + const samples = new Float32Array(analyser.fftSize); + const segmenter = new SpeechSegmenter(); + let recorder: MediaRecorder | null = null; + let chunks: Blob[] = []; + let startedAt = 0; + + const finish = (rec: MediaRecorder, duration: number) => { + rec.onstop = async () => { + const blob = new Blob(chunks, { type: rec.mimeType || 'audio/webm' }); + chunks = []; + if (stopped || duration < MIN_SEGMENT_MS || busyRef.current) return; + try { + const { text } = await transcribeAudio(blob); + const { woke, command } = matchWakeWord(text || ''); + if (woke && !busyRef.current && !stopped) onWakeRef.current(command); + } catch { + // A failed transcription just means no wake word this time. + } + }; + rec.stop(); + }; + + timer = window.setInterval(() => { + if (!stream) return; + if (busyRef.current) { + if (recorder?.state === 'recording') { + recorder.onstop = null; + recorder.stop(); + } + recorder = null; + return; + } + analyser.getFloatTimeDomainData(samples); + const event = segmenter.push(rms(samples), performance.now()); + if (event === 'start') { + chunks = []; + recorder = new MediaRecorder(stream); + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + recorder.start(); + startedAt = performance.now(); + } else if (event === 'end' && recorder?.state === 'recording') { + finish(recorder, performance.now() - startedAt); + recorder = null; + } + }, 100); + })(); + + return () => { + stopped = true; + if (timer !== null) window.clearInterval(timer); + stream?.getTracks().forEach((t) => t.stop()); + void context?.close().catch(() => {}); + }; + }, [enabled]); +} diff --git a/frontend/src/lib/agent-model.test.ts b/frontend/src/lib/agent-model.test.ts new file mode 100644 index 0000000..d8dc4b3 --- /dev/null +++ b/frontend/src/lib/agent-model.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; + +import { shouldSuggestAgentModel } from './model-capabilities'; + +describe('shouldSuggestAgentModel', () => { + it('suggests a bigger model over small ones', () => { + expect(shouldSuggestAgentModel('qwen3.5:4b', 'qwen3.5:9b')).toBe(true); + expect(shouldSuggestAgentModel('qwen3:0.6b', 'qwen3.5:9b')).toBe(true); + }); + + it('leaves good, unknown and matching models alone', () => { + expect(shouldSuggestAgentModel('qwen3.5:9b', 'qwen3.5:9b')).toBe(false); + expect(shouldSuggestAgentModel('llama3.1:8b', 'qwen3.5:9b')).toBe(false); + expect(shouldSuggestAgentModel('gpt-4o', 'qwen3.5:9b')).toBe(false); + expect(shouldSuggestAgentModel('qwen3.5:4b', '')).toBe(false); + }); +}); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 22543e3..65fc890 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -221,23 +221,55 @@ export async function pullModel(modelName: string): Promise { } } -export async function deleteModel(modelName: string): Promise { +export interface UninstallResult { + /** Disk space the model used, when the server could tell. */ + freedBytes: number; + /** It was the model Locus started with. */ + wasDefault: boolean; +} + +/** Uninstall a local (Ollama) model, freeing its disk space. */ +export async function deleteModel(modelName: string): Promise { if (isTauri()) { try { const { invoke } = await import('@tauri-apps/api/core'); await invoke('delete_ollama_model', { modelName }); - return; + return { freedBytes: 0, wasDefault: false }; } catch (e: any) { - throw new Error(e?.message || e || 'Delete failed'); + throw new Error(e?.message || e || 'Uninstall failed'); } } const res = await apiFetch(`/v1/models/${encodeURIComponent(modelName)}`, { method: 'DELETE', }); + const body = await res.json().catch(() => ({})); if (!res.ok) { - const detail = await res.text().catch(() => res.statusText); - throw new Error(`Failed to delete model: ${detail}`); + throw new Error(body.detail || `Couldn't uninstall ${modelName} (${res.status}).`); } + return { freedBytes: body.freed_bytes ?? 0, wasDefault: Boolean(body.was_default) }; +} + +export interface InstalledModel { + name: string; + size_bytes: number; + modified_at?: string | null; + parameter_size?: string | null; + is_default: boolean; +} + +/** Models downloaded to Ollama on this computer, with their sizes. */ +export async function fetchInstalledModels(): Promise<{ models: InstalledModel[]; total_bytes: number }> { + const res = await apiFetch('/v1/models/installed'); + const body = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(body.detail || `Couldn't list installed models (${res.status}).`); + return body; +} + +/** A byte count as GB or MB, like "6.6 GB". */ +export function formatBytes(bytes: number): string { + if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`; + if (bytes >= 1e6) return `${Math.round(bytes / 1e6)} MB`; + return `${bytes} bytes`; } const _CLOUD_PREFIXES = ['gpt-', 'o1-', 'o3-', 'o4-', 'claude-', 'gemini-', 'openrouter/']; @@ -1273,3 +1305,18 @@ export async function removeAgentPermission(key: string): Promise { const res = await apiFetch(`/v1/chat/permissions/${encodeURIComponent(key)}`, { method: 'DELETE' }); if (!res.ok && res.status !== 404) throw new Error(`Could not remove the rule: ${res.status}`); } + +export interface AgentModelRecommendation { + /** Empty when the computer has too little memory for Agent mode. */ + model: string; + installed: boolean; + ram_gb: number; + reason: string; +} + +/** The model Agent mode works best with on this computer. */ +export async function fetchAgentModel(): Promise { + const res = await apiFetch('/v1/agent-model'); + if (!res.ok) throw new Error(`Could not get a model suggestion: ${res.status}`); + return res.json(); +} diff --git a/frontend/src/lib/model-capabilities.ts b/frontend/src/lib/model-capabilities.ts index a52b52a..3d38f09 100644 --- a/frontend/src/lib/model-capabilities.ts +++ b/frontend/src/lib/model-capabilities.ts @@ -33,3 +33,27 @@ export function isSmallForAgent(modelId: string): boolean { const size = modelSizeBillions(modelId); return size !== null && size < 4; } + +/** Below this size, Agent mode suggests a bigger model. */ +const AGENT_GOOD_SIZE_B = 7; + +/** Whether to suggest `recommended` instead of `selected` for Agent mode. */ +export function shouldSuggestAgentModel(selected: string, recommended: string): boolean { + if (!recommended || recommended === selected) return false; + const size = modelSizeBillions(selected); + // Unknown sizes are usually cloud or custom models; leave those alone. + return size !== null && size < AGENT_GOOD_SIZE_B; +} + +/** + * After uninstalling `removed`: the model chat should switch to, '' if none + * is left, or null when chat wasn't using it. + */ +export function nextModelAfterUninstall( + selected: string, + removed: string, + remaining: string[], +): string | null { + if (selected !== removed) return null; + return remaining.find((m) => m !== removed && !isEmbedOnlyModel(m)) ?? ''; +} diff --git a/frontend/src/lib/store.ts b/frontend/src/lib/store.ts index eae499c..06b9fcf 100644 --- a/frontend/src/lib/store.ts +++ b/frontend/src/lib/store.ts @@ -115,6 +115,8 @@ interface Settings { voiceAutoplay: boolean; /** Ctrl/Cmd+Shift+L listens, sends when you stop talking, and speaks the reply. */ handsFree: boolean; + /** With hands-free on, "Hey Locus" starts listening (local Whisper only). */ + wakeWord: boolean; } function loadSettings(): Settings { @@ -131,6 +133,7 @@ function loadSettings(): Settings { voiceOutputEnabled: false, voiceAutoplay: false, handsFree: false, + wakeWord: false, }; try { const raw = localStorage.getItem(SETTINGS_KEY); diff --git a/frontend/src/lib/tts.ts b/frontend/src/lib/tts.ts index b1e9617..fc32396 100644 --- a/frontend/src/lib/tts.ts +++ b/frontend/src/lib/tts.ts @@ -154,6 +154,31 @@ export const useTtsStore = create((set, get) => ({ }, })); +/** + * Speak `text` and resolve once playback ends (or fails, or something else + * starts speaking). Resolves false when voice output isn't available. + */ +export async function speakAndWait(id: string, text: string, timeoutMs = 60000): Promise { + const tts = useTtsStore.getState(); + await tts.ensureHealth(); + if (useTtsStore.getState().available !== true) return false; + await tts.speak(id, text); + if (useTtsStore.getState().speakingId !== id) return useTtsStore.getState().errorId !== id; + return new Promise((resolve) => { + const timer = setTimeout(() => { + unsubscribe(); + resolve(true); + }, timeoutMs); + const unsubscribe = useTtsStore.subscribe((state) => { + if (state.speakingId !== id) { + clearTimeout(timer); + unsubscribe(); + resolve(state.errorId !== id); + } + }); + }); +} + /** Test seam: reset module-level playback handles between cases. */ export function __resetTtsForTests(): void { teardown(); diff --git a/frontend/src/lib/uninstall-model.test.ts b/frontend/src/lib/uninstall-model.test.ts new file mode 100644 index 0000000..12a227f --- /dev/null +++ b/frontend/src/lib/uninstall-model.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; + +import { formatBytes } from './api'; +import { nextModelAfterUninstall } from './model-capabilities'; + +describe('nextModelAfterUninstall', () => { + it('keeps chat on its model when another one is removed', () => { + expect(nextModelAfterUninstall('qwen3.5:9b', 'qwen3.5:4b', ['qwen3.5:9b'])).toBeNull(); + }); + + it('switches chat to a remaining chat model', () => { + expect( + nextModelAfterUninstall('qwen3.5:4b', 'qwen3.5:4b', ['nomic-embed-text', 'qwen3.5:9b']), + ).toBe('qwen3.5:9b'); + }); + + it('says when no chat model is left', () => { + expect(nextModelAfterUninstall('qwen3.5:4b', 'qwen3.5:4b', ['nomic-embed-text'])).toBe(''); + }); +}); + +describe('formatBytes', () => { + it('shows GB and MB', () => { + expect(formatBytes(6_600_000_000)).toBe('6.6 GB'); + expect(formatBytes(274_000_000)).toBe('274 MB'); + expect(formatBytes(512)).toBe('512 bytes'); + }); +}); diff --git a/frontend/src/lib/voice-approval.test.ts b/frontend/src/lib/voice-approval.test.ts new file mode 100644 index 0000000..c331c8a --- /dev/null +++ b/frontend/src/lib/voice-approval.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; + +import { approvalQuestion, canAnswerByVoice, parseYesNo } from './voice-approval'; +import type { ToolCallInfo } from '../types'; + +const tc = (tool: string, args: object, risk: 'change' | 'run'): ToolCallInfo => ({ + id: 'x', + tool, + arguments: JSON.stringify(args), + status: 'awaiting_approval', + risk, +}); + +describe('parseYesNo', () => { + it.each(['Yes', 'yeah sure', 'Okay, go ahead.', 'allow it', 'Yes please do'])('hears yes in %s', (t) => { + expect(parseYesNo(t)).toBe('yes'); + }); + + it.each(['No', 'nope', "Don't do that", 'cancel', 'stop', 'never mind'])('hears no in %s', (t) => { + expect(parseYesNo(t)).toBe('no'); + }); + + it.each(['', 'hmm', 'what was that', 'yes no wait', 'nobody knows'])('is unsure about %s', (t) => { + expect(parseYesNo(t)).toBeNull(); + }); +}); + +describe('approvals by voice', () => { + it('asks aloud for change-level steps only', () => { + const volume = tc('system_control', { action: 'set_volume', level: 30 }, 'change'); + const shell = tc('shell_exec', { command: 'dir' }, 'run'); + + expect(canAnswerByVoice(volume)).toBe(true); + expect(canAnswerByVoice(shell)).toBe(false); + expect(approvalQuestion(volume)).toBe( + 'Locus wants to set the volume to 30%. Should I go ahead? Say yes or no.', + ); + expect(approvalQuestion(shell)).toContain('please confirm it on screen'); + }); +}); diff --git a/frontend/src/lib/voice-approval.ts b/frontend/src/lib/voice-approval.ts new file mode 100644 index 0000000..2711014 --- /dev/null +++ b/frontend/src/lib/voice-approval.ts @@ -0,0 +1,34 @@ +import type { ToolCallInfo } from '../types'; +import { describeToolCall } from '../components/Chat/ToolCallCard'; + +const YES = ['yes', 'yeah', 'yep', 'yup', 'sure', 'ok', 'okay', 'allow', 'go ahead', 'do it', 'approve', 'confirm', 'please do']; +const NO = ['no', 'nope', 'nah', 'deny', "don't", 'do not', 'stop', 'cancel', 'skip', 'never mind', 'nevermind']; + +function hasPhrase(text: string, phrases: string[]): boolean { + return phrases.some((p) => new RegExp(`(^|\\b)${p.replace(/'/g, "'?")}(\\b|$)`).test(text)); +} + +/** What a spoken answer to an approval means, or null if it's unclear. */ +export function parseYesNo(transcript: string): 'yes' | 'no' | null { + const text = transcript.toLowerCase().replace(/[^a-z' ]+/g, ' ').replace(/\s+/g, ' ').trim(); + if (!text) return null; + const yes = hasPhrase(text, YES); + const no = hasPhrase(text, NO); + if (yes === no) return null; // neither, or both ("yes, no, wait") + return yes ? 'yes' : 'no'; +} + +/** Commands and code always need a click; other steps can be answered aloud. */ +export function canAnswerByVoice(tc: ToolCallInfo): boolean { + return tc.risk !== 'run'; +} + +/** What Locus says when it asks for an approval during a hands-free turn. */ +export function approvalQuestion(tc: ToolCallInfo): string { + const step = describeToolCall(tc.tool, tc.arguments); + const lowered = step.charAt(0).toLowerCase() + step.slice(1); + if (!canAnswerByVoice(tc)) { + return `Locus wants to ${lowered}. That runs a command, so please confirm it on screen.`; + } + return `Locus wants to ${lowered}. Should I go ahead? Say yes or no.`; +} diff --git a/frontend/src/lib/wake-word.test.ts b/frontend/src/lib/wake-word.test.ts new file mode 100644 index 0000000..a85d474 --- /dev/null +++ b/frontend/src/lib/wake-word.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; + +import { SpeechSegmenter, matchWakeWord } from './wake-word'; + +describe('matchWakeWord', () => { + it.each([ + ['Hey Locus.', ''], + ['hey locus, set the volume to 30', 'set the volume to 30'], + ['Locus open my downloads folder.', 'open my downloads folder'], + ['OK Locus: what time is it?', 'what time is it'], + ['Hey Lokus', ''], + ['Hey Lotus, mute', 'mute'], + ])('wakes on %s', (transcript, command) => { + expect(matchWakeWord(transcript)).toEqual({ woke: true, command }); + }); + + it.each(['', 'Thank you.', 'the locus of control', 'lotus flower', 'focus on this'])( + 'stays asleep for %s', + (transcript) => { + expect(matchWakeWord(transcript).woke).toBe(false); + }, + ); +}); + +describe('SpeechSegmenter', () => { + function feed(seg: SpeechSegmenter, level: number, from: number, to: number) { + const events = []; + for (let t = from; t < to; t += 100) { + const e = seg.push(level, t); + if (e) events.push([e, t]); + } + return events; + } + + it('finds a burst of speech', () => { + const seg = new SpeechSegmenter(700, 3500); + expect(feed(seg, 0.005, 0, 1000)).toEqual([]); + expect(feed(seg, 0.2, 1000, 1800)).toEqual([['start', 1000]]); + expect(feed(seg, 0.005, 1800, 2700)).toEqual([['end', 2400]]); + }); + + it('cuts long speech into pieces', () => { + const seg = new SpeechSegmenter(700, 1000); + feed(seg, 0.005, 0, 500); + expect(feed(seg, 0.2, 500, 1600).map(([e]) => e)).toEqual(['start', 'end']); + }); + + it('ignores steady background noise', () => { + const seg = new SpeechSegmenter(); + expect(feed(seg, 0.012, 0, 3000)).toEqual([]); + }); +}); diff --git a/frontend/src/lib/wake-word.ts b/frontend/src/lib/wake-word.ts new file mode 100644 index 0000000..5b573d4 --- /dev/null +++ b/frontend/src/lib/wake-word.ts @@ -0,0 +1,64 @@ +/** + * Wake word ("Hey Locus") for hands-free voice. + * + * There's no separate wake-word model: Locus listens for short bursts of + * speech, transcribes only those with the local Whisper backend, and checks + * them for its name. Audio never leaves the computer, which is why this only + * runs with the `faster-whisper` backend. + */ + +/** Speech backends that run on this computer. */ +export const LOCAL_SPEECH_BACKENDS = ['faster-whisper']; + +// Whisper often hears "Locus" as one of these. +const NAME = '(?:locus|lokus|locas|lockus|low cus)'; +const WAKE = new RegExp(`^\\s*(?:(?:hey|hi|okay|ok|hello)[\\s,]+)?${NAME}\\b[\\s,.!?:;-]*(.*)$`, 'i'); +// "Hey Lotus" is a common mishearing, but "lotus" alone is a real word. +const WAKE_LOTUS = /^\s*(?:hey|hi|okay|ok)[\s,]+lotus\b[\s,.!?:;-]*(.*)$/i; + +/** Whether `transcript` starts with the wake word, and anything said after it. */ +export function matchWakeWord(transcript: string): { woke: boolean; command: string } { + const match = transcript.match(WAKE) ?? transcript.match(WAKE_LOTUS); + if (!match) return { woke: false, command: '' }; + return { woke: true, command: match[1].trim().replace(/[.!?]+$/, '').trim() }; +} + +export type SegmentEvent = 'start' | 'end' | null; + +/** + * Finds bursts of speech in the microphone's loudness (RMS every ~100 ms). + * The noise floor follows quiet moments, so a fan or traffic doesn't count. + */ +export class SpeechSegmenter { + private floor = 0.01; + private speaking = false; + private startedAt = 0; + private lastLoudAt = 0; + + constructor( + private readonly silenceMs = 700, + private readonly maxMs = 3500, + ) {} + + get threshold(): number { + return Math.max(this.floor * 3, 0.02); + } + + push(rms: number, now: number): SegmentEvent { + const loud = rms >= this.threshold; + if (!loud) this.floor = this.floor * 0.95 + rms * 0.05; + if (!this.speaking) { + if (!loud) return null; + this.speaking = true; + this.startedAt = now; + this.lastLoudAt = now; + return 'start'; + } + if (loud) this.lastLoudAt = now; + if (now - this.lastLoudAt >= this.silenceMs || now - this.startedAt >= this.maxMs) { + this.speaking = false; + return 'end'; + } + return null; + } +} diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 09d912b..55f562a 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -37,31 +37,10 @@ import { } from '../lib/api'; import { isAutoUpdateDisabled, setAutoUpdateDisabled } from '../components/Desktop/UpdateChecker'; import { AgentPermissionsList } from '../components/Chat/AgentPermissionsList'; +import { InstalledModels } from '../components/InstalledModels'; const CLOUD_KEY_STATUS_CHANGED = 'locus-cloud-key-status-changed'; -function OllamaModelList() { - const [models, setModels] = useState>([]); - useEffect(() => { - fetch('http://localhost:11434/api/tags') - .then(r => r.json()) - .then(data => setModels((data.models || []).map((m: any) => ({ name: m.name, size: m.size })))) - .catch(() => setModels([])); - }, []); - if (models.length === 0) return No models loaded; - return ( -
- {models.map(m => ( - - - {m.name} ({(m.size / 1e9).toFixed(1)} GB) - - ))} -
- ); -} - function ApiKeyInput({ keyName, placeholder, @@ -617,11 +596,15 @@ export function SettingsPage() { {/* Models */}
- - - +
+
Local models (Ollama)
+
+ Models downloaded to this computer. Uninstall one to free its disk space; you can download it again later. +
+ +
- Run ollama pull <model-name> in your terminal to add more models + Add models from the model picker (⌘K), or run ollama pull <model-name> in a terminal
@@ -866,6 +849,30 @@ export function SettingsPage() { /> + + +
Optional[ToolResult]: return None return self._governance_denial(tc.name, "policy denied the call") + @staticmethod + def _attach_tool_images( + messages: list[Message], results: list[ToolResult], vision: bool + ) -> None: + """Show the model images its tools returned (``metadata["images"]``). + + Tool messages can't carry images in most chat APIs, so they follow as + one user message after this round's tool results. + """ + if not vision: + return + images = [ + image + for result in results + for image in (result.metadata or {}).get("images") or [] + if isinstance(image, str) and image + ] + if images: + messages.append( + Message( + role=Role.USER, + content="Here is the image from the last tool call.", + images=images, + ) + ) + # ------------------------------------------------------------------ # Structured mode (THOUGHT/TOOL/INPUT/FINAL_ANSWER) # ------------------------------------------------------------------ @@ -364,6 +390,9 @@ def _run_function_calling( turns = 0 total_prompt_tokens = 0 total_completion_tokens = 0 + # Callers set metadata["vision"] = False for models that can't see + # images, so tool images (screenshots) aren't sent to them. + vision = (context.metadata.get("vision") if context else None) is not False for _turn in range(self._max_turns): turns += 1 @@ -422,6 +451,7 @@ def _run_function_calling( ) # Execute each tool (with loop guard check) and append results + round_start = len(all_tool_results) if self._parallel_tools and len(tool_calls) > 1: # Governance callbacks often wrap stateful policy engines and # are not assumed to be thread-safe. Evaluate them in request @@ -525,6 +555,8 @@ def _exec_tool(tc: ToolCall) -> tuple: ) ) + self._attach_tool_images(messages, all_tool_results[round_start:], vision) + # Max turns exceeded final_content = self._strip_think_tags(content) if content else "" self._emit_turn_end(turns=turns, max_turns_exceeded=True) diff --git a/src/locus/server/model_capabilities.py b/src/locus/server/model_capabilities.py index 9e334e3..1907352 100644 --- a/src/locus/server/model_capabilities.py +++ b/src/locus/server/model_capabilities.py @@ -32,3 +32,74 @@ def is_embed_only_model(model_name: str) -> bool: __all__ = ["is_embed_only_model"] + + +#: Agent mode's recommended models, by the memory they need (GB). Smaller +#: models often miss or garble tool calls; 9B is the reliable floor that still +#: runs on a typical laptop. +AGENT_MODEL_LADDER = ( + ("qwen3.5:9b", 12.0), + ("qwen3.5:4b", 8.0), +) +#: Installed models this size or bigger are good enough for Agent mode. +AGENT_MIN_PARAMS_B = 7.0 + + +def model_params_b(model_name: str) -> float: + """Parameter count in billions from a name like ``qwen3.5:9b`` (else 0).""" + import re + + match = re.search(r"(?:^|[:\-_/])(\d+(?:\.\d+)?)b(?=$|[-_.:])", model_name.lower()) + return float(match.group(1)) if match else 0.0 + + +def _fits(params_b: float, ram_gb: float) -> bool: + # A 4-bit model needs about 0.75 GB per billion parameters, plus headroom. + return params_b * 0.75 + 4 <= ram_gb + + +def agent_model_recommendation(installed: list, ram_gb: float) -> dict: + """The model Agent mode should use on this computer. + + Prefers the largest installed model that's big enough and fits in memory, + then the ladder's best fit to download. ``model`` is empty when the + computer has too little memory for Agent mode to work well. + """ + usable = sorted( + ( + m + for m in installed + if not is_embed_only_model(m) + and model_params_b(m) >= AGENT_MIN_PARAMS_B + and _fits(model_params_b(m), ram_gb) + ), + key=model_params_b, + reverse=True, + ) + if usable: + return { + "model": usable[0], + "installed": True, + "ram_gb": round(ram_gb), + "reason": f"{usable[0]} is installed and handles tools well.", + } + for model, min_ram in AGENT_MODEL_LADDER: + if ram_gb >= min_ram: + reason = ( + f"{model} handles tools reliably on {round(ram_gb)} GB of memory." + if model_params_b(model) >= AGENT_MIN_PARAMS_B + else f"{model} is the largest that fits in {round(ram_gb)} GB;" + " Agent mode works, but may miss steps." + ) + return { + "model": model, + "installed": model in installed, + "ram_gb": round(ram_gb), + "reason": reason, + } + return { + "model": "", + "installed": False, + "ram_gb": round(ram_gb), + "reason": "Agent mode needs at least 8 GB of memory to work well.", + } diff --git a/src/locus/server/routes.py b/src/locus/server/routes.py index 859c523..359d1ba 100644 --- a/src/locus/server/routes.py +++ b/src/locus/server/routes.py @@ -8,7 +8,7 @@ import uuid import weakref from dataclasses import replace -from typing import Any +from typing import Any, Optional from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse @@ -292,6 +292,12 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request await _drop_images_without_vision(request_body, engine, model) + # Agent and developer mode tools can return images (screenshots); show + # them only to models that can see images. + agent_vision: Optional[bool] = None + if use_server_agent and request_body.mode in ("agent", "developer"): + agent_vision = await _model_accepts_images(engine, model) + # Search the web for the latest user message (before memory context, which # rewrites messages with attachments folded in). web_result = None @@ -459,6 +465,7 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request trace_store=getattr(request.app.state, "trace_store", None), bus=getattr(request.app.state, "bus", None), memory_service=getattr(request.app.state, "memory_service", None), + vision=agent_vision, ) else: response = await _handle_stream( @@ -502,6 +509,7 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request complexity_info, trace_store=getattr(request.app.state, "trace_store", None), bus=getattr(request.app.state, "bus", None), + vision=agent_vision, ) else: bus = getattr(request.app.state, "bus", None) @@ -536,6 +544,22 @@ async def answer_approval(approval_id: str, body: ApprovalAnswer): return {"id": approval_id, "decision": body.decision} +@router.get("/v1/agent-model") +async def agent_model(request: Request): + """The model Agent mode works best with on this computer.""" + import psutil + + from locus.server.model_capabilities import agent_model_recommendation + + engine = request.app.state.engine + try: + installed = list(await asyncio.to_thread(engine.list_models)) + except Exception: + installed = [] + ram_gb = psutil.virtual_memory().total / 1024**3 + return agent_model_recommendation(installed, ram_gb) + + @router.get("/v1/chat/permissions") async def list_agent_permissions(): """Agent mode's "always allow" rules.""" @@ -800,6 +824,7 @@ def _handle_agent( *, trace_store=None, bus=None, + vision: Optional[bool] = None, ) -> ChatCompletionResponse: """Run through agent. @@ -822,6 +847,9 @@ def _handle_agent( input_text = messages[-1].content if messages else "" if messages and messages[-1].images: ctx.metadata["input_images"] = list(messages[-1].images) + if vision is not None: + # Whether tool images (screenshots) may be shown to the model. + ctx.metadata["vision"] = vision # Override agent model for this request if the caller specified one. # Locked for the full override-run-restore cycle (#759): only the @@ -941,6 +969,7 @@ async def _handle_agent_stream( trace_store=None, bus=None, memory_service=None, + vision: Optional[bool] = None, ): """Run the configured agent and return its result as an SSE response. @@ -996,6 +1025,7 @@ def run_agent(): complexity_info, trace_store=trace_store, bus=bus, + vision=vision, ) finally: current_session.reset(token) @@ -1500,17 +1530,67 @@ async def pull_model(request: Request): return {"status": "ok", "model": model_name} -@router.delete("/v1/models/{model_name:path}") -async def delete_model(model_name: str, request: Request): - """Delete a model from Ollama.""" +def _ollama_host(request: Request) -> str: + """Ollama's address, or 501 when the server isn't using Ollama.""" engine = request.app.state.engine engine_name = getattr(request.app.state, "engine_name", "") if engine_name != "ollama" and getattr(engine, "engine_id", "") != "ollama": raise HTTPException(status_code=501, detail="Only supported with Ollama engine") + return getattr(engine, "_host", "http://localhost:11434") + +async def _installed_ollama_models(host: str) -> list[dict]: import httpx as _httpx - host = getattr(engine, "_host", "http://localhost:11434") + try: + async with _httpx.AsyncClient(base_url=host, timeout=15.0) as client: + resp = await client.get("/api/tags") + resp.raise_for_status() + except (_httpx.ConnectError, _httpx.TimeoutException) as exc: + raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}") + except _httpx.HTTPStatusError as exc: + raise HTTPException( + status_code=exc.response.status_code, + detail=f"Ollama error: {exc.response.text[:300]}", + ) + return [ + { + "name": m.get("name") or m.get("model", ""), + "size_bytes": int(m.get("size") or 0), + "modified_at": m.get("modified_at"), + "parameter_size": (m.get("details") or {}).get("parameter_size"), + } + for m in resp.json().get("models") or [] + if m.get("name") or m.get("model") + ] + + +@router.get("/v1/models/installed") +async def installed_models(request: Request): + """Models downloaded to Ollama on this computer, with their sizes.""" + host = _ollama_host(request) + models = await _installed_ollama_models(host) + default = getattr(request.app.state, "model", "") + for m in models: + m["is_default"] = m["name"] == default + return { + "models": sorted(models, key=lambda m: m["name"]), + "total_bytes": sum(m["size_bytes"] for m in models), + } + + +@router.delete("/v1/models/{model_name:path}") +async def delete_model(model_name: str, request: Request): + """Uninstall a model from Ollama, freeing its disk space.""" + import httpx as _httpx + + host = _ollama_host(request) + # Look the model up first, so the answer says how much space it freed. + # Best effort: a failed lookup mustn't stop the uninstall. + try: + installed = {m["name"]: m for m in await _installed_ollama_models(host)} + except Exception: + installed = {} try: async with _httpx.AsyncClient(base_url=host, timeout=30.0) as client: resp = await client.request( @@ -1522,12 +1602,23 @@ async def delete_model(model_name: str, request: Request): except (_httpx.ConnectError, _httpx.TimeoutException) as exc: raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}") except _httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + raise HTTPException( + status_code=404, detail=f"{model_name} isn't installed." + ) raise HTTPException( status_code=exc.response.status_code, detail=f"Ollama error: {exc.response.text[:300]}", ) - return {"status": "deleted", "model": model_name} + _vision_cache.pop(model_name, None) + return { + "status": "deleted", + "model": model_name, + "freed_bytes": installed.get(model_name, {}).get("size_bytes", 0), + # The model Locus started with; chats need another model picked. + "was_default": model_name == getattr(request.app.state, "model", ""), + } @router.post("/v1/cloud/reload") diff --git a/src/locus/tools/desktop/screenshot.py b/src/locus/tools/desktop/screenshot.py index ed34235..513b9d5 100644 --- a/src/locus/tools/desktop/screenshot.py +++ b/src/locus/tools/desktop/screenshot.py @@ -2,9 +2,11 @@ from __future__ import annotations +import base64 +import io import time from pathlib import Path -from typing import Any +from typing import Any, Optional from locus.core.registry import ToolRegistry from locus.core.types import ToolResult @@ -22,6 +24,27 @@ ) +#: Screenshots sent to a model are scaled down to fit this many pixels a side. +LOOK_MAX_PIXELS = 1280 + + +def image_for_model(path: Path) -> Optional[str]: + """The screenshot as a base64 JPEG small enough to send to a model.""" + try: + from PIL import Image + except ImportError: + return None + try: + with Image.open(path) as image: + image = image.convert("RGB") + image.thumbnail((LOOK_MAX_PIXELS, LOOK_MAX_PIXELS)) + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=70) + except OSError: + return None + return base64.b64encode(buffer.getvalue()).decode("ascii") + + def screenshots_dir() -> Path: from locus.core.paths import get_config_dir @@ -40,9 +63,19 @@ def spec(self) -> ToolSpec: name="screenshot", description=( "Take a screenshot of the whole screen and save it as a PNG." - " Returns the file's path." + " Returns the file's path. With look=true you also see the" + " screenshot, to read what's on screen (needs a model that can" + " see images)." ), - parameters={"type": "object", "properties": {}}, + parameters={ + "type": "object", + "properties": { + "look": { + "type": "boolean", + "description": "Also show you the screenshot.", + }, + }, + }, category="desktop", ) @@ -74,7 +107,24 @@ def execute(self, **params: Any) -> ToolResult: return _os.unsupported("screenshot", "Screenshots") if not done.ok or not path.exists(): return _os.fail("screenshot", f"Couldn't take a screenshot: {done.error()}") - return _os.ok("screenshot", f"Saved a screenshot to {path}.", path=str(path)) + if not params.get("look"): + return _os.ok( + "screenshot", f"Saved a screenshot to {path}.", path=str(path) + ) + image = image_for_model(path) + if image is None: + return _os.ok( + "screenshot", + f"Saved a screenshot to {path}, but couldn't prepare it for you" + " to see (Pillow is needed).", + path=str(path), + ) + return _os.ok( + "screenshot", + f"Saved a screenshot to {path}. It's attached so you can see it.", + path=str(path), + images=[image], + ) __all__ = ["ScreenshotTool", "screenshots_dir"] diff --git a/src/locus/tools/desktop/wayland.py b/src/locus/tools/desktop/wayland.py new file mode 100644 index 0000000..885559c --- /dev/null +++ b/src/locus/tools/desktop/wayland.py @@ -0,0 +1,249 @@ +"""Window control on Wayland, one backend per compositor. + +Wayland has no shared window API, so Locus talks to the compositor itself: +``swaymsg`` on Sway, ``hyprctl`` on Hyprland and ``kdotool`` on KDE Plasma. +GNOME doesn't let other apps control windows, so it isn't supported. +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from locus.core.types import ToolResult +from locus.tools.desktop import _os + +Window = Tuple[Any, str] +Rect = Tuple[int, int, int, int] +Backend = Tuple[Callable[[], Any], Callable[..., Optional[str]]] + + +def is_wayland() -> bool: + return os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland" or bool( + os.environ.get("WAYLAND_DISPLAY") + ) + + +def _placement_rect(action: str, placement: dict, current: Rect, work: Rect) -> Rect: + from locus.tools.desktop.window_control import placement_rect + + return placement_rect(action, placement, current, work) + + +def _json(args: List[str]) -> Union[Any, ToolResult]: + done = _os.run(args) + if not done.ok: + return _os.fail("window_control", f"Couldn't list windows: {done.error()}") + try: + return json.loads(done.stdout or "null") + except ValueError: + return _os.fail("window_control", f"{args[0]} gave output Locus can't read.") + + +# ── Sway ───────────────────────────────────────────────────────── + + +def _sway_nodes(tree: Any) -> List[dict]: + """Every window in a sway tree, including floating and scratchpad ones.""" + found: List[dict] = [] + stack = [tree] if isinstance(tree, dict) else [] + while stack: + node = stack.pop() + children = node.get("nodes", []) + node.get("floating_nodes", []) + if ( + not children + and node.get("type") in ("con", "floating_con") + and node.get("name") + ): + found.append(node) + stack.extend(children) + return found + + +def _sway_windows() -> Union[List[Window], ToolResult]: + tree = _json(["swaymsg", "-t", "get_tree", "-r"]) + if isinstance(tree, ToolResult): + return tree + return [(node, node["name"]) for node in _sway_nodes(tree)] + + +def _sway_output_rect() -> Optional[Rect]: + outputs = _json(["swaymsg", "-t", "get_outputs", "-r"]) + if isinstance(outputs, ToolResult) or not isinstance(outputs, list): + return None + for output in outputs: + if output.get("focused") and output.get("rect"): + r = output["rect"] + return r["x"], r["y"], r["width"], r["height"] + return None + + +def _sway_act(node: dict, title: str, action: str, placement: Optional[dict] = None): + target = f"[con_id={node['id']}]" + commands = { + "focus": f"{target} focus", + "minimize": f"{target} move scratchpad", + "maximize": f"{target} fullscreen enable", + "restore": f"{target} scratchpad show, {target} floating disable, " + f"{target} fullscreen disable, {target} focus", + } + if action in commands: + command = commands[action] + else: + rect = node.get("rect", {}) + current = ( + rect.get("x", 0), + rect.get("y", 0), + rect.get("width", 800), + rect.get("height", 600), + ) + work = _sway_output_rect() or current + x, y, w, h = _placement_rect(action, placement or {}, current, work) + command = ( + f"{target} floating enable, {target} resize set {w} px {h} px, " + f"{target} move absolute position {x} px {y} px" + ) + done = _os.run(["swaymsg", command]) + return None if done.ok else f"Couldn't change {title}: {done.error()}" + + +# ── Hyprland ───────────────────────────────────────────────────── + + +def _hypr_windows() -> Union[List[Window], ToolResult]: + clients = _json(["hyprctl", "clients", "-j"]) + if isinstance(clients, ToolResult): + return clients + return [(c, c["title"]) for c in clients or [] if c.get("title")] + + +def _hypr_monitor_rect() -> Optional[Rect]: + monitors = _json(["hyprctl", "monitors", "-j"]) + if isinstance(monitors, ToolResult) or not isinstance(monitors, list): + return None + for m in monitors: + if m.get("focused"): + scale = float(m.get("scale") or 1) or 1 + reserved = m.get("reserved") or [0, 0, 0, 0] # left, top, right, bottom + width = int(m["width"] / scale) - reserved[0] - reserved[2] + height = int(m["height"] / scale) - reserved[1] - reserved[3] + return m["x"] + reserved[0], m["y"] + reserved[1], width, height + return None + + +def _hypr_act(client: dict, title: str, action: str, placement: Optional[dict] = None): + window = f"address:{client['address']}" + if action == "focus": + batches = [f"dispatch focuswindow {window}"] + elif action == "minimize": + # Hyprland has no minimize; a special workspace hides the window. + batches = [f"dispatch movetoworkspacesilent special:minimized,{window}"] + elif action == "maximize": + batches = [f"dispatch focuswindow {window}", "dispatch fullscreen 1"] + elif action == "restore": + batches = [ + f"dispatch movetoworkspace e+0,{window}", + f"dispatch focuswindow {window}", + ] + else: + at, size = client.get("at", [0, 0]), client.get("size", [800, 600]) + current = (at[0], at[1], size[0], size[1]) + work = _hypr_monitor_rect() or current + x, y, w, h = _placement_rect(action, placement or {}, current, work) + batches = [] + if not client.get("floating"): + batches.append(f"dispatch setfloating {window}") + batches += [ + f"dispatch resizewindowpixel exact {w} {h},{window}", + f"dispatch movewindowpixel exact {x} {y},{window}", + ] + done = _os.run(["hyprctl", "--batch", " ; ".join(batches)]) + return None if done.ok else f"Couldn't change {title}: {done.error()}" + + +# ── KDE Plasma (kdotool) ───────────────────────────────────────── + + +def _kde_windows() -> Union[List[Window], ToolResult]: + done = _os.run(["kdotool", "search", "--name", "."]) + if not done.ok: + return _os.fail("window_control", f"Couldn't list windows: {done.error()}") + windows: List[Window] = [] + for window_id in done.stdout.split(): + name = _os.run(["kdotool", "getwindowname", window_id]) + if name.ok and name.stdout.strip(): + windows.append((window_id, name.stdout.strip())) + return windows + + +def _kde_act(window_id: str, title: str, action: str, placement: Optional[dict] = None): + if action in ("focus", "restore"): + steps = [["kdotool", "windowactivate", window_id]] + elif action == "minimize": + steps = [["kdotool", "windowminimize", window_id]] + elif action == "maximize": + steps = [["kdotool", "windowstate", "--add", "MAXIMIZED", window_id]] + else: + placement = placement or {} + if action == "move": + steps = [ + [ + "kdotool", + "windowmove", + window_id, + str(placement["x"]), + str(placement["y"]), + ] + ] + elif action == "resize": + steps = [ + [ + "kdotool", + "windowsize", + window_id, + str(placement["width"]), + str(placement["height"]), + ] + ] + else: + return "Snapping isn't available through kdotool; use KDE's own shortcuts." + for step in steps: + done = _os.run(step) + if not done.ok: + return f"Couldn't change {title}: {done.error()}" + return None + + +# ── Choosing one ───────────────────────────────────────────────── + +_BACKENDS: Dict[str, Tuple[str, Backend]] = { + "sway": ("swaymsg", (lambda: _sway_windows(), lambda *a: _sway_act(*a))), + "hyprland": ("hyprctl", (lambda: _hypr_windows(), lambda *a: _hypr_act(*a))), + "kde": ("kdotool", (lambda: _kde_windows(), lambda *a: _kde_act(*a))), +} + + +def backend() -> Union[Backend, ToolResult]: + """The running compositor's backend, or a result explaining what's missing.""" + if os.environ.get("SWAYSOCK"): + name = "sway" + elif os.environ.get("HYPRLAND_INSTANCE_SIGNATURE"): + name = "hyprland" + elif "kde" in os.environ.get("XDG_CURRENT_DESKTOP", "").lower(): + name = "kde" + else: + desktop = os.environ.get("XDG_CURRENT_DESKTOP") or "this desktop" + return _os.fail( + "window_control", + f"Window control on Wayland works on Sway, Hyprland and KDE Plasma," + f" not {desktop}. GNOME doesn't let apps control other windows.", + ) + command, functions = _BACKENDS[name] + if not _os.has_command(command): + hint = " (install kdotool)" if name == "kde" else "" + return _os.fail("window_control", f"{command} isn't available{hint}.") + return functions + + +__all__ = ["backend", "is_wayland"] diff --git a/src/locus/tools/desktop/window_control.py b/src/locus/tools/desktop/window_control.py index d36404d..2b5cda8 100644 --- a/src/locus/tools/desktop/window_control.py +++ b/src/locus/tools/desktop/window_control.py @@ -1,8 +1,8 @@ -"""List, focus, minimize, maximize and restore windows. +"""List, focus, minimize, maximize, restore, move, resize and snap windows. Windows uses the window API through ctypes, macOS uses System Events (Locus -needs Accessibility permission), and Linux uses wmctrl, which works on X11 -but not Wayland. +needs Accessibility permission), and Linux uses wmctrl on X11 or the +compositor's own tool on Wayland (see ``wayland.py``). """ from __future__ import annotations @@ -15,15 +15,104 @@ from locus.tools._stubs import BaseTool, ToolSpec from locus.tools.desktop import _os -ACTIONS = ("list", "focus", "minimize", "maximize", "restore") +ACTIONS = ( + "list", + "focus", + "minimize", + "maximize", + "restore", + "move", + "resize", + "snap", +) +PLACE_ACTIONS = ("move", "resize", "snap") +SNAP_POSITIONS = ( + "left", + "right", + "top", + "bottom", + "top_left", + "top_right", + "bottom_left", + "bottom_right", + "center", +) +#: Smallest window placement Locus will make, so a window can't vanish. +MIN_SIZE = 200 _PAST = { "focus": "Focused", "minimize": "Minimized", "maximize": "Maximized", "restore": "Restored", + "move": "Moved", + "resize": "Resized", + "snap": "Snapped", } +#: (left, top, width, height) in screen pixels. +Rect = Tuple[int, int, int, int] + + +def placement_rect(action: str, placement: dict, current: Rect, work: Rect) -> Rect: + """Where a move, resize or snap puts a window. + + ``work`` is the usable area of the window's screen (without the taskbar + or menu bar). Move coordinates are relative to it, and the result always + stays inside it. + """ + wl, wt, ww, wh = work + cx, cy, cw, ch = current + if action == "snap": + position = placement["position"] + half_w, half_h = ww // 2, wh // 2 + if position == "center": + w, h = ww * 2 // 3, wh * 2 // 3 + return wl + (ww - w) // 2, wt + (wh - h) // 2, w, h + x = wl + (half_w if "right" in position else 0) + y = wt + (half_h if "bottom" in position else 0) + w = ww if position in ("top", "bottom") else half_w + h = wh if position in ("left", "right") else half_h + if "right" in position: + w = ww - half_w + if "bottom" in position: + h = wh - half_h + return x, y, w, h + if action == "resize": + w = max(MIN_SIZE, min(int(placement["width"]), ww)) + h = max(MIN_SIZE, min(int(placement["height"]), wh)) + x, y = cx, cy + else: # move + w, h = min(cw, ww), min(ch, wh) + x, y = wl + int(placement["x"]), wt + int(placement["y"]) + x = max(wl, min(x, wl + ww - w)) + y = max(wt, min(y, wt + wh - h)) + return x, y, w, h + + +def _number(raw: Any) -> Union[int, None]: + try: + return int(round(float(raw))) + except (TypeError, ValueError): + return None + + +def _placement(action: str, params: dict) -> Union[dict, str]: + """The move, resize or snap arguments, or a message saying what's wrong.""" + if action == "snap": + position = params.get("position") + if position not in SNAP_POSITIONS: + return f"Give a position: {', '.join(SNAP_POSITIONS)}." + return {"position": position} + keys = ("x", "y") if action == "move" else ("width", "height") + values = {key: _number(params.get(key)) for key in keys} + if any(v is None for v in values.values()): + return f"Give {' and '.join(keys)} in pixels." + if action == "move" and any(v < 0 for v in values.values()): + return "x and y start at the screen's top-left corner; they can't be negative." + return values + + #: (handle, title): the handle is whatever the OS needs to act on the window. Window = Tuple[Any, str] @@ -87,7 +176,45 @@ def _apply(hwnd: int, action: str) -> bool: return True -def _windows_act(hwnd: int, title: str, action: str) -> Union[None, str]: +def _windows_place(hwnd: int, action: str, placement: dict) -> bool: + import ctypes + from ctypes import wintypes + + class MONITORINFO(ctypes.Structure): + _fields_ = [ + ("cbSize", wintypes.DWORD), + ("rcMonitor", wintypes.RECT), + ("rcWork", wintypes.RECT), + ("dwFlags", wintypes.DWORD), + ] + + user32 = ctypes.windll.user32 # type: ignore[attr-defined] + if user32.IsIconic(hwnd) or user32.IsZoomed(hwnd): + user32.ShowWindow(hwnd, _SW["restore"]) + rect = wintypes.RECT() + user32.GetWindowRect(hwnd, ctypes.byref(rect)) + info = MONITORINFO() + info.cbSize = ctypes.sizeof(MONITORINFO) + monitor = user32.MonitorFromWindow(hwnd, 2) # MONITOR_DEFAULTTONEAREST + user32.GetMonitorInfoW(monitor, ctypes.byref(info)) + work = info.rcWork + x, y, w, h = placement_rect( + action, + placement, + (rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top), + (work.left, work.top, work.right - work.left, work.bottom - work.top), + ) + # SWP_NOZORDER | SWP_NOACTIVATE: change only position and size. + return bool(user32.SetWindowPos(hwnd, 0, x, y, w, h, 0x0014)) + + +def _windows_act( + hwnd: int, title: str, action: str, placement: Union[dict, None] = None +) -> Union[None, str]: + if action in PLACE_ACTIONS: + if not _windows_place(hwnd, action, placement or {}): + return f"Windows didn't let Locus move {title}." + return None if not _apply(hwnd, action): return f"Windows didn't let Locus focus {title}." return None @@ -128,6 +255,36 @@ def _windows_act(hwnd: int, title: str, action: str) -> Union[None, str]: end run""" +_MAC_PLACE = """on run argv + set appName to item 1 of argv + set winName to item 2 of argv + set x to (item 3 of argv) as integer + set y to (item 4 of argv) as integer + set w to (item 5 of argv) as integer + set h to (item 6 of argv) as integer + tell application "System Events" + tell process appName + set win to (first window whose name is winName) + set value of attribute "AXMinimized" of win to false + if x >= 0 and y >= 0 then set position of win to {x, y} + if w > 0 and h > 0 then set size of win to {w, h} + end tell + end tell +end run""" +_MAC_SCREEN = 'tell application "Finder" to get bounds of window of desktop' +_MAC_MENU_BAR = 25 + + +def _mac_work_area() -> Union[Rect, None]: + done = _os.run(["osascript", "-e", _MAC_SCREEN]) + try: + left, top, right, bottom = (int(v) for v in done.stdout.split(",")) + except ValueError: + return None + top = max(top, _MAC_MENU_BAR) + return left, top, right - left, bottom - top + + def _mac_windows() -> Union[List[Window], ToolResult]: done = _os.run(["osascript", "-e", _MAC_LIST]) if not done.ok: @@ -144,8 +301,30 @@ def _mac_windows() -> Union[List[Window], ToolResult]: return windows -def _mac_act(handle: Tuple[str, str], title: str, action: str) -> Union[None, str]: +def _mac_act( + handle: Tuple[str, str], + title: str, + action: str, + placement: Union[dict, None] = None, +) -> Union[None, str]: app, name = handle + if action in PLACE_ACTIONS: + placement = placement or {} + if action == "resize": + x, y = -1, -1 + w, h = (max(MIN_SIZE, placement[k]) for k in ("width", "height")) + else: + work = _mac_work_area() + if work is None: + return "Couldn't read the screen size." + if action == "move": + x, y = work[0] + placement["x"], work[1] + placement["y"] + w = h = -1 + else: + x, y, w, h = placement_rect("snap", placement, work, work) + args = [str(v) for v in (x, y, w, h)] + done = _os.run(["osascript", "-e", _MAC_PLACE, app, name, *args]) + return None if done.ok else f"Couldn't move {title}: {done.error()}" done = _os.run(["osascript", "-e", _MAC_ACT, app, name, action]) return None if done.ok else f"Couldn't change {title}: {done.error()}" @@ -175,7 +354,45 @@ def _linux_windows() -> Union[List[Window], ToolResult]: return windows -def _linux_act(window_id: str, title: str, action: str) -> Union[None, str]: +def _linux_work_area() -> Union[Rect, None]: + done = _os.run(["wmctrl", "-d"]) + for line in done.stdout.splitlines(): + parts = line.split() + if len(parts) > 1 and parts[1] == "*" and "WA:" in parts: + i = parts.index("WA:") + try: + x, y = (int(v) for v in parts[i + 1].split(",")) + w, h = (int(v) for v in parts[i + 2].split("x")) + except (IndexError, ValueError): + return None + return x, y, w, h + return None + + +def _linux_act( + window_id: str, title: str, action: str, placement: Union[dict, None] = None +) -> Union[None, str]: + if action in PLACE_ACTIONS: + placement = placement or {} + if action == "resize": + geometry = ( + -1, + -1, + *(max(MIN_SIZE, placement[k]) for k in ("width", "height")), + ) + else: + work = _linux_work_area() + if work is None: + return "Couldn't read the screen's work area from wmctrl." + if action == "move": + geometry = (work[0] + placement["x"], work[1] + placement["y"], -1, -1) + else: + geometry = placement_rect("snap", placement, work, work) + _os.run(["wmctrl", "-i", "-r", window_id, "-b", _WMCTRL_STATES["restore"]]) + done = _os.run( + ["wmctrl", "-i", "-r", window_id, "-e", "0," + ",".join(map(str, geometry))] + ) + return None if done.ok else f"Couldn't move {title}: {done.error()}" if action in _WMCTRL_STATES: done = _os.run(["wmctrl", "-i", "-r", window_id, "-b", _WMCTRL_STATES[action]]) if not done.ok: @@ -209,9 +426,13 @@ def spec(self) -> ToolSpec: name="window_control", description=( "Work with open windows. 'list' shows window titles. 'focus'," - " 'minimize', 'maximize' and 'restore' act on the first window" - " whose title contains 'title' (not case-sensitive). To close" - " an app, use app_control instead." + " 'minimize', 'maximize', 'restore', 'move', 'resize' and 'snap'" + " act on the first window whose title contains 'title' (not" + " case-sensitive). 'move' takes x and y and 'resize' takes width" + " and height, in pixels from the top-left of the screen's usable" + " area. 'snap' puts the window in a 'position' (left, right, top," + " bottom, top_left, top_right, bottom_left, bottom_right or" + " center). To close an app, use app_control instead." ), parameters={ "type": "object", @@ -221,6 +442,11 @@ def spec(self) -> ToolSpec: "type": "string", "description": "Part of the window's title, like 'Notepad'.", }, + "x": {"type": "number"}, + "y": {"type": "number"}, + "width": {"type": "number"}, + "height": {"type": "number"}, + "position": {"type": "string", "enum": list(SNAP_POSITIONS)}, }, "required": ["action"], }, @@ -237,7 +463,19 @@ def execute(self, **params: Any) -> ToolResult: platform = _PLATFORMS.get(_os.system()) if platform is None: return _os.unsupported("window_control", "Controlling windows") + if _os.system() == "linux": + from locus.tools.desktop import wayland + + if wayland.is_wayland(): + platform = wayland.backend() + if isinstance(platform, ToolResult): + return platform list_fn, act_fn = platform + placement = None + if action in PLACE_ACTIONS: + placement = _placement(action, params) + if isinstance(placement, str): + return _os.fail("window_control", placement) windows = list_fn() if isinstance(windows, ToolResult): @@ -256,10 +494,19 @@ def execute(self, **params: Any) -> ToolResult: "window_control", f"No open window has {wanted!r} in its title." ) handle, title = match - error = act_fn(handle, title, action) + error = ( + act_fn(handle, title, action, placement) + if placement is not None + else act_fn(handle, title, action) + ) if error: return _os.fail("window_control", error) - return _os.ok("window_control", f"{_PAST[action]} {title}.") + where = ( + f" to the {placement['position'].replace('_', ' ')}" + if action == "snap" + else "" + ) + return _os.ok("window_control", f"{_PAST[action]} {title}{where}.") __all__ = ["WindowControlTool", "list_windows"] diff --git a/tests/server/test_agent_mode.py b/tests/server/test_agent_mode.py index c5df629..e2c9808 100644 --- a/tests/server/test_agent_mode.py +++ b/tests/server/test_agent_mode.py @@ -340,3 +340,82 @@ def test_agents_send_input_images_with_the_last_message(): assert with_images[-1].content == "what is this?" assert with_images[-1].images == ["aW1n"] assert without[-1].images is None + + +def test_tool_images_are_shown_only_to_vision_models(): + from locus.agents.orchestrator import OrchestratorAgent + from locus.core.types import Message, Role, ToolResult + + shot = ToolResult( + tool_name="screenshot", content="ok", metadata={"images": ["aW1n"]} + ) + plain = ToolResult(tool_name="file_read", content="text") + + messages: list = [] + OrchestratorAgent._attach_tool_images(messages, [plain, shot], vision=True) + assert len(messages) == 1 + assert messages[0].role == Role.USER + assert messages[0].images == ["aW1n"] + + blind: list[Message] = [] + OrchestratorAgent._attach_tool_images(blind, [shot], vision=False) + OrchestratorAgent._attach_tool_images(blind, [plain], vision=True) + assert blind == [] + + +@pytest.mark.parametrize("sees", [True, False]) +def test_agent_mode_tells_the_agent_whether_the_model_sees_images(monkeypatch, sees): + agent = FakeAgent() + monkeypatch.setattr(task_agent, "_build_task_agent", lambda state: agent) + + async def accepts(engine, model): + return sees + + monkeypatch.setattr("locus.server.routes._model_accepts_images", accepts) + app = create_app(_engine([]), "test-model", config=_config()) + + _post(TestClient(app), mode="agent") + + assert agent.contexts[0].metadata["vision"] is sees + + +@pytest.mark.parametrize( + "installed, ram, model, is_installed", + [ + ([], 31, "qwen3.5:9b", False), + (["qwen3.5:9b"], 16, "qwen3.5:9b", True), + (["qwen3.5:4b", "llama3.1:8b", "nomic-embed-text"], 16, "llama3.1:8b", True), + (["qwen3.5:27b"], 16, "qwen3.5:9b", False), # 27b doesn't fit in 16 GB + (["qwen3.5:27b", "qwen3.5:9b"], 32, "qwen3.5:27b", True), + ([], 8, "qwen3.5:4b", False), + ([], 6, "", False), + ], +) +def test_agent_model_recommendation(installed, ram, model, is_installed): + from locus.server.model_capabilities import agent_model_recommendation + + rec = agent_model_recommendation(installed, ram) + + assert rec["model"] == model + assert rec["installed"] is is_installed + assert rec["reason"] + + +def test_agent_model_endpoint(monkeypatch): + import psutil + + engine = _engine([]) + engine.list_models.return_value = ["qwen3.5:4b"] + monkeypatch.setattr( + psutil, "virtual_memory", lambda: SimpleNamespace(total=16 * 1024**3) + ) + client = TestClient(create_app(engine, "test-model", config=_config())) + + body = client.get("/v1/agent-model").json() + + assert body == { + "model": "qwen3.5:9b", + "installed": False, + "ram_gb": 16, + "reason": "qwen3.5:9b handles tools reliably on 16 GB of memory.", + } diff --git a/tests/server/test_uninstall_models.py b/tests/server/test_uninstall_models.py new file mode 100644 index 0000000..f151fe0 --- /dev/null +++ b/tests/server/test_uninstall_models.py @@ -0,0 +1,134 @@ +"""Listing and uninstalling local (Ollama) models.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +fastapi = pytest.importorskip("fastapi") +httpx = pytest.importorskip("httpx") + +from fastapi.testclient import TestClient # noqa: E402 + +from locus.server import routes # noqa: E402 +from locus.server.app import create_app # noqa: E402 + +TAGS = { + "models": [ + { + "name": "qwen3.5:9b", + "size": 6_600_000_000, + "modified_at": "2026-11-09T10:00:00Z", + "details": {"parameter_size": "9B"}, + }, + {"name": "qwen3.5:4b", "size": 3_400_000_000, "details": {}}, + ] +} + + +class FakeOllama: + """Stands in for httpx.AsyncClient talking to Ollama.""" + + deleted: list = [] + missing: set = set() + down = False + + def __init__(self, base_url="", timeout=0): + self.base_url = base_url + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + def _response(self, method, path, status=200, body=None): + request = httpx.Request(method, f"{self.base_url}{path}") + return httpx.Response(status, json=body or {}, request=request) + + async def get(self, path): + if self.down: + raise httpx.ConnectError("refused") + return self._response("GET", path, body=TAGS) + + async def request(self, method, path, json=None): + if self.down: + raise httpx.ConnectError("refused") + name = json["name"] + if name in self.missing: + return self._response(method, path, 404, {"error": "model not found"}) + type(self).deleted.append(name) + return self._response(method, path) + + +@pytest.fixture +def client(monkeypatch): + FakeOllama.deleted = [] + FakeOllama.missing = set() + FakeOllama.down = False + monkeypatch.setattr(httpx, "AsyncClient", FakeOllama) + engine = SimpleNamespace(engine_id="ollama", _host="http://ollama:11434") + from locus.core.config import JarvisConfig + + cfg = JarvisConfig() + cfg.analytics.enabled = False + cfg.traces.enabled = False + return TestClient(create_app(engine, "qwen3.5:4b", config=cfg)) + + +def test_installed_models_have_sizes_and_the_default(client): + body = client.get("/v1/models/installed").json() + + assert body["total_bytes"] == 10_000_000_000 + assert [m["name"] for m in body["models"]] == ["qwen3.5:4b", "qwen3.5:9b"] + nine = body["models"][1] + assert nine["size_bytes"] == 6_600_000_000 + assert nine["parameter_size"] == "9B" + assert [m["is_default"] for m in body["models"]] == [True, False] + + +def test_uninstall_reports_the_space_freed(client): + routes._vision_cache["qwen3.5:9b"] = True + + body = client.delete("/v1/models/qwen3.5:9b").json() + + assert body == { + "status": "deleted", + "model": "qwen3.5:9b", + "freed_bytes": 6_600_000_000, + "was_default": False, + } + assert FakeOllama.deleted == ["qwen3.5:9b"] + assert "qwen3.5:9b" not in routes._vision_cache + + +def test_uninstalling_the_default_model_says_so(client): + assert client.delete("/v1/models/qwen3.5:4b").json()["was_default"] is True + + +def test_uninstalling_a_missing_model_is_404(client): + FakeOllama.missing = {"gone:1b"} + resp = client.delete("/v1/models/gone:1b") + assert resp.status_code == 404 + assert resp.json()["detail"] == "gone:1b isn't installed." + + +def test_ollama_down_is_502(client): + FakeOllama.down = True + assert client.get("/v1/models/installed").status_code == 502 + assert client.delete("/v1/models/qwen3.5:9b").status_code == 502 + + +def test_other_engines_cant_uninstall(monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", FakeOllama) + engine = SimpleNamespace(engine_id="vllm") + from locus.core.config import JarvisConfig + + cfg = JarvisConfig() + cfg.analytics.enabled = False + cfg.traces.enabled = False + client = TestClient(create_app(engine, "m", config=cfg)) + + assert client.get("/v1/models/installed").status_code == 501 + assert client.delete("/v1/models/m").status_code == 501 diff --git a/tests/tools/test_desktop_tools.py b/tests/tools/test_desktop_tools.py index 8a3bec1..f6b7960 100644 --- a/tests/tools/test_desktop_tools.py +++ b/tests/tools/test_desktop_tools.py @@ -24,6 +24,19 @@ ) +@pytest.fixture(autouse=True) +def _no_wayland(monkeypatch): + """Linux tests mean X11 unless they set a Wayland session themselves.""" + for name in ( + "WAYLAND_DISPLAY", + "XDG_SESSION_TYPE", + "SWAYSOCK", + "HYPRLAND_INSTANCE_SIGNATURE", + "XDG_CURRENT_DESKTOP", + ): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture(autouse=True) def _isolate_reminders(tmp_path, monkeypatch): """Keep reminders out of the real ``~/.locus`` and drop leftover timers.""" @@ -557,3 +570,292 @@ def test_radio_failures_are_reported(monkeypatch): def test_radio_risk(action, risk): tool = SystemControlTool() assert tool_risk("system_control", tool, {"action": action}) == risk + + +# ── Moving, resizing and snapping windows ──────────────────────── + +WORK = (0, 40, 1600, 900) # a screen with a 40 px bar on top + + +@pytest.mark.parametrize( + "position, rect", + [ + ("left", (0, 40, 800, 900)), + ("right", (800, 40, 800, 900)), + ("top", (0, 40, 1600, 450)), + ("bottom", (0, 490, 1600, 450)), + ("top_left", (0, 40, 800, 450)), + ("bottom_right", (800, 490, 800, 450)), + ("center", (267, 190, 1066, 600)), + ], +) +def test_snap_rects(position, rect): + current = (10, 50, 500, 400) + assert ( + window_control.placement_rect("snap", {"position": position}, current, WORK) + == rect + ) + + +def test_snapping_an_odd_sized_screen_covers_it(): + work = (0, 0, 1365, 767) + left = window_control.placement_rect("snap", {"position": "left"}, work, work) + right = window_control.placement_rect("snap", {"position": "right"}, work, work) + assert left[2] + right[2] == 1365 + assert right[0] == left[2] + + +def test_moves_and_resizes_stay_on_screen(): + current = (100, 100, 600, 400) + place = window_control.placement_rect + assert place("move", {"x": 50, "y": 20}, current, WORK) == (50, 60, 600, 400) + # Too far right or down: pulled back so the whole window shows. + assert place("move", {"x": 5000, "y": 5000}, current, WORK) == (1000, 540, 600, 400) + assert place("resize", {"width": 800, "height": 500}, current, WORK) == ( + 100, + 100, + 800, + 500, + ) + # Bigger than the screen, or tiny: limited to the screen and MIN_SIZE. + assert place("resize", {"width": 9999, "height": 10}, current, WORK) == ( + 0, + 100, + 1600, + window_control.MIN_SIZE, + ) + + +@pytest.mark.parametrize( + "params, message", + [ + ({"action": "snap", "title": "x", "position": "middle"}, "Give a position"), + ({"action": "move", "title": "x", "x": 10}, "Give x and y"), + ({"action": "move", "title": "x", "x": -5, "y": 0}, "can't be negative"), + ({"action": "resize", "title": "x", "width": "big", "height": 4}, "Give width"), + ], +) +def test_placement_arguments_are_checked(monkeypatch, params, message): + FakeOS(monkeypatch, "windows") + result = WindowControlTool().execute(**params) + assert not result.success + assert message in result.content + + +def test_snapping_on_windows(monkeypatch): + FakeOS(monkeypatch, "windows") + placed = [] + monkeypatch.setattr( + window_control, "list_windows", lambda: [(7, "notes.txt - Notepad")] + ) + monkeypatch.setattr( + window_control, + "_windows_place", + lambda hwnd, action, placement: ( + placed.append((hwnd, action, placement)) or True + ), + ) + + result = WindowControlTool().execute( + action="snap", title="notepad", position="top_right" + ) + + assert result.content == "Snapped notes.txt - Notepad to the top right." + assert placed == [(7, "snap", {"position": "top_right"})] + + +def test_snapping_on_linux_uses_the_work_area(monkeypatch): + fake = FakeOS(monkeypatch, "linux") + outputs = { + "-l": "0x05 0 host Editor\n", + "-d": "0 * DG: 1920x1080 VP: 0,0 WA: 0,27 1920x1053 Workspace 1\n", + } + + def run(args, *, env=None, input=None, timeout=0): + fake.calls.append({"args": args}) + return _os.Completed(0, outputs.get(args[1], ""), "") + + monkeypatch.setattr(_os, "run", run) + + assert ( + WindowControlTool() + .execute(action="snap", title="editor", position="left") + .success + ) + assert fake.calls[-1]["args"] == [ + "wmctrl", + "-i", + "-r", + "0x05", + "-e", + "0,0,27,960,1053", + ] + WindowControlTool().execute(action="resize", title="editor", width=900, height=700) + assert fake.calls[-1]["args"][-1] == "0,-1,-1,900,700" + + +def test_moving_on_macos(monkeypatch): + fake = FakeOS(monkeypatch, "macos") + + def run(args, *, env=None, input=None, timeout=0): + fake.calls.append({"args": args}) + script = args[2] if len(args) > 2 else "" + if "bounds of window of desktop" in script: + return _os.Completed(0, "0, 0, 1440, 900", "") + if "repeat with p in" in script: + return _os.Completed(0, "Notes\tShopping list\n", "") + return _os.Completed(0, "", "") + + monkeypatch.setattr(_os, "run", run) + + assert ( + WindowControlTool().execute(action="move", title="shopping", x=10, y=10).success + ) + assert fake.calls[-1]["args"][-6:] == [ + "Notes", + "Shopping list", + "10", + "35", + "-1", + "-1", + ] + + +# ── Wayland ────────────────────────────────────────────────────── + + +def _scripted_run(monkeypatch, fake, outputs): + """Answer commands by their first two words; record every call.""" + + def run(args, *, env=None, input=None, timeout=0): + fake.calls.append({"args": args}) + return _os.Completed(0, outputs.get(" ".join(args[:2]), ""), "") + + monkeypatch.setattr(_os, "run", run) + + +def test_sway_windows_are_listed_and_snapped(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-1") + monkeypatch.setenv("SWAYSOCK", "/run/sway.sock") + fake = FakeOS(monkeypatch, "linux") + tree = { + "type": "root", + "nodes": [ + { + "type": "workspace", + "nodes": [ + { + "type": "con", + "id": 12, + "name": "notes - Kate", + "rect": {"x": 0, "y": 0, "width": 900, "height": 700}, + "nodes": [], + } + ], + "floating_nodes": [], + } + ], + } + _scripted_run(monkeypatch, fake, {"swaymsg -t": json.dumps(tree)}) + monkeypatch.setattr( + "locus.tools.desktop.wayland._sway_output_rect", lambda: (0, 0, 1920, 1080) + ) + + listed = WindowControlTool().execute(action="list") + snapped = WindowControlTool().execute(action="snap", title="kate", position="left") + + assert listed.metadata["titles"] == ["notes - Kate"] + assert snapped.success + command = fake.calls[-1]["args"][1] + assert "[con_id=12] floating enable" in command + assert "resize set 960 px 1080 px" in command + assert "move absolute position 0 px 0 px" in command + + +def test_hyprland_uses_hyprctl(monkeypatch): + monkeypatch.setenv("XDG_SESSION_TYPE", "wayland") + monkeypatch.setenv("HYPRLAND_INSTANCE_SIGNATURE", "abc") + fake = FakeOS(monkeypatch, "linux") + clients = [ + {"address": "0xa1", "title": "Firefox", "at": [0, 0], "size": [800, 600]} + ] + _scripted_run(monkeypatch, fake, {"hyprctl clients": json.dumps(clients)}) + + assert WindowControlTool().execute(action="minimize", title="fire").success + assert fake.calls[-1]["args"] == [ + "hyprctl", + "--batch", + "dispatch movetoworkspacesilent special:minimized,address:0xa1", + ] + + +def test_kde_uses_kdotool(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + monkeypatch.setenv("XDG_CURRENT_DESKTOP", "KDE") + fake = FakeOS(monkeypatch, "linux") + _scripted_run( + monkeypatch, + fake, + {"kdotool search": "{aa}\n", "kdotool getwindowname": "Dolphin"}, + ) + + assert WindowControlTool().execute(action="focus", title="dolphin").success + assert fake.calls[-1]["args"] == ["kdotool", "windowactivate", "{aa}"] + + +def test_gnome_on_wayland_is_explained(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + monkeypatch.setenv("XDG_CURRENT_DESKTOP", "GNOME") + FakeOS(monkeypatch, "linux") + result = WindowControlTool().execute(action="list") + assert not result.success + assert "GNOME" in result.content + + +def test_wayland_tool_must_be_installed(monkeypatch): + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-0") + monkeypatch.setenv("XDG_CURRENT_DESKTOP", "KDE") + FakeOS(monkeypatch, "linux") + monkeypatch.setattr(_os, "has_command", lambda name: False) + assert "install kdotool" in WindowControlTool().execute(action="list").content + + +# ── Screenshots the model can see ──────────────────────────────── + + +def test_screenshot_look_attaches_a_small_jpeg(monkeypatch, tmp_path): + import base64 + import io + + from PIL import Image + + monkeypatch.setenv("LOCUS_HOME", str(tmp_path)) + fake = FakeOS(monkeypatch, "windows") + fake.on_call = lambda call: Image.new("RGB", (3000, 1500), "white").save( + call["args"]["path"] + ) + + plain = ScreenshotTool().execute() + looked = ScreenshotTool().execute(look=True) + + assert "images" not in plain.metadata + assert "attached" in looked.content + data = base64.b64decode(looked.metadata["images"][0]) + with Image.open(io.BytesIO(data)) as image: + assert image.format == "JPEG" + assert max(image.size) == 1280 + + +def test_screenshot_look_without_pillow(monkeypatch, tmp_path): + from locus.tools.desktop import screenshot + + monkeypatch.setenv("LOCUS_HOME", str(tmp_path)) + fake = FakeOS(monkeypatch, "windows") + fake.on_call = lambda call: Path(call["args"]["path"]).write_bytes(b"png") + monkeypatch.setattr(screenshot, "image_for_model", lambda path: None) + + result = ScreenshotTool().execute(look=True) + + assert result.success + assert "couldn't prepare it" in result.content + assert "images" not in result.metadata