From ac8e49c1a57d82b0c5f88b17d8e4809789bd32d3 Mon Sep 17 00:00:00 2001 From: Aryan Kumar Date: Sun, 1 Nov 2026 21:35:02 +0530 Subject: [PATCH 1/4] feat(chat): attach files and images, and search the web Attachments - New POST /v1/attachments reads uploaded files (nothing is stored): text from PDF, Office Open XML, OpenDocument, EPUB, HTML, RTF, notebooks and anything that decodes as text; images are converted to a resized JPEG. Other binary files are described by name and type. - Chat messages carry `attachments` (documents) and `images`. The server folds documents into the prompt within a size budget that gives the newest message priority, and forwards images to vision models. For an Ollama model without the vision capability, images are replaced with a note so the model can suggest a vision model. - The chat box gets an attach button, drag and drop, and paste. Sent messages show the files. Their content is kept in IndexedDB rather than localStorage, and deleted with the conversation. Web search - New Web search toggle (off by default, remembered). With it on, the server searches for the latest message, reads the top pages, and adds them as context the model cites as [1], [2]. A `web_search` SSE event sends the sources first so the answer's citations link to them. - Requests with web search or attachments go straight to the engine, since agents only see the latest message's text. Also send images in the right shape on the direct Ollama and OpenAI paths, and add pdfplumber and Pillow to the desktop and server extras. --- README.md | 3 + .../src/components/Chat/AttachmentChip.tsx | 77 +++ frontend/src/components/Chat/InputArea.tsx | 265 ++++++++- .../src/components/Chat/MessageBubble.tsx | 42 +- frontend/src/components/Chat/WebSources.tsx | 54 ++ frontend/src/lib/attachments.test.ts | 50 ++ frontend/src/lib/attachments.ts | 191 +++++++ frontend/src/lib/sse.ts | 11 +- frontend/src/lib/store.ts | 35 ++ frontend/src/types/index.ts | 25 + pyproject.toml | 6 + src/locus/server/app.py | 2 + src/locus/server/attachments.py | 522 ++++++++++++++++++ src/locus/server/cloud_router.py | 28 +- src/locus/server/models.py | 16 + src/locus/server/routes.py | 174 +++++- src/locus/server/web_context.py | 174 ++++++ tests/server/test_attachments.py | 306 ++++++++++ tests/server/test_chat_extras.py | 251 +++++++++ uv.lock | 8 + 20 files changed, 2189 insertions(+), 51 deletions(-) create mode 100644 frontend/src/components/Chat/AttachmentChip.tsx create mode 100644 frontend/src/components/Chat/WebSources.tsx create mode 100644 frontend/src/lib/attachments.test.ts create mode 100644 frontend/src/lib/attachments.ts create mode 100644 src/locus/server/attachments.py create mode 100644 src/locus/server/web_context.py create mode 100644 tests/server/test_attachments.py create mode 100644 tests/server/test_chat_extras.py diff --git a/README.md b/README.md index f62c5d0..5b251ac 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Locus is that stack. It is a framework for local-first personal AI, built around ## What Locus can do - **Chat on your PC:** talk to a local AI model in the desktop app, your browser, or the terminal, by typing or by voice. +- **Attach files and images:** drop documents (PDF, Word, Excel, PowerPoint, OpenDocument, EPUB, HTML, CSV, code, plain text and more) or images into a chat and ask about them. Images need a vision model such as `qwen3.5` or `gemma3`. +- **Search the web:** turn on **Web search** and Locus looks up your question online first, reads the top pages, and answers with numbered links to its sources. It's off until you turn it on, since it sends your question to a search engine. - **Work with your files:** read and write files and PDFs, search your own documents, run code in a sandbox, and remember facts between conversations. - **Connect your apps:** Gmail, Outlook, Google Calendar, Google Drive, Dropbox, Notion, Obsidian, GitHub, Slack and more, each only after you sign in. - **Reply on messaging apps:** run it as a bot on Telegram, Slack, Discord, WhatsApp, Teams, Signal, email, SMS and about 25 other channels. @@ -188,6 +190,7 @@ Full documentation—including Docker deployment, cloud engines, development set - **Local by default:** models run on your machine through Ollama or another local engine. Nothing goes to a cloud model unless you add your own API key. - **No analytics:** Locus ships with no analytics destination, so no usage data is sent anywhere. See [telemetry](https://aryanjsx.github.io/Locus/telemetry/) for details. +- **Attachments stay local:** attached files are read by Locus on your machine and kept in the app's own storage. Web search is off by default; when you turn it on, only your question goes to the search engine, never your files. - **Opt-in connections:** email, calendar, messaging and other services only work after you connect them, and the data you index stays on your machine. - **Approval for risky actions:** shell commands and similar tools go through an approval step first. diff --git a/frontend/src/components/Chat/AttachmentChip.tsx b/frontend/src/components/Chat/AttachmentChip.tsx new file mode 100644 index 0000000..b4bf301 --- /dev/null +++ b/frontend/src/components/Chat/AttachmentChip.tsx @@ -0,0 +1,77 @@ +import { AlertCircle, FileText, Loader2, X } from 'lucide-react'; +import { formatBytes } from '../../lib/attachments'; +import type { AttachmentMeta } from '../../types'; + +interface Props { + name: string; + meta?: AttachmentMeta; + status?: 'reading' | 'ready' | 'error'; + error?: string; + onRemove?: () => void; +} + +/** One attached file: a thumbnail or file icon, its name, and what it is. */ +export function AttachmentChip({ name, meta, status = 'ready', error, onRemove }: Props) { + const problem = status === 'error' ? error : meta?.kind === 'unsupported' ? meta.note : undefined; + const detail = + status === 'reading' + ? 'Reading...' + : status === 'error' + ? "Couldn't read" + : meta + ? [meta.format, formatBytes(meta.size), meta.truncated ? 'long, will be shortened' : ''] + .filter(Boolean) + .join(' · ') + : ''; + + return ( +
+ {meta?.thumbnail ? ( + + ) : ( + + {status === 'reading' ? ( + + ) : problem ? ( + + ) : ( + + )} + + )} + + + {name} + + + {detail} + + + {onRemove && ( + + )} +
+ ); +} diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx index d0b2855..541f5c0 100644 --- a/frontend/src/components/Chat/InputArea.tsx +++ b/frontend/src/components/Chat/InputArea.tsx @@ -1,5 +1,5 @@ import { useState, useRef, useCallback, useEffect } from 'react'; -import { Send, Square, Paperclip, Search } from 'lucide-react'; +import { Send, Square, Paperclip, Search, Globe } from 'lucide-react'; import { toast } from 'sonner'; import { useAppStore, generateId } from '../../lib/store'; import { streamChat, streamResearch } from '../../lib/sse'; @@ -11,8 +11,18 @@ import { resolveChatEngine, } from '../../lib/chat-telemetry'; import { MicButton } from './MicButton'; +import { AttachmentChip } from './AttachmentChip'; import { useSpeech } from '../../hooks/useSpeech'; +import { + apiAttachmentFields, + deletePayloads, + MAX_FILE_BYTES, + MAX_FILES_PER_MESSAGE, + readAttachments, + savePayload, +} from '../../lib/attachments'; import type { + AttachmentMeta, ChatMessage, MessageTelemetry, ResearchSearchTrace, @@ -21,6 +31,15 @@ import type { ToolCallInfo, } from '../../types'; +/** A file in the input box: being read by the server, ready, or failed. */ +interface PendingAttachment { + key: string; + name: string; + status: 'reading' | 'ready' | 'error'; + meta?: AttachmentMeta; + error?: string; +} + // While Deep Research is toggled on, poll connected sources for sync // progress so we can surface "Searching over N items — sync in progress" // next to the toggle. Polling is gated on `enabled` so toggling DR off @@ -100,7 +119,76 @@ export function InputArea() { const modelLoading = useAppStore((s) => s.modelLoading); const deepResearch = useAppStore((s) => s.deepResearch); const setDeepResearch = useAppStore((s) => s.setDeepResearch); + const webSearch = useAppStore((s) => s.webSearch); + const setWebSearch = useAppStore((s) => s.setWebSearch); + const setLastAssistantWebSearch = useAppStore((s) => s.setLastAssistantWebSearch); const corpusSync = useResearchCorpusSync(deepResearch); + + const [pending, setPending] = useState([]); + const [dragging, setDragging] = useState(false); + const fileInputRef = useRef(null); + const readyAttachments = pending.filter((p) => p.status === 'ready' && p.meta); + const attachmentsReading = pending.some((p) => p.status === 'reading'); + + const addFiles = useCallback(async (incoming: File[]) => { + let files = incoming; + if (files.length === 0) return; + const room = MAX_FILES_PER_MESSAGE - pending.filter((p) => p.status !== 'error').length; + if (room <= 0) { + toast.error(`Attach at most ${MAX_FILES_PER_MESSAGE} files per message.`); + return; + } + if (files.length > room) { + toast.error(`Only the first ${room} file(s) were added (${MAX_FILES_PER_MESSAGE} per message).`); + files = files.slice(0, room); + } + for (const f of files.filter((f) => f.size > MAX_FILE_BYTES)) { + toast.error(`${f.name} is larger than 25 MB.`); + } + files = files.filter((f) => f.size <= MAX_FILE_BYTES); + if (files.length === 0) return; + + const keys = files.map(() => generateId()); + setPending((prev) => [ + ...prev, + ...files.map((f, i) => ({ key: keys[i], name: f.name, status: 'reading' as const })), + ]); + try { + const results = await readAttachments(files); + await Promise.all(results.map((res) => savePayload(res.meta.id, res.payload))); + setPending((prev) => + prev.map((p) => { + const i = keys.indexOf(p.key); + return i === -1 ? p : { ...p, status: 'ready', meta: results[i]?.meta }; + }), + ); + } catch (err: any) { + const message = err?.message || 'Could not read the file.'; + toast.error(message); + setPending((prev) => + prev.map((p) => (keys.includes(p.key) ? { ...p, status: 'error', error: message } : p)), + ); + } + }, [pending]); + + const removePending = useCallback((key: string) => { + setPending((prev) => { + const item = prev.find((p) => p.key === key); + if (item?.meta) void deletePayloads([item.meta.id]); + return prev.filter((p) => p.key !== key); + }); + }, []); + + // Web search and Deep Research are separate modes; turning one on turns + // the other off. + const toggleWebSearch = () => { + setWebSearch(!webSearch); + if (!webSearch && deepResearch) setDeepResearch(false); + }; + const toggleDeepResearch = () => { + setDeepResearch(!deepResearch); + if (!deepResearch && webSearch) setWebSearch(false); + }; const isCurrentChatStreaming = streamState.isStreaming && streamState.conversationId === activeId; const { @@ -173,13 +261,24 @@ export function InputArea() { const sendMessage = useCallback(async () => { const content = input.trim(); - if (!content || streamState.isStreaming) return; + const attachments = readyAttachments.map((p) => p.meta!); + if ((!content && attachments.length === 0) || streamState.isStreaming) return; + if (attachmentsReading) { + toast.error('Wait for the attached files to finish reading.'); + return; + } if (!selectedModel) { toast.error('Pick a model first (⌘K)'); return; } + if (deepResearch && attachments.length > 0) { + toast.error("Deep Research can't use attached files. Turn it off to ask about them."); + return; + } + const searchWeb = webSearch && !deepResearch; setInput(''); + setPending([]); let convId = activeId; if (!convId) { @@ -191,15 +290,20 @@ export function InputArea() { role: 'user', content, timestamp: Date.now(), + ...(attachments.length > 0 ? { attachments } : {}), }; addMessage(convId, userMsg); - // Build API messages before adding assistant placeholder + // Build API messages before adding assistant placeholder. Attachment + // content is read back from IndexedDB for every message that has some. const currentMessages = useAppStore.getState().messages; - const apiMessages = currentMessages.map((m) => ({ - role: m.role, - content: m.content, - })); + const apiMessages = await Promise.all( + currentMessages.map(async (m) => ({ + role: m.role, + content: m.content, + ...(m.role === 'user' ? await apiAttachmentFields(m.attachments) : {}), + })), + ); const assistantMsg: ChatMessage = { id: generateId(), @@ -235,7 +339,7 @@ export function InputArea() { setStreamState({ conversationId: convId, isStreaming: true, - phase: deepResearch ? 'Researching...' : 'Generating...', + phase: deepResearch ? 'Researching...' : searchWeb ? 'Searching the web...' : 'Generating...', elapsedMs: 0, activeToolCalls: [], content: '', @@ -376,12 +480,44 @@ export function InputArea() { } } else { for await (const sseEvent of streamChat( - { model: selectedModel, messages: apiMessages, stream: true, temperature, max_tokens: maxTokens }, + { + model: selectedModel, + messages: apiMessages, + stream: true, + temperature, + max_tokens: maxTokens, + ...(searchWeb ? { web_search: true } : {}), + }, controller.signal, )) { const eventName = sseEvent.event; - if (eventName === 'agent_turn_start') { + if (eventName === 'web_search') { + try { + const data = JSON.parse(sseEvent.data); + const sources: ResearchSource[] = (data.sources ?? []).map((src: any) => ({ + ref: src.ref, + title: src.title, + url: src.url, + })); + setLastAssistantWebSearch( + convId, + { query: data.query, engine: data.engine || undefined, error: data.error || undefined }, + sources, + ); + setStreamState({ + phase: data.error ? 'Web search failed, answering without it...' : 'Generating...', + }); + useAppStore.getState().addLogEntry({ + timestamp: Date.now(), + level: data.error ? 'warn' : 'info', + category: 'chat', + message: data.error + ? `Web search failed: ${data.error}` + : `Web search found ${sources.length} source(s)`, + }); + } catch {} + } else if (eventName === 'agent_turn_start') { setStreamState({ phase: 'Agent thinking...' }); } else if (eventName === 'inference_start') { setStreamState({ phase: 'Generating...' }); @@ -552,6 +688,10 @@ export function InputArea() { deepResearch, temperature, maxTokens, + readyAttachments, + attachmentsReading, + webSearch, + setLastAssistantWebSearch, ]); const handleKeyDown = (e: React.KeyboardEvent) => { @@ -567,7 +707,27 @@ export function InputArea() {
+